diff options
| author | Matias Elo <matias.elo@nokia.com> | 2022-07-01 11:44:35 +0300 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2022-07-01 11:44:35 +0300 |
| commit | 3b9d1514e55955953786211f1fdec8e97f3688c7 (patch) | |
| tree | 1d8ffcc8573670792ec788fd4a006103ff5e8a8e | |
| parent | 58fe883d5e43dc315d4751433f73c5d8ed7e5ba7 (diff) | |
| parent | 3f7e35f89ff349d531542d73ba854de9695568f4 (diff) | |
Merge ODP v1.37.1.0v1.37.1.0_DPDK_19.11
Merge ODP linux-generic v1.37.1.0 into linux-dpdk
82 files changed, 2280 insertions, 990 deletions
@@ -1,3 +1,17 @@ +== OpenDataPlane (1.37.1.0) + +=== Backward compatible API changes +==== Packet IO +* Clarify that `odp_pktout_send_lso()` can be used also for packets that have +payload less than `max_payload_len` bytes. + +==== Stash +* Change 32-bit and 64-bit specific get/put functions' parameter names to avoid +name conflicts. + +==== Traffic Manager +* Add option to do rate limiting instead of rate shaping in TM nodes and queues. + == OpenDataPlane (1.37.0.0) === Backward incompatible API changes diff --git a/DEPENDENCIES b/DEPENDENCIES index b18279da2..6d5433094 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -336,26 +336,39 @@ Prerequisites for building the OpenDataPlane (ODP) API 3.6.3 Running ODP with AF_XDP socket packet I/O - At the moment, each AF_XDP socket packet I/O binds to a single TRX queue, - this means that NIC(s) of the environment have to be configured accordingly. + AF_XDP socket packet I/Os bind to TRX-combined queues. Based on the packet + prosessing needs, NIC(s) of the environment should be configured + accordingly. - $ ethtool -L <if name> combined 1 + $ ethtool -L <if name> combined <count> + + For example, with four (4) TRX queues, packet processing can be divided to + four (4) processing cores. Additionally, with some NICs (e.g. Mellanox), when zero-copy XDP is in use, the queue configuration is adjusted by the NIC with additional queues on top - of the configured single TRX queue. This requires a forwarding rule: + of the configured amount of TRX queues which are then used as the actual + queues. This will require additional forwarding rules as otherwise packets + do not get forwarded to the zero-copy queues. + + For example: + + ethtool -N <if name> flow-type ether dst <mac of if> action 5 - $ ethtool -N <if name> flow-type ether dst <mac of if> action 1 + Would forward Ethernet frames with a given destination address to queue 5, + which should be one of the zero-copy queues, i.e. queue index on top of the + amount configured with "ethtool -L" option. - Which queue to bind to in a given interface can be controlled with an + Which queues to bind to a given interface can be controlled with an environment variable when starting an ODP executable: $ ODP_PKTIO_XDP_PARAMS="<if name>:<queue index> <if name>:<queue index> ..." ./<odp executable> ... parameter being a string of interface-queue index pairs, where interface and - queue is separated by a colon and pairs separated by a whitespace. If no - environment variable is passed, zero (0) queue is chosen for all AF_XDP - interfaces. + queue are separated by a colon and pairs separated by a whitespace. The queue + index dictates the starting queue in case of multiple TRX queues per packet + I/O. If no environment variable is passed, zero (0) starting queue is chosen + for all AF_XDP interfaces. 4.0 Packages needed to build API tests diff --git a/config/odp-linux-generic.conf b/config/odp-linux-generic.conf index 9534d2a5d..9c5e85242 100644 --- a/config/odp-linux-generic.conf +++ b/config/odp-linux-generic.conf @@ -16,7 +16,7 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.20" +config_file_version = "0.1.21" # System options system: { @@ -149,6 +149,17 @@ pktio_netmap: { } } +# XDP pktio options +pktio_xdp: { + # Number of RX and TX descriptors to be reserved for AF_XDP socket + # memory. Adjusting these may improve performance depending on NIC ring + # configuration. In zero-copy mode, packet pools used as pktio pools + # need to be large enough to accommodate RX and TX descriptors of every + # pktio queue. Values must be a power of two. + num_rx_desc = 1024 + num_tx_desc = 1024 +} + queue_basic: { # Maximum queue size. Value must be a power of two. max_queue_size = 8192 diff --git a/configure.ac b/configure.ac index 8e1ffbd61..16045b942 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ AC_PREREQ([2.5]) ########################################################################## m4_define([odpapi_generation_version], [1]) m4_define([odpapi_major_version], [37]) -m4_define([odpapi_minor_version], [0]) +m4_define([odpapi_minor_version], [1]) m4_define([odpapi_point_version], [0]) m4_define([odpapi_version], [odpapi_generation_version.odpapi_major_version.odpapi_minor_version.odpapi_point_version]) diff --git a/include/odp/api/spec/packet_io.h b/include/odp/api/spec/packet_io.h index 4a8635d0a..3b636c42a 100644 --- a/include/odp/api/spec/packet_io.h +++ b/include/odp/api/spec/packet_io.h @@ -532,6 +532,9 @@ int odp_lso_profile_destroy(odp_lso_profile_t lso_profile); * performance as a number of packet metadata writes/reads are avoided. Results are undefined if * 'lso_opt' is NULL and a packet misses LSO options. * + * Packets with less than (or equal to) 'max_payload_len' payload bytes can be sent also, however + * odp_pktout_send() should be more optimal for those than this function. + * * Check LSO support level from packet IO capabilities (odp_pktio_capability_t). * * @param queue Packet output queue handle diff --git a/include/odp/api/spec/stash.h b/include/odp/api/spec/stash.h index 12214e200..81a6281fa 100644 --- a/include/odp/api/spec/stash.h +++ b/include/odp/api/spec/stash.h @@ -313,14 +313,14 @@ int32_t odp_stash_put(odp_stash_t stash, const void *obj, int32_t num); * integers. The stash must have been created with 'obj_size' of 4. * * @param stash Stash handle - * @param u32 Points to an array of 32-bit integers to be stored. The array + * @param val Points to an array of 32-bit integers to be stored. The array * must be 32-bit aligned in memory. * @param num Number of integers to store * * @return Number of integers actually stored (0 ... num) * @retval <0 on failure */ -int32_t odp_stash_put_u32(odp_stash_t stash, const uint32_t u32[], int32_t num); +int32_t odp_stash_put_u32(odp_stash_t stash, const uint32_t val[], int32_t num); /** * Put 64-bit integers into a stash @@ -329,14 +329,14 @@ int32_t odp_stash_put_u32(odp_stash_t stash, const uint32_t u32[], int32_t num); * integers. The stash must have been created with 'obj_size' of 8. * * @param stash Stash handle - * @param u64 Points to an array of 64-bit integers to be stored. The array + * @param val Points to an array of 64-bit integers to be stored. The array * must be 64-bit aligned in memory. * @param num Number of integers to store * * @return Number of integers actually stored (0 ... num) * @retval <0 on failure */ -int32_t odp_stash_put_u64(odp_stash_t stash, const uint64_t u64[], int32_t num); +int32_t odp_stash_put_u64(odp_stash_t stash, const uint64_t val[], int32_t num); /** * Put pointers into a stash @@ -380,14 +380,14 @@ int32_t odp_stash_get(odp_stash_t stash, void *obj, int32_t num); * integers. The stash must have been created with 'obj_size' of 4. * * @param stash Stash handle - * @param[out] u32 Points to an array of 32-bit integers for output. The + * @param[out] val Points to an array of 32-bit integers for output. The * array must be 32-bit aligned in memory. * @param num Maximum number of integers to get from the stash * - * @return Number of integers actually output (0 ... num) to 'u32' array + * @return Number of integers actually output (0 ... num) to 'val' array * @retval <0 on failure */ -int32_t odp_stash_get_u32(odp_stash_t stash, uint32_t u32[], int32_t num); +int32_t odp_stash_get_u32(odp_stash_t stash, uint32_t val[], int32_t num); /** * Get 64-bit integers from a stash @@ -396,14 +396,14 @@ int32_t odp_stash_get_u32(odp_stash_t stash, uint32_t u32[], int32_t num); * integers. The stash must have been created with 'obj_size' of 8. * * @param stash Stash handle - * @param[out] u64 Points to an array of 64-bit integers for output. The + * @param[out] val Points to an array of 64-bit integers for output. The * array must be 64-bit aligned in memory. * @param num Maximum number of integers to get from the stash * - * @return Number of integers actually output (0 ... num) to 'u64' array + * @return Number of integers actually output (0 ... num) to 'val' array * @retval <0 on failure */ -int32_t odp_stash_get_u64(odp_stash_t stash, uint64_t u64[], int32_t num); +int32_t odp_stash_get_u64(odp_stash_t stash, uint64_t val[], int32_t num); /** * Get pointers from a stash diff --git a/include/odp/api/spec/traffic_mngr.h b/include/odp/api/spec/traffic_mngr.h index e10f6e9e2..f634cfb44 100644 --- a/include/odp/api/spec/traffic_mngr.h +++ b/include/odp/api/spec/traffic_mngr.h @@ -268,7 +268,7 @@ typedef struct { */ uint32_t min_burst_packets; - /** Maximum allowed value fand odp_tm_shaper_params_t::commit_burst and + /** Maximum allowed value for odp_tm_shaper_params_t::commit_burst and * odp_tm_shaper_params_t::peak_burst when * odp_tm_shaper_params_t::packet_mode is true. */ @@ -310,10 +310,12 @@ typedef struct { */ uint64_t max_rate; - /** tm_node_shaper_supported indicates that the tm_nodes at this level - * all support TM shaping, */ + /** Shaper is supported in rate shape mode */ odp_bool_t tm_node_shaper_supported; + /** Shaper is supported in rate limit mode */ + odp_bool_t tm_node_rate_limiter_supported; + /** tm_node_shaper_packet_mode indicates that tm_nodes at this level * support shaper in packet mode */ odp_bool_t tm_node_shaper_packet_mode; @@ -410,16 +412,12 @@ typedef struct { */ odp_bool_t egress_fcn_supported; - /** tm_queue_shaper_supported indicates that the tm_queues support - * proper TM shaping. Note that TM Shaping is NOT the same thing as - * Ingress Metering/Policing as specified by RFC 2697 (A Single Rate - * Three Color Marker) or RFC 2698 (A Two Rate Three Color Marker). - * These RFCs can be used for a Diffserv traffic conditioner, or - * other ingress policing. They make no mention of and have no - * algorithms for delaying packets - which is what TM shapers are - * expected to do. */ + /** Shaper is supported in rate shape mode */ odp_bool_t tm_queue_shaper_supported; + /** Shaper is supported in rate limit mode */ + odp_bool_t tm_queue_rate_limiter_supported; + /** tm_queue_shaper_packet_mode indicates that tm_queues support * shaper in packet mode */ odp_bool_t tm_queue_shaper_packet_mode; @@ -1047,6 +1045,31 @@ int odp_tm_drop_prec_marking(odp_tm_t tm, /* Shaper profile types and functions * -------------------------------------------------------- */ +/** Mode selection between rate shaping and rate limiting */ +typedef enum { + /** Rate shape traffic to the specified burst and rate by delaying + * packets. + * + * The shaper does not drop packets in normal operation, but since + * it delays packets, it can cause queues to fill up and cause queue + * management to drop packets. + */ + ODP_TM_SHAPER_RATE_SHAPE, + + /** Rate limit traffic to the specified burst and rate by dropping + * excess packets. + * + * It is implementation dependent when exactly the limiter state + * update and packet drop happens. For example, they may occur + * immediately when packets are available from the source or when + * the downstream node and scheduler are accepting new packets from + * this node/queue. It is possible that in some cases a delayed + * packet drop causes queues to fill up. + */ + ODP_TM_SHAPER_RATE_LIMIT + +} odp_tm_shaper_mode_t; + /** * TM shaper parameters * @@ -1054,6 +1077,19 @@ int odp_tm_drop_prec_marking(odp_tm_t tm, * values. */ typedef struct { + /** Shaper mode. The default value is ODP_TM_SHAPER_RATE_SHAPE. + * + * A shaper profile must not be used in a TM queue or TM node if + * the queue/node does not support shaper or if it does not support + * the shaper mode set in the profile. + * + * @see odp_tm_capabilities_t::tm_queue_shaper_supported + * @see odp_tm_capabilities_t::tm_queue_rate_limiter_supported + * @see odp_tm_level_capabilities_t::tm_node_shaper_supported + * @see odp_tm_level_capabilities_t::tm_node_rate_limiter_supported + */ + odp_tm_shaper_mode_t mode; + /** The committed information rate for this shaper profile. The units * for this integer is in bits per second when packet_mode is * not TRUE while packets per second when packet mode is TRUE. @@ -1118,6 +1154,7 @@ typedef struct { * * The default value is false. */ odp_bool_t packet_mode; + } odp_tm_shaper_params_t; /** diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index b3365be71..620aeddb4 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -28,11 +28,11 @@ if !ODP_ABI_COMPAT odpapiplatincludedir= $(includedir)/odp/api/plat odpapiplatinclude_HEADERS = \ include/odp/api/plat/atomic_inlines.h \ - include/odp/api/plat/buffer_inline_types.h \ include/odp/api/plat/buffer_inlines.h \ include/odp/api/plat/byteorder_inlines.h \ include/odp/api/plat/cpu_inlines.h \ include/odp/api/plat/event_inlines.h \ + include/odp/api/plat/event_inline_types.h \ include/odp/api/plat/event_vector_inline_types.h \ include/odp/api/plat/hash_inlines.h \ include/odp/api/plat/packet_flag_inlines.h \ @@ -43,6 +43,8 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/pool_inline_types.h \ include/odp/api/plat/queue_inlines.h \ include/odp/api/plat/queue_inline_types.h \ + include/odp/api/plat/spinlock_inlines.h \ + include/odp/api/plat/spinlock_recursive_inlines.h \ include/odp/api/plat/std_inlines.h \ include/odp/api/plat/strong_types.h \ include/odp/api/plat/sync_inlines.h \ @@ -182,7 +184,7 @@ __LIB__libodp_dpdk_la_SOURCES = \ ../linux-generic/odp_dma.c \ odp_crypto.c \ odp_errno.c \ - ../linux-generic/odp_event.c \ + odp_event.c \ ../linux-generic/odp_hash_crc_gen.c \ odp_init.c \ ../linux-generic/odp_impl.c \ @@ -218,8 +220,6 @@ __LIB__libodp_dpdk_la_SOURCES = \ ../linux-generic/odp_schedule_sp.c \ odp_shared_memory.c \ ../linux-generic/odp_sorted_list.c \ - ../linux-generic/odp_spinlock.c \ - ../linux-generic/odp_spinlock_recursive.c \ ../linux-generic/odp_stash.c \ ../linux-generic/odp_std.c \ odp_system_info.c \ @@ -245,6 +245,8 @@ __LIB__libodp_dpdk_la_SOURCES += \ ../linux-generic/odp_packet_flags_api.c \ ../linux-generic/odp_packet_io_api.c \ ../linux-generic/odp_queue_api.c \ + ../linux-generic/odp_spinlock_api.c \ + ../linux-generic/odp_spinlock_recursive_api.c \ odp_std_api.c \ ../linux-generic/odp_sync_api.c \ ../linux-generic/odp_thread_api.c \ diff --git a/platform/linux-dpdk/include/odp/api/plat/buffer_inline_types.h b/platform/linux-dpdk/include/odp/api/plat/buffer_inline_types.h deleted file mode 100644 index f59df6705..000000000 --- a/platform/linux-dpdk/include/odp/api/plat/buffer_inline_types.h +++ /dev/null @@ -1,36 +0,0 @@ -/* Copyright (c) 2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_PLAT_BUFFER_INLINE_TYPES_H_ -#define ODP_PLAT_BUFFER_INLINE_TYPES_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include <stdint.h> - -/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ - -/* Buffer header field accessor */ -#define _odp_buf_hdr_field(buf_hdr, cast, field) \ - (*(cast *)(uintptr_t)((uint8_t *)buf_hdr + \ - _odp_buffer_inline_offset.field)) - -/* Buffer header field offsets for inline functions */ -typedef struct _odp_buffer_inline_offset_t { - uint16_t event_type; - uint16_t base_data; - -} _odp_buffer_inline_offset_t; - -/** @endcond */ - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h b/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h index 9e0ae3f50..f536f728f 100644 --- a/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2019, Nokia +/* Copyright (c) 2019-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -10,11 +10,11 @@ #include <odp/api/abi/buffer.h> #include <odp/api/abi/event_types.h> -#include <odp/api/plat/buffer_inline_types.h> +#include <odp/api/plat/event_inline_types.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -extern const _odp_buffer_inline_offset_t _odp_buffer_inline_offset; +extern const _odp_event_inline_offset_t _odp_event_inline_offset; #ifndef _ODP_NO_INLINE /* Inline functions by default */ @@ -38,7 +38,7 @@ _ODP_INLINE odp_event_t odp_buffer_to_event(odp_buffer_t buf) _ODP_INLINE void *odp_buffer_addr(odp_buffer_t buf) { - return _odp_buf_hdr_field(buf, void *, base_data); + return _odp_event_hdr_field(buf, void *, base_data); } /** @endcond */ diff --git a/platform/linux-dpdk/include/odp/api/plat/event_inline_types.h b/platform/linux-dpdk/include/odp/api/plat/event_inline_types.h new file mode 120000 index 000000000..8899b80a6 --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/event_inline_types.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/event_inline_types.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/spinlock_inlines.h b/platform/linux-dpdk/include/odp/api/plat/spinlock_inlines.h new file mode 120000 index 000000000..bbec045a5 --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/spinlock_inlines.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/spinlock_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/spinlock_recursive_inlines.h b/platform/linux-dpdk/include/odp/api/plat/spinlock_recursive_inlines.h new file mode 120000 index 000000000..580627ade --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/plat/spinlock_recursive_inlines.h @@ -0,0 +1 @@ +../../../../../linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp_buffer_internal.h b/platform/linux-dpdk/include/odp_buffer_internal.h index 3b99961c1..e7b1d215d 100644 --- a/platform/linux-dpdk/include/odp_buffer_internal.h +++ b/platform/linux-dpdk/include/odp_buffer_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2021, Nokia + * Copyright (c) 2021-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -82,20 +82,6 @@ static inline odp_buffer_hdr_t *_odp_buf_hdr(odp_buffer_t buf) return (odp_buffer_hdr_t *)(uintptr_t)buf; } -static inline uint32_t event_flow_id(odp_event_t ev) -{ - odp_buffer_hdr_t *buf_hdr = (odp_buffer_hdr_t *)(uintptr_t)ev; - - return buf_hdr->event_hdr.flow_id; -} - -static inline void event_flow_id_set(odp_event_t ev, uint32_t flow_id) -{ - odp_buffer_hdr_t *buf_hdr = (odp_buffer_hdr_t *)(uintptr_t)ev; - - buf_hdr->event_hdr.flow_id = flow_id; -} - #ifdef __cplusplus } #endif diff --git a/platform/linux-dpdk/include/odp_event_internal.h b/platform/linux-dpdk/include/odp_event_internal.h index b3ce6c795..0614bbeae 100644 --- a/platform/linux-dpdk/include/odp_event_internal.h +++ b/platform/linux-dpdk/include/odp_event_internal.h @@ -79,11 +79,6 @@ static inline struct rte_mbuf *_odp_event_to_mbuf(odp_event_t event) return (struct rte_mbuf *)(uintptr_t)event; } -static inline odp_event_type_t _odp_event_type(odp_event_t event) -{ - return _odp_event_hdr(event)->event_type; -} - static inline void _odp_event_type_set(odp_event_t event, int ev) { _odp_event_hdr(event)->event_type = ev; diff --git a/platform/linux-dpdk/include/odp_packet_internal.h b/platform/linux-dpdk/include/odp_packet_internal.h index 02d7ff7b0..f1ae8dab0 100644 --- a/platform/linux-dpdk/include/odp_packet_internal.h +++ b/platform/linux-dpdk/include/odp_packet_internal.h @@ -410,7 +410,7 @@ int _odp_packet_udp_chksum_insert(odp_packet_t pkt); int _odp_packet_sctp_chksum_insert(odp_packet_t pkt); int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, - odp_proto_chksums_t chksums, uint64_t l4_part_sum); + odp_pktin_config_opt_t opt, uint64_t l4_part_sum); #ifdef __cplusplus } diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h b/platform/linux-dpdk/include/odp_packet_io_internal.h index 59410eef6..91f8350a8 100644 --- a/platform/linux-dpdk/include/odp_packet_io_internal.h +++ b/platform/linux-dpdk/include/odp_packet_io_internal.h @@ -111,14 +111,13 @@ struct pktio_entry { classifier_t cls; /**< classifier linked with this pktio*/ /* Driver level statistics counters */ odp_pktio_stats_t stats; - /* Statistics counters used outside drivers */ + /* Statistics counters used also outside drivers */ struct { odp_atomic_u64_t in_discards; odp_atomic_u64_t out_discards; } stats_extra; /* Latest Tx timestamp */ odp_atomic_u64_t tx_ts; - odp_proto_chksums_t in_chksums; /**< Checksums validation settings */ char name[PKTIO_NAME_LEN]; /**< name of pktio provided to internal pktio_open() calls */ char full_name[PKTIO_NAME_LEN]; /**< original pktio name passed to @@ -333,6 +332,27 @@ int _odp_lso_create_packets(odp_packet_t packet, const odp_packet_lso_opt_t *lso void _odp_pktio_allocate_and_send_tx_compl_events(const pktio_entry_t *entry, const odp_packet_t packets[], int num); +static inline int _odp_pktio_packet_to_pool(odp_packet_t *pkt, + odp_packet_hdr_t **pkt_hdr, + odp_pool_t new_pool) +{ + odp_packet_t new_pkt; + + if (odp_likely(new_pool == odp_packet_pool(*pkt))) + return 0; + + new_pkt = odp_packet_copy(*pkt, new_pool); + + if (odp_unlikely(new_pkt == ODP_PACKET_INVALID)) + return 1; + + odp_packet_free(*pkt); + *pkt = new_pkt; + *pkt_hdr = packet_hdr(new_pkt); + + return 0; +} + #ifdef __cplusplus } #endif diff --git a/platform/linux-dpdk/odp_buffer.c b/platform/linux-dpdk/odp_buffer.c index 21956be2f..58d6eaf83 100644 --- a/platform/linux-dpdk/odp_buffer.c +++ b/platform/linux-dpdk/odp_buffer.c @@ -1,30 +1,20 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2021, Nokia + * Copyright (c) 2021-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ #include <odp/api/buffer.h> + #include <odp_buffer_internal.h> #include <odp_debug_internal.h> #include <odp_pool_internal.h> -#include <odp/api/plat/buffer_inline_types.h> #include <string.h> #include <stdio.h> #include <inttypes.h> -#include <odp/visibility_begin.h> - -/* Fill in buffer header field offsets for inline functions */ -const _odp_buffer_inline_offset_t _odp_buffer_inline_offset ODP_ALIGNED_CACHE = { - .event_type = offsetof(odp_buffer_hdr_t, event_hdr.event_type), - .base_data = offsetof(odp_buffer_hdr_t, event_hdr.mb.buf_addr) -}; - -#include <odp/visibility_end.h> - uint32_t odp_buffer_size(odp_buffer_t buf) { struct rte_mbuf *mbuf = _odp_buf_to_mbuf(buf); diff --git a/platform/linux-dpdk/odp_event.c b/platform/linux-dpdk/odp_event.c new file mode 100644 index 000000000..f5b502205 --- /dev/null +++ b/platform/linux-dpdk/odp_event.c @@ -0,0 +1,127 @@ +/* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2020-2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/event.h> +#include <odp/api/buffer.h> +#include <odp/api/crypto.h> +#include <odp/api/dma.h> +#include <odp/api/packet.h> +#include <odp/api/timer.h> +#include <odp/api/pool.h> + +#include <odp_buffer_internal.h> +#include <odp_ipsec_internal.h> +#include <odp_debug_internal.h> +#include <odp_packet_internal.h> +#include <odp_event_internal.h> +#include <odp_event_vector_internal.h> + +/* Inlined API functions */ +#include <odp/api/plat/event_inlines.h> +#include <odp/api/plat/packet_inlines.h> +#include <odp/api/plat/packet_vector_inlines.h> +#include <odp/api/plat/timer_inlines.h> + +#include <odp/api/plat/event_inline_types.h> + +#include <odp/visibility_begin.h> + +/* Fill in event header field offsets for inline functions */ +const _odp_event_inline_offset_t +_odp_event_inline_offset ODP_ALIGNED_CACHE = { + .event_type = offsetof(_odp_event_hdr_t, event_type), + .base_data = offsetof(_odp_event_hdr_t, mb.buf_addr), + .flow_id = offsetof(_odp_event_hdr_t, flow_id) +}; + +#include <odp/visibility_end.h> + +void odp_event_free(odp_event_t event) +{ + switch (odp_event_type(event)) { + case ODP_EVENT_BUFFER: + odp_buffer_free(odp_buffer_from_event(event)); + break; + case ODP_EVENT_PACKET: + odp_packet_free(odp_packet_from_event(event)); + break; + case ODP_EVENT_PACKET_VECTOR: + _odp_packet_vector_free_full(odp_packet_vector_from_event(event)); + break; + case ODP_EVENT_TIMEOUT: + odp_timeout_free(odp_timeout_from_event(event)); + break; +#if ODP_DEPRECATED_API + case ODP_EVENT_CRYPTO_COMPL: + odp_crypto_compl_free(odp_crypto_compl_from_event(event)); + break; +#endif + case ODP_EVENT_IPSEC_STATUS: + _odp_ipsec_status_free(_odp_ipsec_status_from_event(event)); + break; + case ODP_EVENT_PACKET_TX_COMPL: + odp_packet_tx_compl_free(odp_packet_tx_compl_from_event(event)); + break; + case ODP_EVENT_DMA_COMPL: + odp_dma_compl_free(odp_dma_compl_from_event(event)); + break; + default: + ODP_ABORT("Invalid event type: %d\n", odp_event_type(event)); + } +} + +void odp_event_free_multi(const odp_event_t event[], int num) +{ + int i; + + for (i = 0; i < num; i++) + odp_event_free(event[i]); +} + +void odp_event_free_sp(const odp_event_t event[], int num) +{ + odp_event_free_multi(event, num); +} + +uint64_t odp_event_to_u64(odp_event_t hdl) +{ + return _odp_pri(hdl); +} + +int odp_event_is_valid(odp_event_t event) +{ + if (event == ODP_EVENT_INVALID) + return 0; + + if (_odp_event_is_valid(event) == 0) + return 0; + + switch (odp_event_type(event)) { + case ODP_EVENT_BUFFER: + /* Fall through */ + case ODP_EVENT_PACKET: + /* Fall through */ + case ODP_EVENT_TIMEOUT: + /* Fall through */ +#if ODP_DEPRECATED_API + case ODP_EVENT_CRYPTO_COMPL: + /* Fall through */ +#endif + case ODP_EVENT_IPSEC_STATUS: + /* Fall through */ + case ODP_EVENT_PACKET_VECTOR: + /* Fall through */ + case ODP_EVENT_DMA_COMPL: + /* Fall through */ + case ODP_EVENT_PACKET_TX_COMPL: + break; + default: + return 0; + } + + return 1; +} diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index daef605d4..c7f166f7e 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -1477,13 +1477,14 @@ int _odp_packet_sctp_chksum_insert(odp_packet_t pkt) return odp_packet_copy_from_mem(pkt, pkt_hdr->p.l4_offset + 8, 4, &sum); } -int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, odp_proto_chksums_t chksums, - uint64_t l4_part_sum) +int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, + odp_pktin_config_opt_t opt, uint64_t l4_part_sum) + { uint32_t frame_len = odp_packet_len(packet_handle(pkt_hdr)); /* UDP chksum == 0 case is covered in parse_udp() */ - if (chksums.chksum.udp && + if (opt.bit.udp_chksum && pkt_hdr->p.input_flags.udp && !pkt_hdr->p.input_flags.ipfrag && !pkt_hdr->p.input_flags.udp_chksum_zero) { @@ -1502,7 +1503,7 @@ int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, odp_proto_chksums_t chksums } } - if (chksums.chksum.tcp && + if (opt.bit.tcp_chksum && pkt_hdr->p.input_flags.tcp && !pkt_hdr->p.input_flags.ipfrag) { uint16_t sum = ~packet_sum(pkt_hdr, @@ -1520,7 +1521,7 @@ int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, odp_proto_chksums_t chksums } } - if (chksums.chksum.sctp && + if (opt.bit.sctp_chksum && pkt_hdr->p.input_flags.sctp && !pkt_hdr->p.input_flags.ipfrag) { uint32_t seg_len = 0; @@ -1609,17 +1610,20 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, } opt.all_bits = 0; + opt.bit.ipv4_chksum = param->chksums.chksum.ipv4; + opt.bit.udp_chksum = param->chksums.chksum.udp; + opt.bit.tcp_chksum = param->chksums.chksum.tcp; + opt.bit.sctp_chksum = param->chksums.chksum.sctp; ret = _odp_packet_parse_common_l3_l4(&pkt_hdr->p, data, offset, packet_len, seg_len, layer, - ethtype, param->chksums, - &l4_part_sum, opt); + ethtype, &l4_part_sum, opt); if (ret) return -1; if (layer >= ODP_PROTO_LAYER_L4) { - ret = _odp_packet_l4_chksum(pkt_hdr, param->chksums, l4_part_sum); + ret = _odp_packet_l4_chksum(pkt_hdr, opt, l4_part_sum); if (ret) return -1; } diff --git a/platform/linux-dpdk/odp_packet_dpdk.c b/platform/linux-dpdk/odp_packet_dpdk.c index b44e0bda8..b0f19ea4f 100644 --- a/platform/linux-dpdk/odp_packet_dpdk.c +++ b/platform/linux-dpdk/odp_packet_dpdk.c @@ -930,7 +930,7 @@ static inline void prefetch_pkt(odp_packet_t pkt) odp_prefetch(&pkt_hdr->p); } -int _odp_input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int num) +static inline int input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int num) { pkt_dpdk_t * const pkt_dpdk = pkt_priv(pktio_entry); uint16_t i; @@ -939,6 +939,7 @@ int _odp_input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int nu odp_pktio_t input = pktio_entry->s.handle; odp_time_t ts_val; odp_time_t *ts = NULL; + const uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; uint16_t num_prefetch = RTE_MIN(num, NUM_RX_PREFETCH); const odp_proto_layer_t layer = pktio_entry->s.parse_layer; @@ -966,7 +967,7 @@ int _odp_input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int nu rte_pktmbuf_pkt_len(mbuf), rte_pktmbuf_data_len(mbuf), mbuf, layer, - pkt_dpdk->supported_ptypes, pktin_cfg)) { + supported_ptypes, pktin_cfg)) { odp_packet_free(pkt); continue; } @@ -1017,18 +1018,24 @@ int _odp_input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int nu return num_pkts; } +int _odp_input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int num) +{ + return input_pkts(pktio_entry, pkt_table, num); +} + static int recv_pkt_dpdk(pktio_entry_t *pktio_entry, int index, odp_packet_t pkt_table[], int num) { pkt_dpdk_t * const pkt_dpdk = pkt_priv(pktio_entry); uint16_t nb_rx; - uint8_t min = pkt_dpdk->min_rx_burst; + const uint16_t port_id = pkt_dpdk->port_id; + const uint8_t min = pkt_dpdk->min_rx_burst; if (!pkt_dpdk->flags.lockless_rx) odp_ticketlock_lock(&pkt_dpdk->rx_lock[index]); if (odp_likely(num >= min)) { - nb_rx = rte_eth_rx_burst(pkt_dpdk->port_id, (uint16_t)index, + nb_rx = rte_eth_rx_burst(port_id, (uint16_t)index, (struct rte_mbuf **)pkt_table, (uint16_t)num); } else { @@ -1037,7 +1044,7 @@ static int recv_pkt_dpdk(pktio_entry_t *pktio_entry, int index, ODP_DBG("PMD requires >%d buffers burst. Current %d, dropped " "%d\n", min, num, min - num); - nb_rx = rte_eth_rx_burst(pkt_dpdk->port_id, (uint16_t)index, + nb_rx = rte_eth_rx_burst(port_id, (uint16_t)index, (struct rte_mbuf **)min_burst, min); for (i = 0; i < nb_rx; i++) { @@ -1054,10 +1061,10 @@ static int recv_pkt_dpdk(pktio_entry_t *pktio_entry, int index, odp_ticketlock_unlock(&pkt_dpdk->rx_lock[index]); /* Packets may also me received through eventdev, so don't add any - * processing here. Instead, perform all processing in _odp_input_pkts() + * processing here. Instead, perform all processing in input_pkts() * which is also called by eventdev. */ if (nb_rx) - return _odp_input_pkts(pktio_entry, pkt_table, nb_rx); + return input_pkts(pktio_entry, pkt_table, nb_rx); return 0; } diff --git a/platform/linux-dpdk/odp_system_info.c b/platform/linux-dpdk/odp_system_info.c index c506dbc1e..dcbfe4a06 100644 --- a/platform/linux-dpdk/odp_system_info.c +++ b/platform/linux-dpdk/odp_system_info.c @@ -288,6 +288,21 @@ static int read_config_file(void) return 0; } +static void print_compiler_info(void) +{ + ODP_PRINT("Compiler defines:\n"); + ODP_PRINT(" __GCC_ATOMIC_LLONG_LOCK_FREE: %d\n", __GCC_ATOMIC_LLONG_LOCK_FREE); + ODP_PRINT(" __GCC_ATOMIC_LONG_LOCK_FREE: %d\n", __GCC_ATOMIC_LONG_LOCK_FREE); + ODP_PRINT(" __GCC_ATOMIC_INT_LOCK_FREE: %d\n", __GCC_ATOMIC_INT_LOCK_FREE); + ODP_PRINT(" __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16: "); +#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 + ODP_PRINT("1\n"); +#else + ODP_PRINT("0\n"); +#endif + ODP_PRINT("\n"); +} + /* * System info initialisation */ @@ -341,6 +356,8 @@ int _odp_system_info_init(void) system_hp(&odp_global_ro.hugepage_info); + print_compiler_info(); + return 0; } diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 6e64df740..7cec4799e 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -30,11 +30,11 @@ if !ODP_ABI_COMPAT odpapiplatincludedir= $(includedir)/odp/api/plat odpapiplatinclude_HEADERS = \ include/odp/api/plat/atomic_inlines.h \ - include/odp/api/plat/buffer_inline_types.h \ include/odp/api/plat/buffer_inlines.h \ include/odp/api/plat/byteorder_inlines.h \ include/odp/api/plat/cpu_inlines.h \ include/odp/api/plat/event_inlines.h \ + include/odp/api/plat/event_inline_types.h \ include/odp/api/plat/event_vector_inline_types.h \ include/odp/api/plat/hash_inlines.h \ include/odp/api/plat/packet_flag_inlines.h \ @@ -45,6 +45,8 @@ odpapiplatinclude_HEADERS = \ include/odp/api/plat/pool_inline_types.h \ include/odp/api/plat/queue_inlines.h \ include/odp/api/plat/queue_inline_types.h \ + include/odp/api/plat/spinlock_inlines.h \ + include/odp/api/plat/spinlock_recursive_inlines.h \ include/odp/api/plat/std_inlines.h \ include/odp/api/plat/strong_types.h \ include/odp/api/plat/sync_inlines.h \ @@ -228,8 +230,6 @@ __LIB__libodp_linux_la_SOURCES = \ odp_schedule_sp.c \ odp_shared_memory.c \ odp_sorted_list.c \ - odp_spinlock.c \ - odp_spinlock_recursive.c \ odp_stash.c \ odp_std.c \ odp_system_info.c \ @@ -284,6 +284,8 @@ __LIB__libodp_linux_la_SOURCES += \ odp_packet_flags_api.c \ odp_packet_io_api.c \ odp_queue_api.c \ + odp_spinlock_api.c \ + odp_spinlock_recursive_api.c \ odp_std_api.c \ odp_sync_api.c \ odp_thread_api.c \ diff --git a/platform/linux-generic/include-abi/odp/api/abi/spinlock.h b/platform/linux-generic/include-abi/odp/api/abi/spinlock.h index fbfbce5cc..d1e5fa1e9 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/spinlock.h +++ b/platform/linux-generic/include-abi/odp/api/abi/spinlock.h @@ -5,3 +5,6 @@ */ #include <odp/api/abi-default/spinlock.h> + +/* Include inlined versions of API functions */ +#include <odp/api/plat/spinlock_inlines.h> diff --git a/platform/linux-generic/include-abi/odp/api/abi/spinlock_recursive.h b/platform/linux-generic/include-abi/odp/api/abi/spinlock_recursive.h index cc93b6acb..cdcbae1b4 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/spinlock_recursive.h +++ b/platform/linux-generic/include-abi/odp/api/abi/spinlock_recursive.h @@ -5,3 +5,6 @@ */ #include <odp/api/abi-default/spinlock_recursive.h> + +/* Include inlined versions of API functions */ +#include <odp/api/plat/spinlock_recursive_inlines.h> diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h b/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h deleted file mode 100644 index f59df6705..000000000 --- a/platform/linux-generic/include/odp/api/plat/buffer_inline_types.h +++ /dev/null @@ -1,36 +0,0 @@ -/* Copyright (c) 2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_PLAT_BUFFER_INLINE_TYPES_H_ -#define ODP_PLAT_BUFFER_INLINE_TYPES_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include <stdint.h> - -/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ - -/* Buffer header field accessor */ -#define _odp_buf_hdr_field(buf_hdr, cast, field) \ - (*(cast *)(uintptr_t)((uint8_t *)buf_hdr + \ - _odp_buffer_inline_offset.field)) - -/* Buffer header field offsets for inline functions */ -typedef struct _odp_buffer_inline_offset_t { - uint16_t event_type; - uint16_t base_data; - -} _odp_buffer_inline_offset_t; - -/** @endcond */ - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h index 9e0ae3f50..3da402a83 100644 --- a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2019, Nokia +/* Copyright (c) 2019-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -10,11 +10,11 @@ #include <odp/api/abi/buffer.h> #include <odp/api/abi/event_types.h> -#include <odp/api/plat/buffer_inline_types.h> +#include <odp/api/plat/event_inline_types.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -extern const _odp_buffer_inline_offset_t _odp_buffer_inline_offset; +extern const _odp_event_inline_offset_t _odp_event_inline_offset; #ifndef _ODP_NO_INLINE /* Inline functions by default */ @@ -38,7 +38,7 @@ _ODP_INLINE odp_event_t odp_buffer_to_event(odp_buffer_t buf) _ODP_INLINE void *odp_buffer_addr(odp_buffer_t buf) { - return _odp_buf_hdr_field(buf, void *, base_data); + return _odp_event_hdr_field((odp_event_t)buf, void *, base_data); } /** @endcond */ diff --git a/platform/linux-generic/include/odp/api/plat/event_inline_types.h b/platform/linux-generic/include/odp/api/plat/event_inline_types.h new file mode 100644 index 000000000..c2727ec6f --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/event_inline_types.h @@ -0,0 +1,41 @@ +/* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_EVENT_INLINE_TYPES_H_ +#define ODP_PLAT_EVENT_INLINE_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +/* Event header field accessors */ +#define _odp_event_hdr_field(event_hdr, cast, field) \ + (*(cast *)(uintptr_t)((uint8_t *)event_hdr + \ + _odp_event_inline_offset.field)) +#define _odp_event_hdr_ptr(event_hdr, cast, field) \ + ((cast *)(uintptr_t)((uint8_t *)event_hdr + \ + _odp_event_inline_offset.field)) + +/* Event header field offsets for inline functions */ +typedef struct _odp_event_inline_offset_t { + uint16_t event_type; + uint16_t base_data; + uint16_t flow_id; + +} _odp_event_inline_offset_t; + +/** @endcond */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/event_inlines.h b/platform/linux-generic/include/odp/api/plat/event_inlines.h index 43ee5e09a..27c2a82fd 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_inlines.h @@ -1,4 +1,5 @@ /* Copyright (c) 2018, Linaro Limited + * Copyright (c) 2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,18 +8,26 @@ #ifndef ODP_PLAT_EVENT_INLINES_H_ #define ODP_PLAT_EVENT_INLINES_H_ -#include <odp/api/abi/buffer.h> -#include <odp/api/plat/buffer_inline_types.h> +#include <odp/api/event_types.h> +#include <odp/api/packet.h> + +#include <odp/api/plat/event_inline_types.h> /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ -extern const _odp_buffer_inline_offset_t _odp_buffer_inline_offset; +extern const _odp_event_inline_offset_t _odp_event_inline_offset; #ifndef _ODP_NO_INLINE /* Inline functions by default */ #define _ODP_INLINE static inline #define odp_event_type __odp_event_type #define odp_event_type_multi __odp_event_type_multi + #define odp_event_subtype __odp_event_subtype + #define odp_event_types __odp_event_types + #define odp_event_flow_id __odp_event_flow_id + #define odp_event_flow_id_set __odp_event_flow_id_set + + #include <odp/api/plat/packet_inlines.h> #else #define _ODP_INLINE #endif @@ -26,9 +35,8 @@ extern const _odp_buffer_inline_offset_t _odp_buffer_inline_offset; static inline odp_event_type_t __odp_event_type_get(odp_event_t event) { int8_t type; - odp_buffer_t buf = (odp_buffer_t)event; - type = _odp_buf_hdr_field(buf, int8_t, event_type); + type = _odp_event_hdr_field(event, int8_t, event_type); return (odp_event_type_t)type; } @@ -54,6 +62,38 @@ _ODP_INLINE int odp_event_type_multi(const odp_event_t event[], int num, return i; } +_ODP_INLINE odp_event_subtype_t odp_event_subtype(odp_event_t event) +{ + if (__odp_event_type_get(event) != ODP_EVENT_PACKET) + return ODP_EVENT_NO_SUBTYPE; + + return odp_packet_subtype(odp_packet_from_event(event)); +} + +_ODP_INLINE odp_event_type_t odp_event_types(odp_event_t event, + odp_event_subtype_t *subtype) +{ + odp_event_type_t event_type = __odp_event_type_get(event); + + *subtype = event_type == ODP_EVENT_PACKET ? + odp_packet_subtype(odp_packet_from_event(event)) : + ODP_EVENT_NO_SUBTYPE; + + return event_type; +} + +_ODP_INLINE uint32_t odp_event_flow_id(odp_event_t event) +{ + return _odp_event_hdr_field(event, uint8_t, flow_id); +} + +_ODP_INLINE void odp_event_flow_id_set(odp_event_t event, uint32_t id) +{ + uint8_t *flow_id = _odp_event_hdr_ptr(event, uint8_t, flow_id); + + *flow_id = (uint8_t)id; +} + /** @endcond */ #endif diff --git a/platform/linux-generic/include/odp/api/plat/spinlock_inlines.h b/platform/linux-generic/include/odp/api/plat/spinlock_inlines.h new file mode 100644 index 000000000..a04c43f88 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/spinlock_inlines.h @@ -0,0 +1,65 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_SPINLOCK_INLINES_H_ +#define ODP_PLAT_SPINLOCK_INLINES_H_ + +#include <odp/api/cpu.h> + +#include <odp/api/abi/spinlock.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_spinlock_init __odp_spinlock_init + #define odp_spinlock_lock __odp_spinlock_lock + #define odp_spinlock_trylock __odp_spinlock_trylock + #define odp_spinlock_unlock __odp_spinlock_unlock + #define odp_spinlock_is_locked __odp_spinlock_is_locked + + #include <odp/api/plat/cpu_inlines.h> +#else + #undef _ODP_INLINE + #define _ODP_INLINE +#endif + +_ODP_INLINE void odp_spinlock_init(odp_spinlock_t *spinlock) +{ + __atomic_clear(&spinlock->lock, __ATOMIC_RELAXED); +} + +_ODP_INLINE void odp_spinlock_lock(odp_spinlock_t *spinlock) +{ + /* While the lock is already taken... */ + while (__atomic_test_and_set(&spinlock->lock, __ATOMIC_ACQUIRE)) + /* ...spin reading the flag (relaxed MM), + * the loop will exit when the lock becomes available + * and we will retry the TAS operation above */ + while (__atomic_load_n(&spinlock->lock, __ATOMIC_RELAXED)) + odp_cpu_pause(); +} + +_ODP_INLINE int odp_spinlock_trylock(odp_spinlock_t *spinlock) +{ + return (__atomic_test_and_set(&spinlock->lock, __ATOMIC_ACQUIRE) == 0); +} + +_ODP_INLINE void odp_spinlock_unlock(odp_spinlock_t *spinlock) +{ + __atomic_clear(&spinlock->lock, __ATOMIC_RELEASE); +} + +_ODP_INLINE int odp_spinlock_is_locked(odp_spinlock_t *spinlock) +{ + return __atomic_load_n(&spinlock->lock, __ATOMIC_RELAXED) != 0; +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h b/platform/linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h new file mode 100644 index 000000000..2dd846fe9 --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/spinlock_recursive_inlines.h @@ -0,0 +1,91 @@ +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_SPINLOCK_RECURSIVE_INLINES_H_ +#define ODP_PLAT_SPINLOCK_RECURSIVE_INLINES_H_ + +#include <odp/api/spinlock.h> +#include <odp/api/thread.h> + +#include <odp/api/abi/spinlock_recursive.h> + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + +#ifndef _ODP_NO_INLINE + /* Inline functions by default */ + #define _ODP_INLINE static inline + #define odp_spinlock_recursive_init __odp_spinlock_recursive_init + #define odp_spinlock_recursive_lock __odp_spinlock_recursive_lock + #define odp_spinlock_recursive_trylock __odp_spinlock_recursive_trylock + #define odp_spinlock_recursive_unlock __odp_spinlock_recursive_unlock + #define odp_spinlock_recursive_is_locked __odp_spinlock_recursive_is_locked + + #include <odp/api/plat/spinlock_inlines.h> + #include <odp/api/plat/thread_inlines.h> +#else + #undef _ODP_INLINE + #define _ODP_INLINE +#endif + +_ODP_INLINE void odp_spinlock_recursive_init(odp_spinlock_recursive_t *rlock) +{ + odp_spinlock_init(&rlock->lock); + rlock->owner = -1; + rlock->cnt = 0; +} + +_ODP_INLINE void odp_spinlock_recursive_lock(odp_spinlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->owner == thr) { + rlock->cnt++; + return; + } + + odp_spinlock_lock(&rlock->lock); + rlock->owner = thr; + rlock->cnt = 1; +} + +_ODP_INLINE int odp_spinlock_recursive_trylock(odp_spinlock_recursive_t *rlock) +{ + int thr = odp_thread_id(); + + if (rlock->owner == thr) { + rlock->cnt++; + return 1; + } + + if (odp_spinlock_trylock(&rlock->lock)) { + rlock->owner = thr; + rlock->cnt = 1; + return 1; + } + + return 0; +} + +_ODP_INLINE void odp_spinlock_recursive_unlock(odp_spinlock_recursive_t *rlock) +{ + rlock->cnt--; + + if (rlock->cnt > 0) + return; + + rlock->owner = -1; + odp_spinlock_unlock(&rlock->lock); +} + +_ODP_INLINE int odp_spinlock_recursive_is_locked(odp_spinlock_recursive_t *rlock) +{ + return odp_thread_id() == rlock->owner ? 1 : odp_spinlock_is_locked(&rlock->lock); +} + +/** @endcond */ + +#endif diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 5841720ef..8625fc5dd 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -50,20 +50,6 @@ static inline odp_buffer_hdr_t *_odp_buf_hdr(odp_buffer_t buf) return (odp_buffer_hdr_t *)(uintptr_t)buf; } -static inline uint32_t event_flow_id(odp_event_t ev) -{ - odp_buffer_hdr_t *buf_hdr = (odp_buffer_hdr_t *)(uintptr_t)ev; - - return buf_hdr->event_hdr.flow_id; -} - -static inline void event_flow_id_set(odp_event_t ev, uint32_t flow_id) -{ - odp_buffer_hdr_t *buf_hdr = (odp_buffer_hdr_t *)(uintptr_t)ev; - - buf_hdr->event_hdr.flow_id = flow_id; -} - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 872d6f6d5..d3d09abf4 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -179,6 +179,16 @@ extern "C" { */ #define CONFIG_IPSEC_MAX_NUM_SA 4000 +/* + * Use 128-bit atomics for timer implementation (if available) + * + * On some platforms 128-bit atomic operations may be available, but the + * implementation of used 128-bit GCC built-in functions (e.g. + * __atomic_compare_exchange_n) utilizes expensive locking. Set to zero to use + * ODP lock based implementation instead. + */ +#define CONFIG_TIMER_128BIT_ATOMICS 1 + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_event_internal.h b/platform/linux-generic/include/odp_event_internal.h index 92f201b01..60788fd52 100644 --- a/platform/linux-generic/include/odp_event_internal.h +++ b/platform/linux-generic/include/odp_event_internal.h @@ -79,11 +79,6 @@ static inline _odp_event_hdr_t *_odp_event_hdr(odp_event_t event) return (_odp_event_hdr_t *)(uintptr_t)event; } -static inline odp_event_type_t _odp_event_type(odp_event_t event) -{ - return _odp_event_hdr(event)->event_type; -} - static inline void _odp_event_type_set(odp_event_t event, int ev) { _odp_event_hdr(event)->event_type = ev; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index e54d88f6a..977b3dda0 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -478,7 +478,7 @@ int _odp_packet_udp_chksum_insert(odp_packet_t pkt); int _odp_packet_sctp_chksum_insert(odp_packet_t pkt); int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, - odp_proto_chksums_t chksums, uint64_t l4_part_sum); + odp_pktin_config_opt_t opt, uint64_t l4_part_sum); #ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h index ca9f083da..0505d4378 100644 --- a/platform/linux-generic/include/odp_packet_io_internal.h +++ b/platform/linux-generic/include/odp_packet_io_internal.h @@ -65,12 +65,16 @@ struct pktio_if_ops; #if defined(_ODP_PKTIO_NETMAP) #define PKTIO_PRIVATE_SIZE 74752 +#elif defined(_ODP_PKTIO_XDP) && ODP_CACHE_LINE_SIZE == 128 +#define PKTIO_PRIVATE_SIZE 33792 +#elif defined(_ODP_PKTIO_XDP) +#define PKTIO_PRIVATE_SIZE 29696 #elif defined(_ODP_PKTIO_DPDK) && ODP_CACHE_LINE_SIZE == 128 #define PKTIO_PRIVATE_SIZE 10240 #elif defined(_ODP_PKTIO_DPDK) #define PKTIO_PRIVATE_SIZE 5632 #else -#define PKTIO_PRIVATE_SIZE 512 +#define PKTIO_PRIVATE_SIZE 384 #endif struct pktio_entry { @@ -124,14 +128,13 @@ struct pktio_entry { classifier_t cls; /**< classifier linked with this pktio*/ /* Driver level statistics counters */ odp_pktio_stats_t stats; - /* Statistics counters used outside drivers */ + /* Statistics counters used also outside drivers */ struct { odp_atomic_u64_t in_discards; odp_atomic_u64_t out_discards; } stats_extra; /* Latest Tx timestamp */ odp_atomic_u64_t tx_ts; - odp_proto_chksums_t in_chksums; /**< Checksums validation settings */ pktio_stats_type_t stats_type; char name[PKTIO_NAME_LEN]; /**< name of pktio provided to internal pktio_open() calls */ @@ -357,6 +360,27 @@ int _odp_lso_create_packets(odp_packet_t packet, const odp_packet_lso_opt_t *lso void _odp_pktio_allocate_and_send_tx_compl_events(const pktio_entry_t *entry, const odp_packet_t packets[], int num); +static inline int _odp_pktio_packet_to_pool(odp_packet_t *pkt, + odp_packet_hdr_t **pkt_hdr, + odp_pool_t new_pool) +{ + odp_packet_t new_pkt; + + if (odp_likely(new_pool == odp_packet_pool(*pkt))) + return 0; + + new_pkt = odp_packet_copy(*pkt, new_pool); + + if (odp_unlikely(new_pkt == ODP_PACKET_INVALID)) + return 1; + + odp_packet_free(*pkt); + *pkt = new_pkt; + *pkt_hdr = packet_hdr(new_pkt); + + return 0; +} + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_parse_internal.h b/platform/linux-generic/include/odp_parse_internal.h index 22d8c2cf6..c467abbcd 100644 --- a/platform/linux-generic/include/odp_parse_internal.h +++ b/platform/linux-generic/include/odp_parse_internal.h @@ -64,7 +64,6 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, uint32_t offset, uint32_t frame_len, uint32_t seg_len, int layer, uint16_t ethtype, - odp_proto_chksums_t chksums, uint64_t *l4_part_sum, odp_pktin_config_opt_t opt); @@ -77,17 +76,18 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, * Returns 0 on success, 1 on packet errors, and -1 if the packet should be * dropped. */ -static inline int _odp_packet_parse_common(packet_parser_t *prs, +static inline int _odp_packet_parse_common(odp_packet_hdr_t *pkt_hdr, const uint8_t *ptr, uint32_t frame_len, uint32_t seg_len, int layer, - odp_proto_chksums_t chksums, - uint64_t *l4_part_sum, odp_pktin_config_opt_t opt) { + int r; uint32_t offset; uint16_t ethtype; const uint8_t *parseptr; + packet_parser_t *prs = &pkt_hdr->p; + uint64_t l4_part_sum = 0; parseptr = ptr; offset = 0; @@ -100,9 +100,14 @@ static inline int _odp_packet_parse_common(packet_parser_t *prs, ethtype = _odp_parse_eth(prs, &parseptr, &offset, frame_len); - return _odp_packet_parse_common_l3_l4(prs, parseptr, offset, frame_len, - seg_len, layer, ethtype, chksums, - l4_part_sum, opt); + r = _odp_packet_parse_common_l3_l4(prs, parseptr, offset, frame_len, + seg_len, layer, ethtype, + &l4_part_sum, opt); + + if (!r && layer >= ODP_PROTO_LAYER_L4) + r = _odp_packet_l4_chksum(pkt_hdr, opt, l4_part_sum); + + return r; } #ifdef __cplusplus diff --git a/platform/linux-generic/m4/odp_libconfig.m4 b/platform/linux-generic/m4/odp_libconfig.m4 index ab4994d61..886cc07e8 100644 --- a/platform/linux-generic/m4/odp_libconfig.m4 +++ b/platform/linux-generic/m4/odp_libconfig.m4 @@ -3,7 +3,7 @@ ########################################################################## m4_define([_odp_config_version_generation], [0]) m4_define([_odp_config_version_major], [1]) -m4_define([_odp_config_version_minor], [20]) +m4_define([_odp_config_version_minor], [21]) m4_define([_odp_config_version], [_odp_config_version_generation._odp_config_version_major._odp_config_version_minor]) diff --git a/platform/linux-generic/m4/odp_xdp.m4 b/platform/linux-generic/m4/odp_xdp.m4 index 2c6179df9..dcfd39ed7 100644 --- a/platform/linux-generic/m4/odp_xdp.m4 +++ b/platform/linux-generic/m4/odp_xdp.m4 @@ -5,7 +5,7 @@ AC_ARG_ENABLE([xdp], AS_HELP_STRING([--enable-xdp], [enable experimental XDP support for Packet I/O [default=disabled] (linux-generic)])) AS_IF([test "x$enable_xdp" = "xyes"], [ - PKG_CHECK_MODULES([LIBXDP], [libxdp], + PKG_CHECK_MODULES([LIBXDP], [libxdp >= 1.2.3], [ AC_DEFINE(_ODP_PKTIO_XDP, [1], [Define to 1 to enable xdp packet I/O support]) ], diff --git a/platform/linux-generic/odp_buffer.c b/platform/linux-generic/odp_buffer.c index f9f67a2e4..df3f047b1 100644 --- a/platform/linux-generic/odp_buffer.c +++ b/platform/linux-generic/odp_buffer.c @@ -6,26 +6,15 @@ */ #include <odp/api/buffer.h> + #include <odp_pool_internal.h> #include <odp_buffer_internal.h> #include <odp_debug_internal.h> -#include <odp/api/plat/buffer_inline_types.h> #include <string.h> #include <stdio.h> #include <inttypes.h> -#include <odp/visibility_begin.h> - -/* Fill in buffer header field offsets for inline functions */ -const _odp_buffer_inline_offset_t -_odp_buffer_inline_offset ODP_ALIGNED_CACHE = { - .event_type = offsetof(odp_buffer_hdr_t, event_hdr.event_type), - .base_data = offsetof(odp_buffer_hdr_t, event_hdr.base_data) -}; - -#include <odp/visibility_end.h> - uint32_t odp_buffer_size(odp_buffer_t buf) { odp_buffer_hdr_t *hdr = _odp_buf_hdr(buf); diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 9f0978b49..9402c805b 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -1957,11 +1957,19 @@ int odp_crypto_capability(odp_crypto_capability_t *capa) /* Initialize crypto capability structure */ memset(capa, 0, sizeof(odp_crypto_capability_t)); + capa->max_sessions = MAX_SESSIONS; capa->sync_mode = ODP_SUPPORT_PREFERRED; capa->async_mode = ODP_SUPPORT_YES; capa->queue_type_plain = 1; capa->queue_type_sched = 1; + /* Memory allocation in libssl is not compatible with process mode */ + if (odp_global_ro.init_param.mem_model == ODP_MEM_MODEL_PROCESS) { + capa->ciphers.bit.null = 1; + capa->auths.bit.null = 1; + return 0; + } + capa->ciphers.bit.null = 1; capa->ciphers.bit.trides_cbc = 1; capa->ciphers.bit.trides_ecb = 1; @@ -2001,8 +2009,6 @@ int odp_crypto_capability(odp_crypto_capability_t *capa) capa->auths.bit.sha384 = 1; capa->auths.bit.sha512 = 1; - capa->max_sessions = MAX_SESSIONS; - return 0; } @@ -2196,6 +2202,16 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param, return -1; } + /* Process mode is not supported with libssl based algos */ + if (odp_global_ro.init_param.mem_model == ODP_MEM_MODEL_PROCESS && + (param->cipher_alg != ODP_CIPHER_ALG_NULL || + param->auth_alg != ODP_AUTH_ALG_NULL)) { + *status = param->cipher_alg != ODP_CIPHER_ALG_NULL ? + ODP_CRYPTO_SES_ERR_CIPHER : ODP_CRYPTO_SES_ERR_AUTH; + *session_out = ODP_CRYPTO_SESSION_INVALID; + return -1; + } + /* Allocate memory for this session */ session = alloc_session(); if (NULL == session) { diff --git a/platform/linux-generic/odp_event.c b/platform/linux-generic/odp_event.c index 90c3e3a3c..50cb03a19 100644 --- a/platform/linux-generic/odp_event.c +++ b/platform/linux-generic/odp_event.c @@ -26,35 +26,19 @@ #include <odp/api/plat/packet_vector_inlines.h> #include <odp/api/plat/timer_inlines.h> -odp_event_subtype_t odp_event_subtype(odp_event_t event) -{ - if (_odp_event_type(event) != ODP_EVENT_PACKET) - return ODP_EVENT_NO_SUBTYPE; - - return odp_packet_subtype(odp_packet_from_event(event)); -} +#include <odp/api/plat/event_inline_types.h> -odp_event_type_t odp_event_types(odp_event_t event, - odp_event_subtype_t *subtype) -{ - odp_event_type_t event_type = _odp_event_type(event); - - *subtype = event_type == ODP_EVENT_PACKET ? - odp_packet_subtype(odp_packet_from_event(event)) : - ODP_EVENT_NO_SUBTYPE; +#include <odp/visibility_begin.h> - return event_type; -} +/* Fill in event header field offsets for inline functions */ +const _odp_event_inline_offset_t +_odp_event_inline_offset ODP_ALIGNED_CACHE = { + .event_type = offsetof(_odp_event_hdr_t, event_type), + .base_data = offsetof(_odp_event_hdr_t, base_data), + .flow_id = offsetof(_odp_event_hdr_t, flow_id) +}; -uint32_t odp_event_flow_id(odp_event_t event) -{ - return event_flow_id(event); -} - -void odp_event_flow_id_set(odp_event_t event, uint32_t flow_id) -{ - event_flow_id_set(event, flow_id); -} +#include <odp/visibility_end.h> void odp_event_free(odp_event_t event) { diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 07e9c2d4d..89de5130d 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -1945,10 +1945,10 @@ int _odp_packet_sctp_chksum_insert(odp_packet_t pkt) } int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, - odp_proto_chksums_t chksums, uint64_t l4_part_sum) + odp_pktin_config_opt_t opt, uint64_t l4_part_sum) { /* UDP chksum == 0 case is covered in parse_udp() */ - if (chksums.chksum.udp && + if (opt.bit.udp_chksum && pkt_hdr->p.input_flags.udp && !pkt_hdr->p.input_flags.ipfrag && !pkt_hdr->p.input_flags.udp_chksum_zero) { @@ -1967,7 +1967,7 @@ int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, } } - if (chksums.chksum.tcp && + if (opt.bit.tcp_chksum && pkt_hdr->p.input_flags.tcp && !pkt_hdr->p.input_flags.ipfrag) { uint16_t sum = ~packet_sum(pkt_hdr, @@ -1985,7 +1985,7 @@ int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, } } - if (chksums.chksum.sctp && + if (opt.bit.sctp_chksum && pkt_hdr->p.input_flags.sctp && !pkt_hdr->p.input_flags.ipfrag) { uint32_t seg_len = 0; @@ -2074,17 +2074,20 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, } opt.all_bits = 0; + opt.bit.ipv4_chksum = param->chksums.chksum.ipv4; + opt.bit.udp_chksum = param->chksums.chksum.udp; + opt.bit.tcp_chksum = param->chksums.chksum.tcp; + opt.bit.sctp_chksum = param->chksums.chksum.sctp; ret = _odp_packet_parse_common_l3_l4(&pkt_hdr->p, data, offset, packet_len, seg_len, layer, - ethtype, param->chksums, - &l4_part_sum, opt); + ethtype, &l4_part_sum, opt); if (ret) return -1; if (layer >= ODP_PROTO_LAYER_L4) { - ret = _odp_packet_l4_chksum(pkt_hdr, param->chksums, l4_part_sum); + ret = _odp_packet_l4_chksum(pkt_hdr, opt, l4_part_sum); if (ret) return -1; } diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 7c68b10c5..02d1a827c 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -612,12 +612,6 @@ int odp_pktio_config(odp_pktio_t hdl, const odp_pktio_config_t *config) entry->s.config = *config; - entry->s.in_chksums.all_chksum = 0; - entry->s.in_chksums.chksum.ipv4 = config->pktin.bit.ipv4_chksum; - entry->s.in_chksums.chksum.tcp = config->pktin.bit.tcp_chksum; - entry->s.in_chksums.chksum.udp = config->pktin.bit.udp_chksum; - entry->s.in_chksums.chksum.sctp = config->pktin.bit.sctp_chksum; - entry->s.enabled.tx_ts = config->pktout.bit.ts_ena; entry->s.enabled.tx_compl = config->pktout.bit.tx_compl_ena; @@ -3102,8 +3096,8 @@ int odp_packet_lso_request(odp_packet_t pkt, const odp_packet_lso_opt_t *lso_opt return -1; } - if (odp_unlikely((payload_offset + lso_opt->max_payload_len) > packet_len(pkt_hdr))) { - ODP_ERR("LSO options larger than packet data length\n"); + if (odp_unlikely(payload_offset > packet_len(pkt_hdr))) { + ODP_ERR("LSO payload offset larger than packet data length\n"); return -1; } @@ -3212,11 +3206,19 @@ int _odp_lso_num_packets(odp_packet_t packet, const odp_packet_lso_opt_t *lso_op return -1; } - if (odp_unlikely((hdr_len + payload_len) > pkt_len)) { - ODP_ERR("LSO options larger than packet data length\n"); + if (odp_unlikely(hdr_len > pkt_len)) { + ODP_ERR("LSO payload offset larger than packet data length\n"); return -1; } + if (odp_unlikely(hdr_len + payload_len > odp_packet_len(packet))) { + /* Packet does not need segmentation */ + *len_out = payload_len; + *left_over_out = 0; + + return 1; + } + if (lso_prof->param.lso_proto == ODP_LSO_PROTO_IPV4) { l3_offset = odp_packet_l3_offset(packet); iphdr_len = hdr_len - l3_offset; @@ -3364,6 +3366,14 @@ static int pktout_send_lso(odp_pktout_queue_t queue, odp_packet_t packet, if (odp_unlikely(num_pkt <= 0)) return -1; + if (odp_unlikely(num_pkt == 1)) { + /* Segmentation not needed */ + if (odp_pktout_send(queue, &packet, 1) != 1) + return -1; + + return 0; + } + /* Create packets */ odp_packet_t pkt_out[num_pkt]; diff --git a/platform/linux-generic/odp_parse.c b/platform/linux-generic/odp_parse.c index 2342a7e49..13935be56 100644 --- a/platform/linux-generic/odp_parse.c +++ b/platform/linux-generic/odp_parse.c @@ -119,7 +119,7 @@ error: */ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, - odp_proto_chksums_t chksums, + odp_pktin_config_opt_t opt, uint64_t *l4_part_sum) { const _odp_ipv4hdr_t *ipv4 = (const _odp_ipv4hdr_t *)*parseptr; @@ -137,7 +137,7 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, return 0; } - if (chksums.chksum.ipv4) { + if (opt.bit.ipv4_chksum) { prs->input_flags.l3_chksum_done = 1; if (chksum_finalize(chksum_partial(ipv4, ihl * 4, 0)) != 0xffff) { prs->flags.ip_err = 1; @@ -149,7 +149,7 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, *offset += ihl * 4; *parseptr += ihl * 4; - if (chksums.chksum.udp || chksums.chksum.tcp) + if (opt.bit.udp_chksum || opt.bit.tcp_chksum) *l4_part_sum = chksum_partial((const uint8_t *)&ipv4->src_addr, 2 * _ODP_IPV4ADDR_LEN, 0); @@ -182,7 +182,7 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, uint32_t seg_len, - odp_proto_chksums_t chksums, + odp_pktin_config_opt_t opt, uint64_t *l4_part_sum) { const _odp_ipv6hdr_t *ipv6 = (const _odp_ipv6hdr_t *)*parseptr; @@ -207,7 +207,7 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, *offset += sizeof(_odp_ipv6hdr_t); *parseptr += sizeof(_odp_ipv6hdr_t); - if (chksums.chksum.udp || chksums.chksum.tcp) + if (opt.bit.udp_chksum || opt.bit.tcp_chksum) *l4_part_sum = chksum_partial((const uint8_t *)&ipv6->src_addr, 2 * _ODP_IPV6ADDR_LEN, 0); @@ -253,7 +253,7 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, */ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t tcp_len, - odp_proto_chksums_t chksums, + odp_pktin_config_opt_t opt, uint64_t *l4_part_sum) { const _odp_tcphdr_t *tcp = (const _odp_tcphdr_t *)*parseptr; @@ -262,7 +262,7 @@ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, if (odp_unlikely(tcp->hl < sizeof(_odp_tcphdr_t) / sizeof(uint32_t))) prs->flags.tcp_err = 1; - if (chksums.chksum.tcp && + if (opt.bit.tcp_chksum && !prs->input_flags.ipfrag) { *l4_part_sum += odp_cpu_to_be_16(tcp_len); #if ODP_BYTE_ORDER == ODP_BIG_ENDIAN @@ -281,7 +281,7 @@ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, * Requires PARSE_UDP_BYTES bytes of contiguous packet data. */ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, - odp_proto_chksums_t chksums, + odp_pktin_config_opt_t opt, uint64_t *l4_part_sum) { const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr; @@ -293,7 +293,7 @@ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, return; } - if (chksums.chksum.udp && + if (opt.bit.udp_chksum && !prs->input_flags.ipfrag) { if (udp->chksum == 0) { prs->input_flags.l4_chksum_done = 1; @@ -330,7 +330,7 @@ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, */ static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t sctp_len, - odp_proto_chksums_t chksums, + odp_pktin_config_opt_t opt, uint64_t *l4_part_sum) { if (odp_unlikely(sctp_len < sizeof(_odp_sctphdr_t))) { @@ -338,7 +338,7 @@ static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, return; } - if (chksums.chksum.sctp && + if (opt.bit.sctp_chksum && !prs->input_flags.ipfrag) { const _odp_sctphdr_t *sctp = (const _odp_sctphdr_t *)*parseptr; @@ -358,7 +358,6 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, uint32_t offset, uint32_t frame_len, uint32_t seg_len, int layer, uint16_t ethtype, - odp_proto_chksums_t chksums, uint64_t *l4_part_sum, odp_pktin_config_opt_t opt) { @@ -377,7 +376,7 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, case _ODP_ETHTYPE_IPV4: prs->input_flags.ipv4 = 1; ip_proto = parse_ipv4(prs, &parseptr, &offset, frame_len, - chksums, l4_part_sum); + opt, l4_part_sum); prs->l4_offset = offset; if (prs->flags.ip_err && opt.bit.drop_ipv4_err) return -1; /* drop */ @@ -386,7 +385,7 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, case _ODP_ETHTYPE_IPV6: prs->input_flags.ipv6 = 1; ip_proto = parse_ipv6(prs, &parseptr, &offset, frame_len, - seg_len, chksums, l4_part_sum); + seg_len, opt, l4_part_sum); prs->l4_offset = offset; if (prs->flags.ip_err && opt.bit.drop_ipv6_err) return -1; /* drop */ @@ -425,7 +424,7 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len)) return -1; prs->input_flags.tcp = 1; - parse_tcp(prs, &parseptr, frame_len - prs->l4_offset, chksums, + parse_tcp(prs, &parseptr, frame_len - prs->l4_offset, opt, l4_part_sum); if (prs->flags.tcp_err && opt.bit.drop_tcp_err) return -1; /* drop */ @@ -435,7 +434,7 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len)) return -1; prs->input_flags.udp = 1; - parse_udp(prs, &parseptr, chksums, l4_part_sum); + parse_udp(prs, &parseptr, opt, l4_part_sum); if (prs->flags.udp_err && opt.bit.drop_udp_err) return -1; /* drop */ break; @@ -452,7 +451,7 @@ int _odp_packet_parse_common_l3_l4(packet_parser_t *prs, case _ODP_IPPROTO_SCTP: prs->input_flags.sctp = 1; - parse_sctp(prs, &parseptr, frame_len - prs->l4_offset, chksums, + parse_sctp(prs, &parseptr, frame_len - prs->l4_offset, opt, l4_part_sum); if (prs->flags.sctp_err && opt.bit.drop_sctp_err) return -1; /* drop */ diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c index 881dbb985..26203fa5a 100644 --- a/platform/linux-generic/odp_queue_scalable.c +++ b/platform/linux-generic/odp_queue_scalable.c @@ -118,6 +118,10 @@ static int queue_init(queue_entry_t *queue, const char *name, ring[ring_idx] = NULL; queue->s.type = queue->s.param.type; + + if (queue->s.type == ODP_QUEUE_TYPE_SCHED) + queue->s.param.deq_mode = ODP_QUEUE_OP_DISABLED; + odp_atomic_init_u64(&queue->s.num_timers, 0); queue->s.enqueue = _queue_enq; diff --git a/platform/linux-generic/odp_spinlock.c b/platform/linux-generic/odp_spinlock.c deleted file mode 100644 index b38cc6a3a..000000000 --- a/platform/linux-generic/odp_spinlock.c +++ /dev/null @@ -1,42 +0,0 @@ -/* Copyright (c) 2013-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp/api/spinlock.h> -#include <odp/api/cpu.h> -#include <odp_atomic_internal.h> - -#include <odp/api/plat/cpu_inlines.h> - -void odp_spinlock_init(odp_spinlock_t *spinlock) -{ - _odp_atomic_flag_init(&spinlock->lock, 0); -} - -void odp_spinlock_lock(odp_spinlock_t *spinlock) -{ - /* While the lock is already taken... */ - while (_odp_atomic_flag_tas(&spinlock->lock)) - /* ...spin reading the flag (relaxed MM), - * the loop will exit when the lock becomes available - * and we will retry the TAS operation above */ - while (_odp_atomic_flag_load(&spinlock->lock)) - odp_cpu_pause(); -} - -int odp_spinlock_trylock(odp_spinlock_t *spinlock) -{ - return (_odp_atomic_flag_tas(&spinlock->lock) == 0); -} - -void odp_spinlock_unlock(odp_spinlock_t *spinlock) -{ - _odp_atomic_flag_clear(&spinlock->lock); -} - -int odp_spinlock_is_locked(odp_spinlock_t *spinlock) -{ - return _odp_atomic_flag_load(&spinlock->lock) != 0; -} diff --git a/platform/linux-generic/odp_spinlock_api.c b/platform/linux-generic/odp_spinlock_api.c new file mode 100644 index 000000000..06925e9a5 --- /dev/null +++ b/platform/linux-generic/odp_spinlock_api.c @@ -0,0 +1,10 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/spinlock.h> + +#define _ODP_NO_INLINE +#include <odp/api/plat/spinlock_inlines.h> diff --git a/platform/linux-generic/odp_spinlock_recursive.c b/platform/linux-generic/odp_spinlock_recursive.c deleted file mode 100644 index 6363a3838..000000000 --- a/platform/linux-generic/odp_spinlock_recursive.c +++ /dev/null @@ -1,71 +0,0 @@ -/* Copyright (c) 2013-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include <odp/api/spinlock_recursive.h> -#include <odp/api/thread.h> -#include <odp/api/plat/thread_inlines.h> - -#define NO_OWNER (-1) - -void odp_spinlock_recursive_init(odp_spinlock_recursive_t *rlock) -{ - odp_spinlock_init(&rlock->lock); - rlock->owner = NO_OWNER; - rlock->cnt = 0; -} - -void odp_spinlock_recursive_lock(odp_spinlock_recursive_t *rlock) -{ - int thr = odp_thread_id(); - - if (rlock->owner == thr) { - rlock->cnt++; - return; - } - - odp_spinlock_lock(&rlock->lock); - rlock->owner = thr; - rlock->cnt = 1; -} - -int odp_spinlock_recursive_trylock(odp_spinlock_recursive_t *rlock) -{ - int thr = odp_thread_id(); - - if (rlock->owner == thr) { - rlock->cnt++; - return 1; - } - - if (odp_spinlock_trylock(&rlock->lock)) { - rlock->owner = thr; - rlock->cnt = 1; - return 1; - } else { - return 0; - } -} - -void odp_spinlock_recursive_unlock(odp_spinlock_recursive_t *rlock) -{ - rlock->cnt--; - - if (rlock->cnt > 0) - return; - - rlock->owner = NO_OWNER; - odp_spinlock_unlock(&rlock->lock); -} - -int odp_spinlock_recursive_is_locked(odp_spinlock_recursive_t *rlock) -{ - int thr = odp_thread_id(); - - if (rlock->owner == thr) - return 1; - - return odp_spinlock_is_locked(&rlock->lock); -} diff --git a/platform/linux-generic/odp_spinlock_recursive_api.c b/platform/linux-generic/odp_spinlock_recursive_api.c new file mode 100644 index 000000000..2b1e8b200 --- /dev/null +++ b/platform/linux-generic/odp_spinlock_recursive_api.c @@ -0,0 +1,10 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/spinlock_recursive.h> + +#define _ODP_NO_INLINE +#include <odp/api/plat/spinlock_recursive_inlines.h> diff --git a/platform/linux-generic/odp_stash.c b/platform/linux-generic/odp_stash.c index e12f1aed3..15d995eaf 100644 --- a/platform/linux-generic/odp_stash.c +++ b/platform/linux-generic/odp_stash.c @@ -166,6 +166,7 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) uint64_t i, ring_size, shm_size; int ring_u64, index; char shm_name[ODP_STASH_NAME_LEN + 8]; + uint32_t shm_flags = 0; if (odp_global_ro.disable.stash) { ODP_ERR("Stash is disabled\n"); @@ -214,7 +215,10 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param) else shm_size = sizeof(stash_t) + (ring_size * sizeof(uint32_t)); - shm = odp_shm_reserve(shm_name, shm_size, ODP_CACHE_LINE_SIZE, 0); + if (odp_global_ro.shm_single_va) + shm_flags |= ODP_SHM_SINGLE_VA; + + shm = odp_shm_reserve(shm_name, shm_size, ODP_CACHE_LINE_SIZE, shm_flags); if (shm == ODP_SHM_INVALID) { ODP_ERR("SHM reserve failed.\n"); @@ -361,7 +365,7 @@ int32_t odp_stash_put(odp_stash_t st, const void *obj, int32_t num) return -1; } -int32_t odp_stash_put_u32(odp_stash_t st, const uint32_t u32[], int32_t num) +int32_t odp_stash_put_u32(odp_stash_t st, const uint32_t val[], int32_t num) { stash_t *stash = (stash_t *)(uintptr_t)st; @@ -371,11 +375,11 @@ int32_t odp_stash_put_u32(odp_stash_t st, const uint32_t u32[], int32_t num) ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); ring_u32_enq_multi(&stash->ring_u32.hdr, stash->ring_mask, - (uint32_t *)(uintptr_t)u32, num); + (uint32_t *)(uintptr_t)val, num); return num; } -int32_t odp_stash_put_u64(odp_stash_t st, const uint64_t u64[], int32_t num) +int32_t odp_stash_put_u64(odp_stash_t st, const uint64_t val[], int32_t num) { stash_t *stash = (stash_t *)(uintptr_t)st; @@ -385,7 +389,7 @@ int32_t odp_stash_put_u64(odp_stash_t st, const uint64_t u64[], int32_t num) ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); ring_u64_enq_multi(&stash->ring_u64.hdr, stash->ring_mask, - (uint64_t *)(uintptr_t)u64, num); + (uint64_t *)(uintptr_t)val, num); return num; } @@ -466,7 +470,7 @@ int32_t odp_stash_get(odp_stash_t st, void *obj, int32_t num) return -1; } -int32_t odp_stash_get_u32(odp_stash_t st, uint32_t u32[], int32_t num) +int32_t odp_stash_get_u32(odp_stash_t st, uint32_t val[], int32_t num) { stash_t *stash = (stash_t *)(uintptr_t)st; @@ -475,11 +479,11 @@ int32_t odp_stash_get_u32(odp_stash_t st, uint32_t u32[], int32_t num) ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); - return ring_u32_deq_multi(&stash->ring_u32.hdr, stash->ring_mask, u32, + return ring_u32_deq_multi(&stash->ring_u32.hdr, stash->ring_mask, val, num); } -int32_t odp_stash_get_u64(odp_stash_t st, uint64_t u64[], int32_t num) +int32_t odp_stash_get_u64(odp_stash_t st, uint64_t val[], int32_t num) { stash_t *stash = (stash_t *)(uintptr_t)st; @@ -488,7 +492,7 @@ int32_t odp_stash_get_u64(odp_stash_t st, uint64_t u64[], int32_t num) ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); - return ring_u64_deq_multi(&stash->ring_u64.hdr, stash->ring_mask, u64, + return ring_u64_deq_multi(&stash->ring_u64.hdr, stash->ring_mask, val, num); } diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index 5665a3ece..69a088032 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -362,6 +362,21 @@ static int read_config_file(void) return 0; } +static void print_compiler_info(void) +{ + ODP_PRINT("Compiler defines:\n"); + ODP_PRINT(" __GCC_ATOMIC_LLONG_LOCK_FREE: %d\n", __GCC_ATOMIC_LLONG_LOCK_FREE); + ODP_PRINT(" __GCC_ATOMIC_LONG_LOCK_FREE: %d\n", __GCC_ATOMIC_LONG_LOCK_FREE); + ODP_PRINT(" __GCC_ATOMIC_INT_LOCK_FREE: %d\n", __GCC_ATOMIC_INT_LOCK_FREE); + ODP_PRINT(" __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16: "); +#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 + ODP_PRINT("1\n"); +#else + ODP_PRINT("0\n"); +#endif + ODP_PRINT("\n"); +} + /* * System info initialisation */ @@ -415,6 +430,8 @@ int _odp_system_info_init(void) system_hp(&odp_global_ro.hugepage_info); + print_compiler_info(); + return 0; } @@ -612,5 +629,6 @@ void odp_sys_config_print(void) ODP_PRINT("CONFIG_BURST_SIZE: %i\n", CONFIG_BURST_SIZE); ODP_PRINT("CONFIG_POOL_MAX_NUM: %i\n", CONFIG_POOL_MAX_NUM); ODP_PRINT("CONFIG_POOL_CACHE_MAX_SIZE: %i\n", CONFIG_POOL_CACHE_MAX_SIZE); + ODP_PRINT("CONFIG_TIMER_128BIT_ATOMICS: %i\n", CONFIG_TIMER_128BIT_ATOMICS); ODP_PRINT("\n"); } diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index 3af1ae737..9457dd4c5 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -38,6 +38,7 @@ #include <odp/api/plat/timer_inline_types.h> #include <odp_atomic_internal.h> +#include <odp_config_internal.h> #include <odp_debug_internal.h> #include <odp_errno_define.h> #include <odp_event_internal.h> @@ -60,6 +61,13 @@ #include <time.h> #include <unistd.h> +/* Check whether 128-bit atomics should be used */ +#if defined(ODP_ATOMIC_U128) && CONFIG_TIMER_128BIT_ATOMICS +#define USE_128BIT_ATOMICS 1 +#else +#define USE_128BIT_ATOMICS 0 +#endif + /* One divided by one nanosecond in Hz */ #define GIGA_HZ 1000000000 @@ -92,9 +100,9 @@ #define MAX_PERIODIC_TIMERS 100 /* Mutual exclusion in the absence of CAS16 */ -#ifndef ODP_ATOMIC_U128 -#define NUM_LOCKS 1024 -#define IDX2LOCK(idx) (&timer_global->locks[(idx) % NUM_LOCKS]) +#if !USE_128BIT_ATOMICS +#define NUM_LOCKS 256 +#define IDX2LOCK(tp, idx) (&(tp)->locks[(idx) % NUM_LOCKS]) #endif #include <odp/visibility_begin.h> @@ -110,20 +118,12 @@ _odp_timeout_inline_offset ODP_ALIGNED_CACHE = { #include <odp/visibility_end.h> typedef struct -#ifdef ODP_ATOMIC_U128 +#if USE_128BIT_ATOMICS ODP_ALIGNED(16) /* 16-byte atomic operations need properly aligned addresses */ #endif tick_buf_s { -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - /* No atomics support for 64-bit variables, will use separate lock */ - /* Use the same layout as odp_atomic_u64_t but without lock variable */ - struct { - uint64_t v; - } exp_tck;/* Expiration tick or TMO_xxx */ -#else - odp_atomic_u64_t exp_tck;/* Expiration tick or TMO_xxx */ -#endif - + /* Expiration tick or TMO_xxx */ + odp_atomic_u64_t exp_tck; union { /* ODP_EVENT_INVALID if timer not active */ odp_event_t tmo_event; @@ -134,8 +134,7 @@ tick_buf_s { } tick_buf_t; -#if __GCC_ATOMIC_LLONG_LOCK_FREE >= 2 -/* Only assert this when we perform atomic operations on tick_buf_t */ +#ifndef ODP_ATOMIC_U64_LOCK ODP_STATIC_ASSERT(sizeof(tick_buf_t) == 16, "sizeof(tick_buf_t) == 16"); #endif @@ -176,6 +175,10 @@ typedef struct timer_pool_s { double base_freq; uint64_t max_multiplier; uint8_t periodic; +#if !USE_128BIT_ATOMICS + /* Multiple locks per cache line! */ + _odp_atomic_flag_t locks[NUM_LOCKS] ODP_ALIGNED_CACHE; +#endif } timer_pool_t; @@ -199,10 +202,7 @@ typedef struct timer_global_t { odp_time_t destroy_time[MAX_TIMER_POOLS]; odp_shm_t tp_shm[MAX_TIMER_POOLS]; timer_pool_t *timer_pool[MAX_TIMER_POOLS]; -#ifndef ODP_ATOMIC_U128 - /* Multiple locks per cache line! */ - _odp_atomic_flag_t locks[NUM_LOCKS] ODP_ALIGNED_CACHE; -#endif + /* These are read frequently from inline timer */ odp_time_t poll_interval_time; odp_bool_t use_inline_timers; @@ -237,11 +237,7 @@ static void timer_init(_odp_timer_t *tim, tick_buf_t *tb, odp_queue_t _q, const tb->tmo_event = ODP_EVENT_INVALID; /* Release the timer by setting timer state to inactive */ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - tb->exp_tck.v = TMO_INACTIVE; -#else odp_atomic_store_rel_u64(&tb->exp_tck, TMO_INACTIVE); -#endif } /* Teardown when timer is freed */ @@ -279,14 +275,15 @@ static inline odp_timer_pool_t timer_pool_to_hdl(timer_pool_t *tp) static inline timer_pool_t *handle_to_tp(odp_timer_t hdl) { uint32_t tp_idx = _odp_typeval(hdl) >> INDEX_BITS; + timer_pool_t *tp; - if (odp_likely(tp_idx < MAX_TIMER_POOLS)) { - timer_pool_t *tp = timer_global->timer_pool[tp_idx]; + ODP_ASSERT(tp_idx < MAX_TIMER_POOLS); - if (odp_likely(tp != NULL)) - return timer_global->timer_pool[tp_idx]; - } - ODP_ABORT("Invalid timer handle %p\n", (void *)hdl); + tp = timer_global->timer_pool[tp_idx]; + + ODP_ASSERT(tp != NULL); + + return tp; } static inline uint32_t handle_to_idx(odp_timer_t hdl, @@ -294,10 +291,11 @@ static inline uint32_t handle_to_idx(odp_timer_t hdl, { uint32_t idx = (_odp_typeval(hdl) & ((1U << INDEX_BITS) - 1U)) - 1; + ODP_ASSERT(idx < odp_atomic_load_u32(&tp->high_wm)); + __builtin_prefetch(&tp->tick_buf[idx], 0, 0); - if (odp_likely(idx < odp_atomic_load_u32(&tp->high_wm))) - return idx; - ODP_ABORT("Invalid timer handle %p\n", (void *)hdl); + + return idx; } static inline odp_timer_t tp_idx_to_handle(timer_pool_t *tp, @@ -486,16 +484,17 @@ static odp_timer_pool_t timer_pool_new(const char *name, tp->tick_buf = (void *)((char *)odp_shm_addr(shm) + sz0); tp->timers = (void *)((char *)odp_shm_addr(shm) + sz0 + sz1); +#if !USE_128BIT_ATOMICS + for (i = 0; i < NUM_LOCKS; i++) + _odp_atomic_flag_clear(&tp->locks[i]); +#endif + /* Initialize all odp_timer entries */ for (i = 0; i < tp->param.num_timers; i++) { tp->timers[i].queue = ODP_QUEUE_INVALID; set_next_free(&tp->timers[i], i + 1); tp->timers[i].user_ptr = NULL; -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - tp->tick_buf[i].exp_tck.v = TMO_UNUSED; -#else odp_atomic_init_u64(&tp->tick_buf[i].exp_tck, TMO_UNUSED); -#endif tp->tick_buf[i].tmo_event = ODP_EVENT_INVALID; } tp->tp_idx = tp_idx; @@ -677,7 +676,7 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event, tick_buf_t *tb = &tp->tick_buf[idx]; if (tmo_event == NULL || *tmo_event == ODP_EVENT_INVALID) { -#ifdef ODP_ATOMIC_U128 /* Target supports 128-bit atomic operations */ +#if USE_128BIT_ATOMICS /* Target supports 128-bit atomic operations */ tick_buf_t new, old; /* Init all bits, also when tmo_event is less than 64 bits */ @@ -706,35 +705,11 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event, } while (!_odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb, (_odp_u128_t *)&old, (_odp_u128_t *)&new, _ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX)); -#elif __GCC_ATOMIC_LLONG_LOCK_FREE >= 2 && \ - defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 - /* Target supports lock-free 64-bit CAS (and probably exchange) */ - /* Since locks/barriers are not good for C-A15, we take an - * alternative approach using relaxed memory model */ - uint64_t old; - /* Swap in new expiration tick, get back old tick which - * will indicate active/inactive timer state */ - old = odp_atomic_xchg_u64(&tb->exp_tck, abs_tck); - - if ((old & TMO_INACTIVE) != 0) { - /* Timer was inactive (cancelled or expired), - * we can't reset a timer without a timeout event. - * Attempt to restore inactive state, we don't - * want this timer to continue as active without - * timeout as this will trigger unnecessary and - * aborted expiration attempts. - * We don't care if we fail, then some other thread - * reset or cancelled the timer. Without any - * synchronization between the threads, we have a - * data race and the behavior is undefined */ - (void)odp_atomic_cas_u64(&tb->exp_tck, &abs_tck, old); - success = false; - } -#else /* Target supports neither 128-bit nor 64-bit CAS => use lock */ +#else /* Take a related lock */ - while (_odp_atomic_flag_tas(IDX2LOCK(idx))) + while (_odp_atomic_flag_tas(IDX2LOCK(tp, idx))) /* While lock is taken, spin using relaxed loads */ - while (_odp_atomic_flag_load(IDX2LOCK(idx))) + while (_odp_atomic_flag_load(IDX2LOCK(tp, idx))) odp_cpu_pause(); /* Only if there is a timeout event can the timer be reset */ @@ -748,7 +723,7 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event, } /* Release the lock */ - _odp_atomic_flag_clear(IDX2LOCK(idx)); + _odp_atomic_flag_clear(IDX2LOCK(tp, idx)); #endif } else { /* We have a new timeout event which replaces any old one */ @@ -763,7 +738,7 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event, } /* Else ignore events of other types */ odp_event_t old_event = ODP_EVENT_INVALID; -#ifdef ODP_ATOMIC_U128 +#if USE_128BIT_ATOMICS tick_buf_t new, old; /* Init all bits, also when tmo_event is less than 64 bits */ @@ -781,9 +756,9 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event, old_event = old.tmo_event; #else /* Take a related lock */ - while (_odp_atomic_flag_tas(IDX2LOCK(idx))) + while (_odp_atomic_flag_tas(IDX2LOCK(tp, idx))) /* While lock is taken, spin using relaxed loads */ - while (_odp_atomic_flag_load(IDX2LOCK(idx))) + while (_odp_atomic_flag_load(IDX2LOCK(tp, idx))) odp_cpu_pause(); /* Swap in new event, save any old event */ @@ -794,7 +769,7 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event, tb->exp_tck.v = abs_tck; /* Release the lock */ - _odp_atomic_flag_clear(IDX2LOCK(idx)); + _odp_atomic_flag_clear(IDX2LOCK(tp, idx)); #endif /* Return old timeout event */ *tmo_event = old_event; @@ -807,7 +782,7 @@ static odp_event_t timer_set_unused(timer_pool_t *tp, uint32_t idx) tick_buf_t *tb = &tp->tick_buf[idx]; odp_event_t old_event; -#ifdef ODP_ATOMIC_U128 +#if USE_128BIT_ATOMICS tick_buf_t new, old; /* Init all bits, also when tmo_event is less than 64 bits */ @@ -824,9 +799,9 @@ static odp_event_t timer_set_unused(timer_pool_t *tp, uint32_t idx) old_event = old.tmo_event; #else /* Take a related lock */ - while (_odp_atomic_flag_tas(IDX2LOCK(idx))) + while (_odp_atomic_flag_tas(IDX2LOCK(tp, idx))) /* While lock is taken, spin using relaxed loads */ - while (_odp_atomic_flag_load(IDX2LOCK(idx))) + while (_odp_atomic_flag_load(IDX2LOCK(tp, idx))) odp_cpu_pause(); /* Update the timer state (e.g. cancel the current timeout) */ @@ -837,7 +812,7 @@ static odp_event_t timer_set_unused(timer_pool_t *tp, uint32_t idx) tb->tmo_event = ODP_EVENT_INVALID; /* Release the lock */ - _odp_atomic_flag_clear(IDX2LOCK(idx)); + _odp_atomic_flag_clear(IDX2LOCK(tp, idx)); #endif /* Return the old event */ return old_event; @@ -848,7 +823,7 @@ static odp_event_t timer_cancel(timer_pool_t *tp, uint32_t idx) tick_buf_t *tb = &tp->tick_buf[idx]; odp_event_t old_event; -#ifdef ODP_ATOMIC_U128 +#if USE_128BIT_ATOMICS tick_buf_t new, old; /* Init all bits, also when tmo_event is less than 64 bits */ @@ -880,9 +855,9 @@ static odp_event_t timer_cancel(timer_pool_t *tp, uint32_t idx) old_event = old.tmo_event; #else /* Take a related lock */ - while (_odp_atomic_flag_tas(IDX2LOCK(idx))) + while (_odp_atomic_flag_tas(IDX2LOCK(tp, idx))) /* While lock is taken, spin using relaxed loads */ - while (_odp_atomic_flag_load(IDX2LOCK(idx))) + while (_odp_atomic_flag_load(IDX2LOCK(tp, idx))) odp_cpu_pause(); /* Swap in new event, save any old event */ @@ -896,7 +871,7 @@ static odp_event_t timer_cancel(timer_pool_t *tp, uint32_t idx) tb->exp_tck.v = TMO_INACTIVE; /* Release the lock */ - _odp_atomic_flag_clear(IDX2LOCK(idx)); + _odp_atomic_flag_clear(IDX2LOCK(tp, idx)); #endif /* Return the old event */ return old_event; @@ -908,7 +883,7 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) tick_buf_t *tb = &tp->tick_buf[idx]; odp_event_t tmo_event = ODP_EVENT_INVALID; uint64_t exp_tck; -#ifdef ODP_ATOMIC_U128 +#if USE_128BIT_ATOMICS /* Atomic re-read for correctness */ exp_tck = odp_atomic_load_u64(&tb->exp_tck); /* Re-check exp_tck */ @@ -940,11 +915,10 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) } /* Else false positive, ignore */ #else - /* Take a related lock */ - while (_odp_atomic_flag_tas(IDX2LOCK(idx))) - /* While lock is taken, spin using relaxed loads */ - while (_odp_atomic_flag_load(IDX2LOCK(idx))) - odp_cpu_pause(); + /* Try to take a related lock */ + if (_odp_atomic_flag_tas(IDX2LOCK(tp, idx))) + return; + /* Proper check for timer expired */ exp_tck = tb->exp_tck.v; if (odp_likely(exp_tck <= tick)) { @@ -963,7 +937,7 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) } /* Else false positive, ignore */ /* Release the lock */ - _odp_atomic_flag_clear(IDX2LOCK(idx)); + _odp_atomic_flag_clear(IDX2LOCK(tp, idx)); #endif if (odp_likely(tmo_event != ODP_EVENT_INVALID)) { /* Fill in expiration tick for timeout events */ @@ -1823,11 +1797,8 @@ int odp_timeout_fresh(odp_timeout_t tmo) timer_pool_t *tp = handle_to_tp(hdl); uint32_t idx = handle_to_idx(hdl, tp); tick_buf_t *tb = &tp->tick_buf[idx]; -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - uint64_t exp_tck = tb->exp_tck.v; -#else uint64_t exp_tck = odp_atomic_load_u64(&tb->exp_tck); -#endif + /* Return true if the timer still has the same expiration tick * (ignoring the inactive/expired bit) as the timeout */ return hdr->expiration == (exp_tck & ~TMO_INACTIVE); @@ -1970,18 +1941,21 @@ int _odp_timer_init_global(const odp_init_t *params) timer_global->tp_shm[i] = ODP_SHM_INVALID; } -#ifndef ODP_ATOMIC_U128 - for (i = 0; i < NUM_LOCKS; i++) - _odp_atomic_flag_clear(&timer_global->locks[i]); +#if USE_128BIT_ATOMICS + ODP_PRINT("Timer using lock-less implementation\n"); #else - ODP_DBG("Using lock-less timer implementation\n"); + ODP_PRINT("Timer using lock-based implementation\n"); #endif + + ODP_PRINT("Timer config:\n"); + conf_str = "timer.inline"; if (!_odp_libconfig_lookup_int(conf_str, &val)) { ODP_ERR("Config option '%s' not found.\n", conf_str); goto error; } timer_global->use_inline_timers = val; + ODP_PRINT(" %s: %i\n", conf_str, val); conf_str = "timer.inline_poll_interval"; if (!_odp_libconfig_lookup_int(conf_str, &val)) { @@ -1989,6 +1963,7 @@ int _odp_timer_init_global(const odp_init_t *params) goto error; } timer_global->poll_interval = val; + ODP_PRINT(" %s: %i\n", conf_str, val); conf_str = "timer.inline_poll_interval_nsec"; if (!_odp_libconfig_lookup_int(conf_str, &val)) { @@ -1998,6 +1973,7 @@ int _odp_timer_init_global(const odp_init_t *params) timer_global->poll_interval_nsec = val; timer_global->poll_interval_time = odp_time_global_from_ns(timer_global->poll_interval_nsec); + ODP_PRINT(" %s: %i\n", conf_str, val); conf_str = "timer.inline_thread_type"; if (!_odp_libconfig_lookup_int(conf_str, &val)) { @@ -2005,6 +1981,8 @@ int _odp_timer_init_global(const odp_init_t *params) goto error; } timer_global->thread_type = val; + ODP_PRINT(" %s: %i\n", conf_str, val); + ODP_PRINT("\n"); if (!timer_global->use_inline_timers) { timer_res_init(); diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index 9bea659e9..9a64a0aab 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -2603,6 +2603,9 @@ static int tm_capabilities(odp_tm_capabilities_t capabilities[], cap_ptr = &capabilities[0]; memset(cap_ptr, 0, sizeof(odp_tm_capabilities_t)); + if (odp_global_ro.init_param.mem_model == ODP_MEM_MODEL_PROCESS) + return 1; + cap_ptr->max_tm_queues = ODP_TM_MAX_TM_QUEUES; cap_ptr->max_levels = ODP_TM_MAX_LEVELS; cap_ptr->tm_queue_shaper_supported = true; @@ -3060,6 +3063,11 @@ odp_tm_t odp_tm_create(const char *name, return ODP_TM_INVALID; } + if (odp_global_ro.init_param.mem_model == ODP_MEM_MODEL_PROCESS) { + ODP_ERR("TM is not supported in process mode\n"); + return ODP_TM_INVALID; + } + /* We only support global pkt priority mode */ if (requirements->pkt_prio_mode != ODP_TM_PKT_PRIO_MODE_PRESERVE) { ODP_ERR("Unsupported Packet priority mode\n"); @@ -4640,6 +4648,17 @@ int odp_tm_enq_multi_lso(odp_tm_queue_t tm_queue, const odp_packet_t packets[], goto error; } + if (odp_unlikely(num_pkt == 1)) { + /* Segmentation not needed */ + if (odp_tm_enq_multi(tm_queue, &pkt, 1) != 1) { + ODP_DBG("TM enqueue failed on packet %i\n", i); + + goto error; + } + + continue; + } + /* Create packets */ odp_packet_t pkt_out[num_pkt]; diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 006344b48..80578c9b0 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -612,6 +612,7 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, odp_pktio_t input = pktio_entry->s.handle; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; /* Allocate maximum sized packets */ max_len = pkt_dpdk->data_room; @@ -640,23 +641,30 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, pkt_hdr = packet_hdr(pkt); if (layer) { - uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; - packet_parse_reset(pkt_hdr, 1); if (_odp_dpdk_packet_parse_common(&pkt_hdr->p, data, pkt_len, pkt_len, mbuf, layer, supported_ptypes, pktin_cfg)) { - odp_packet_free(pkt_table[i]); + odp_packet_free(pkt); rte_pktmbuf_free(mbuf); continue; } if (pktio_cls_enabled(pktio_entry)) { + odp_pool_t new_pool; + if (_odp_cls_classify_packet(pktio_entry, (const uint8_t *)data, - &pool, pkt_hdr)) { - odp_packet_free(pkt_table[i]); + &new_pool, pkt_hdr)) { + odp_packet_free(pkt); + rte_pktmbuf_free(mbuf); + continue; + } + + if (odp_unlikely(_odp_pktio_packet_to_pool( + &pkt, &pkt_hdr, new_pool))) { + odp_packet_free(pkt); rte_pktmbuf_free(mbuf); continue; } @@ -881,17 +889,15 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, struct rte_mbuf *mbuf; void *data; int i, nb_pkts; - odp_pool_t pool; odp_pktin_config_opt_t pktin_cfg; odp_pktio_t input; - pkt_dpdk_t *pkt_dpdk; + pkt_dpdk_t *pkt_dpdk = pkt_priv(pktio_entry); const odp_proto_layer_t layer = pktio_entry->s.parse_layer; + const uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; prefetch_pkt(mbuf_table[0]); - pkt_dpdk = pkt_priv(pktio_entry); nb_pkts = 0; - pool = pkt_dpdk->pool; set_flow_hash = pkt_dpdk->opt.set_flow_hash; pktin_cfg = pktio_entry->s.config.pktin; input = pktio_entry->s.handle; @@ -900,6 +906,8 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, prefetch_pkt(mbuf_table[1]); for (i = 0; i < mbuf_num; i++) { + odp_packet_t pkt; + if (odp_likely((i + 2) < mbuf_num)) prefetch_pkt(mbuf_table[i + 2]); @@ -913,11 +921,14 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, data = rte_pktmbuf_mtod(mbuf, char *); pkt_len = rte_pktmbuf_pkt_len(mbuf); pkt_hdr = pkt_hdr_from_mbuf(mbuf); + pkt = packet_handle(pkt_hdr); packet_init(pkt_hdr, pkt_len); - if (layer) { - uint32_t supported_ptypes = pkt_dpdk->supported_ptypes; + /* Init buffer segments. Currently, only single segment packets + * are supported. */ + pkt_hdr->seg_data = data; + if (layer) { if (_odp_dpdk_packet_parse_common(&pkt_hdr->p, data, pkt_len, pkt_len, mbuf, layer, supported_ptypes, @@ -927,19 +938,23 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, } if (pktio_cls_enabled(pktio_entry)) { + odp_pool_t new_pool; + if (_odp_cls_classify_packet(pktio_entry, (const uint8_t *)data, - &pool, pkt_hdr)) { + &new_pool, pkt_hdr)) { + rte_pktmbuf_free(mbuf); + continue; + } + + if (odp_unlikely(_odp_pktio_packet_to_pool( + &pkt, &pkt_hdr, new_pool))) { rte_pktmbuf_free(mbuf); continue; } } } - /* Init buffer segments. Currently, only single segment packets - * are supported. */ - pkt_hdr->seg_data = data; - pkt_hdr->input = input; if (set_flow_hash && (mbuf->ol_flags & RTE_MBUF_F_RX_RSS_HASH)) @@ -947,7 +962,7 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, packet_set_ts(pkt_hdr, ts); - pkt_table[nb_pkts++] = packet_handle(pkt_hdr); + pkt_table[nb_pkts++] = pkt; } return nb_pkts; diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index c702f9ded..68e1431ab 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -162,7 +162,6 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, int num_rx = 0; int packets = 0, errors = 0; uint32_t octets = 0; - const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; const odp_proto_layer_t layer = pktio_entry->s.parse_layer; const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; @@ -191,7 +190,6 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, uint8_t buf[PARSE_BYTES]; int ret; uint32_t seg_len = odp_packet_seg_len(pkt); - uint64_t l4_part_sum = 0; /* Make sure there is enough data for the packet * parser in the case of a segmented packet. */ @@ -205,9 +203,8 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, } packet_parse_reset(pkt_hdr, 1); - ret = _odp_packet_parse_common(&pkt_hdr->p, pkt_addr, pkt_len, - seg_len, layer, chksums, - &l4_part_sum, opt); + ret = _odp_packet_parse_common(pkt_hdr, pkt_addr, pkt_len, + seg_len, layer, opt); if (ret) errors++; @@ -217,7 +214,6 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, } if (pktio_cls_enabled(pktio_entry)) { - odp_packet_t new_pkt; odp_pool_t new_pool; ret = _odp_cls_classify_packet(pktio_entry, pkt_addr, @@ -227,23 +223,15 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, continue; } - if (new_pool != odp_packet_pool(pkt)) { - new_pkt = odp_packet_copy(pkt, new_pool); - + if (odp_unlikely(_odp_pktio_packet_to_pool( + &pkt, &pkt_hdr, new_pool))) { odp_packet_free(pkt); - - if (new_pkt == ODP_PACKET_INVALID) { - pktio_entry->s.stats.in_discards++; - continue; - } - - pkt = new_pkt; - pkt_hdr = packet_hdr(new_pkt); + odp_atomic_inc_u64( + &pktio_entry->s.stats_extra + .in_discards); + continue; } } - - if (layer >= ODP_PROTO_LAYER_L4) - _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); } packet_set_ts(pkt_hdr, ts); diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c index 342f38431..c843d5367 100644 --- a/platform/linux-generic/pktio/netmap.c +++ b/platform/linux-generic/pktio/netmap.c @@ -828,7 +828,6 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, uint32_t max_len; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; int num_rx = 0; - const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; const odp_proto_layer_t layer = pktio_entry->s.parse_layer; const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; @@ -842,7 +841,6 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, netmap_slot_t slot; uint16_t len; const uint8_t *buf; - uint64_t l4_part_sum = 0; slot = slot_tbl[i]; len = slot.len; @@ -850,39 +848,50 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, odp_prefetch(slot.buf); - pkt = pkt_tbl[num_rx]; + pkt = pkt_tbl[i]; pkt_hdr = packet_hdr(pkt); + pull_tail(pkt_hdr, max_len - len); + if (frame_offset) + pull_head(pkt_hdr, frame_offset); + + if (odp_packet_copy_from_mem(pkt, 0, len, slot.buf) != 0) { + odp_packet_free(pkt); + continue; + } + if (layer) { - if (_odp_packet_parse_common(&pkt_hdr->p, buf, len, len, - layer, chksums, &l4_part_sum, opt) < 0) + if (_odp_packet_parse_common(pkt_hdr, buf, len, len, + layer, opt) < 0) { + odp_packet_free(pkt); continue; + } if (pktio_cls_enabled(pktio_entry)) { - if (_odp_cls_classify_packet(pktio_entry, buf, &pool, - pkt_hdr)) + odp_pool_t new_pool; + + if (_odp_cls_classify_packet(pktio_entry, buf, &new_pool, + pkt_hdr)) { + odp_packet_free(pkt); continue; + } + + if (odp_unlikely(_odp_pktio_packet_to_pool( + &pkt, &pkt_hdr, new_pool))) { + odp_packet_free(pkt); + odp_atomic_inc_u64( + &pktio_entry->s.stats_extra + .in_discards); + continue; + } } } - pull_tail(pkt_hdr, max_len - len); - if (frame_offset) - pull_head(pkt_hdr, frame_offset); - - if (odp_packet_copy_from_mem(pkt, 0, len, slot.buf) != 0) - break; - pkt_hdr->input = pktio_entry->s.handle; - - if (layer >= ODP_PROTO_LAYER_L4) - _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); - packet_set_ts(pkt_hdr, ts); - num_rx++; - } - if (num_rx < num) - odp_packet_free_multi(&pkt_tbl[num_rx], num - num_rx); + pkt_tbl[num_rx++] = pkt; + } return num_rx; } diff --git a/platform/linux-generic/pktio/pcap.c b/platform/linux-generic/pktio/pcap.c index af94ffa72..2de70dd1f 100644 --- a/platform/linux-generic/pktio/pcap.c +++ b/platform/linux-generic/pktio/pcap.c @@ -241,7 +241,6 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, { int i; struct pcap_pkthdr *hdr; - odp_pool_t new_pool; const u_char *data; odp_packet_t pkt; odp_packet_hdr_t *pkt_hdr; @@ -252,7 +251,6 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, int packets = 0, errors = 0; uint32_t octets = 0; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; - const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; const odp_proto_layer_t layer = pktio_entry->s.parse_layer; const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; @@ -297,11 +295,8 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, } if (layer) { - uint64_t l4_part_sum = 0; - - ret = _odp_packet_parse_common(&pkt_hdr->p, data, pkt_len, - pkt_len, layer, chksums, - &l4_part_sum, opt); + ret = _odp_packet_parse_common(pkt_hdr, data, pkt_len, + pkt_len, layer, opt); if (ret) errors++; @@ -311,7 +306,7 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, } if (pktio_cls_enabled(pktio_entry)) { - odp_packet_t new_pkt; + odp_pool_t new_pool; ret = _odp_cls_classify_packet(pktio_entry, data, &new_pool, pkt_hdr); @@ -319,23 +314,16 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_packet_free(pkt); continue; } - if (new_pool != pcap->pool) { - new_pkt = odp_packet_copy(pkt, new_pool); + if (odp_unlikely(_odp_pktio_packet_to_pool( + &pkt, &pkt_hdr, new_pool))) { odp_packet_free(pkt); - - if (odp_unlikely(new_pkt == ODP_PACKET_INVALID)) { - pktio_entry->s.stats.in_discards++; - continue; - } - - pkt = new_pkt; - pkt_hdr = packet_hdr(new_pkt); + odp_atomic_inc_u64( + &pktio_entry->s.stats_extra + .in_discards); + continue; } } - - if (layer >= ODP_PROTO_LAYER_L4) - _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); } packet_set_ts(pkt_hdr, ts); diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c index 0d756c4e1..3ec458c0b 100644 --- a/platform/linux-generic/pktio/socket.c +++ b/platform/linux-generic/pktio/socket.c @@ -235,7 +235,6 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, int i; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; uint32_t alloc_len = pkt_sock->mtu + frame_offset; - const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; const odp_proto_layer_t layer = pktio_entry->s.parse_layer; const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; @@ -266,7 +265,6 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); uint16_t pkt_len = msgvec[i].msg_len; int ret; - uint64_t l4_part_sum = 0; if (odp_unlikely(msgvec[i].msg_hdr.msg_flags & MSG_TRUNC)) { odp_packet_free(pkt); @@ -294,19 +292,29 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, base = buf; } - if (_odp_packet_parse_common(&pkt_hdr->p, base, pkt_len, - seg_len, layer, chksums, - &l4_part_sum, opt) < 0) { + if (_odp_packet_parse_common(pkt_hdr, base, pkt_len, + seg_len, layer, opt) < 0) { odp_packet_free(pkt); continue; } if (pktio_cls_enabled(pktio_entry)) { - if (_odp_cls_classify_packet(pktio_entry, base, &pool, + odp_pool_t new_pool; + + if (_odp_cls_classify_packet(pktio_entry, base, &new_pool, pkt_hdr)) { odp_packet_free(pkt); continue; } + + if (odp_unlikely(_odp_pktio_packet_to_pool( + &pkt, &pkt_hdr, new_pool))) { + odp_packet_free(pkt); + odp_atomic_inc_u64( + &pktio_entry->s.stats_extra + .in_discards); + continue; + } } } @@ -318,10 +326,6 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, } pkt_hdr->input = pktio_entry->s.handle; - - if (layer >= ODP_PROTO_LAYER_L4) - _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); - packet_set_ts(pkt_hdr, ts); pkt_table[nb_rx++] = pkt; diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index 4845b5dab..d509b75f6 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -152,7 +152,6 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, odp_pool_t pool = pkt_sock->pool; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; uint16_t vlan_len = 0; - const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; const odp_proto_layer_t layer = pktio_entry->s.parse_layer; const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; @@ -168,7 +167,6 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, odp_packet_t pkt; odp_packet_hdr_t *hdr; int ret; - uint64_t l4_part_sum = 0; tp_hdr = (void *)next_ptr; @@ -218,27 +216,6 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, hdr = packet_hdr(pkt); - if (layer) { - if (_odp_packet_parse_common(&hdr->p, pkt_buf, pkt_len, - pkt_len, layer, chksums, - &l4_part_sum, opt) < 0) { - odp_packet_free(pkt); - tp_hdr->tp_status = TP_STATUS_KERNEL; - frame_num = next_frame_num; - continue; - } - - if (pktio_cls_enabled(pktio_entry)) { - if (_odp_cls_classify_packet(pktio_entry, pkt_buf, - &pool, hdr)) { - odp_packet_free(pkt); - tp_hdr->tp_status = TP_STATUS_KERNEL; - frame_num = next_frame_num; - continue; - } - } - } - if (frame_offset) pull_head(hdr, frame_offset); @@ -289,11 +266,40 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, *tci = odp_cpu_to_be_16(tp_hdr->tp_vlan_tci); } - hdr->input = pktio_entry->s.handle; + if (layer) { + if (_odp_packet_parse_common(hdr, pkt_buf, pkt_len, + pkt_len, layer, opt) < 0) { + odp_packet_free(pkt); + tp_hdr->tp_status = TP_STATUS_KERNEL; + frame_num = next_frame_num; + continue; + } + + if (pktio_cls_enabled(pktio_entry)) { + odp_pool_t new_pool; + + if (_odp_cls_classify_packet(pktio_entry, pkt_buf, + &new_pool, hdr)) { + odp_packet_free(pkt); + tp_hdr->tp_status = TP_STATUS_KERNEL; + frame_num = next_frame_num; + continue; + } - if (layer >= ODP_PROTO_LAYER_L4) - _odp_packet_l4_chksum(hdr, chksums, l4_part_sum); + if (odp_unlikely(_odp_pktio_packet_to_pool( + &pkt, &hdr, new_pool))) { + odp_packet_free(pkt); + tp_hdr->tp_status = TP_STATUS_KERNEL; + frame_num = next_frame_num; + odp_atomic_inc_u64( + &pktio_entry->s.stats_extra + .in_discards); + continue; + } + } + } + hdr->input = pktio_entry->s.handle; packet_set_ts(hdr, ts); tp_hdr->tp_status = TP_STATUS_KERNEL; diff --git a/platform/linux-generic/pktio/socket_xdp.c b/platform/linux-generic/pktio/socket_xdp.c index e43e4bf89..d00cbb236 100644 --- a/platform/linux-generic/pktio/socket_xdp.c +++ b/platform/linux-generic/pktio/socket_xdp.c @@ -13,6 +13,7 @@ #include <odp/api/hints.h> #include <odp/api/system_info.h> #include <odp/api/ticketlock.h> +#include <odp/api/packet_io_stats.h> #include <odp_debug_internal.h> #include <odp_macros_internal.h> @@ -21,54 +22,97 @@ #include <odp_parse_internal.h> #include <odp_classification_internal.h> #include <odp_socket_common.h> +#include <odp_libconfig_internal.h> #include <string.h> #include <errno.h> #include <sys/socket.h> #include <unistd.h> #include <poll.h> +#include <sys/ioctl.h> +#include <linux/ethtool.h> +#include <linux/sockios.h> +#include <net/if.h> +#include <linux/if_xdp.h> #include <xdp/xsk.h> -#define NUM_XDP_DESCS 1024U +#define NUM_DESCS_DEFAULT 1024U #define MIN_FRAME_SIZE 2048U + #define IF_DELIM " " #define Q_DELIM ':' +#define CONF_BASE_STR "pktio_xdp" +#define RX_DESCS_STR "num_rx_desc" +#define TX_DESCS_STR "num_tx_desc" + +enum { + RX_PKT_ALLOC_ERR, + RX_DESC_RSV_ERR, + TX_PKT_ALLOC_ERR, + TX_DESC_RSV_ERR +}; + +static const char * const internal_stats_strs[] = { + "rx_packet_allocation_errors", + "rx_umem_descriptor_reservation_errors", + "tx_packet_allocation_errors", + "tx_umem_descriptor_reservation_errors" +}; + +#define MAX_INTERNAL_STATS _ODP_ARRAY_SIZE(internal_stats_strs) + +typedef struct { + uint64_t rx_dropped; + uint64_t rx_inv_descs; + uint64_t tx_inv_descs; +} xdp_sock_stats_t; + +typedef struct { + odp_ticketlock_t rx_lock ODP_ALIGNED_CACHE; + odp_ticketlock_t tx_lock ODP_ALIGNED_CACHE; + struct xsk_ring_cons rx; + struct xsk_ring_cons compl_q; + struct xsk_ring_prod tx; + struct xsk_ring_prod fill_q; + odp_pktin_queue_stats_t qi_stats; + odp_pktout_queue_stats_t qo_stats; + xdp_sock_stats_t xdp_stats; + struct xsk_socket *xsk; + uint64_t i_stats[MAX_INTERNAL_STATS]; +} xdp_sock_t; typedef struct { struct xsk_ring_prod fill_q; struct xsk_ring_cons compl_q; struct xsk_umem *umem; pool_t *pool; + int num_rx_desc; + int num_tx_desc; + uint32_t ref_cnt; } xdp_umem_info_t; typedef struct { - struct xsk_ring_cons rx; - struct xsk_ring_cons compl_q; - struct xsk_ring_prod tx; - struct xsk_ring_prod fill_q; + xdp_sock_t qs[PKTIO_MAX_QUEUES]; xdp_umem_info_t *umem_info; - struct xsk_socket *xsk; + uint32_t num_q; int pktio_idx; int helper_sock; uint32_t mtu; uint32_t max_mtu; + uint32_t bind_q; + odp_bool_t lockless_rx; + odp_bool_t lockless_tx; } xdp_sock_info_t; typedef struct { - odp_ticketlock_t rx_lock ODP_ALIGNED_CACHE; - odp_ticketlock_t tx_lock ODP_ALIGNED_CACHE; - xdp_sock_info_t sock_info; -} pkt_xdp_t; - -typedef struct { odp_packet_hdr_t *pkt_hdr; odp_packet_t pkt; uint8_t *data; uint32_t len; } pkt_data_t; -ODP_STATIC_ASSERT(PKTIO_PRIVATE_SIZE >= sizeof(pkt_xdp_t), +ODP_STATIC_ASSERT(PKTIO_PRIVATE_SIZE >= sizeof(xdp_sock_info_t), "PKTIO_PRIVATE_SIZE too small"); static odp_bool_t disable_pktio; @@ -87,18 +131,65 @@ static int sock_xdp_init_global(void) return 0; } -static inline pkt_xdp_t *pkt_priv(pktio_entry_t *pktio_entry) +static inline xdp_sock_info_t *pkt_priv(pktio_entry_t *pktio_entry) { - return (pkt_xdp_t *)(uintptr_t)(pktio_entry->s.pkt_priv); + return (xdp_sock_info_t *)(uintptr_t)(pktio_entry->s.pkt_priv); } -static void fill_socket_config(struct xsk_socket_config *config) +static void parse_options(xdp_umem_info_t *umem_info) { - config->rx_size = NUM_XDP_DESCS; - config->tx_size = NUM_XDP_DESCS; - config->libxdp_flags = 0U; - config->xdp_flags = 0U; - config->bind_flags = XDP_ZEROCOPY; /* TODO: XDP_COPY */ + if (!_odp_libconfig_lookup_ext_int(CONF_BASE_STR, NULL, RX_DESCS_STR, + &umem_info->num_rx_desc) || + !_odp_libconfig_lookup_ext_int(CONF_BASE_STR, NULL, TX_DESCS_STR, + &umem_info->num_tx_desc)) { + ODP_ERR("Unable to parse xdp descriptor configuration, using defaults (%d).\n", + NUM_DESCS_DEFAULT); + goto defaults; + } + + if (umem_info->num_rx_desc <= 0 || umem_info->num_tx_desc <= 0 || + !_ODP_CHECK_IS_POWER2(umem_info->num_rx_desc) || + !_ODP_CHECK_IS_POWER2(umem_info->num_tx_desc)) { + ODP_ERR("Invalid xdp descriptor configuration, using defaults (%d).\n", + NUM_DESCS_DEFAULT); + goto defaults; + } + + return; + +defaults: + umem_info->num_rx_desc = NUM_DESCS_DEFAULT; + umem_info->num_tx_desc = NUM_DESCS_DEFAULT; +} + +static int umem_create(xdp_umem_info_t *umem_info, pool_t *pool) +{ + struct xsk_umem_config cfg; + + if (umem_info->ref_cnt++ > 0U) + return 0; + + parse_options(umem_info); + umem_info->pool = pool; + /* Fill queue size is recommended to be >= HW RX ring size + AF_XDP RX + * ring size, so use size twice the size of AF_XDP RX ring. */ + cfg.fill_size = umem_info->num_rx_desc * 2U; + cfg.comp_size = umem_info->num_tx_desc; + cfg.frame_size = pool->block_size; + cfg.frame_headroom = sizeof(odp_packet_hdr_t) + pool->headroom; + cfg.flags = XDP_UMEM_UNALIGNED_CHUNK_FLAG; + + return xsk_umem__create(&umem_info->umem, pool->base_addr, pool->shm_size, + &umem_info->fill_q, &umem_info->compl_q, &cfg); +} + +static void umem_delete(xdp_umem_info_t *umem_info) +{ + if (umem_info->ref_cnt-- != 1U) + return; + + while (xsk_umem__delete(umem_info->umem) == -EBUSY) + continue; } static uint32_t get_bind_queue_index(const char *devname) @@ -142,89 +233,32 @@ out: return idx; } -static odp_bool_t reserve_fill_queue_elements(xdp_sock_info_t *sock_info, int num) -{ - pool_t *pool; - odp_packet_t packets[num]; - int count; - struct xsk_ring_prod *fill_q; - uint32_t start_idx; - int pktio_idx; - uint32_t block_size; - odp_packet_hdr_t *pkt_hdr; - - pool = sock_info->umem_info->pool; - count = odp_packet_alloc_multi(pool->pool_hdl, sock_info->mtu, packets, num); - - if (count <= 0) - return false; - - fill_q = &sock_info->fill_q; - - if (xsk_ring_prod__reserve(fill_q, count, &start_idx) == 0U) { - odp_packet_free_multi(packets, count); - return false; - } - - pktio_idx = sock_info->pktio_idx; - block_size = pool->block_size; - - for (int i = 0; i < count; ++i) { - pkt_hdr = packet_hdr(packets[i]); - pkt_hdr->ms_pktio_idx = pktio_idx; - *xsk_ring_prod__fill_addr(fill_q, start_idx++) = - pkt_hdr->event_hdr.index.event * block_size; - } - - xsk_ring_prod__submit(&sock_info->fill_q, count); - - return true; -} - static int sock_xdp_open(odp_pktio_t pktio, pktio_entry_t *pktio_entry, const char *devname, odp_pool_t pool_hdl) { - pkt_xdp_t *priv; + xdp_sock_info_t *priv; pool_t *pool; - struct xsk_socket_config config; - uint32_t bind_q; int ret; if (disable_pktio) return -1; priv = pkt_priv(pktio_entry); - memset(priv, 0, sizeof(pkt_xdp_t)); + memset(priv, 0, sizeof(xdp_sock_info_t)); pool = pool_entry_from_hdl(pool_hdl); - priv->sock_info.umem_info = (xdp_umem_info_t *)pool->mem_src_data; - priv->sock_info.xsk = NULL; - /* Mark transitory kernel-owned packets with the pktio index, so that they can be freed on - * close. */ - priv->sock_info.pktio_idx = 1 + odp_pktio_index(pktio); - fill_socket_config(&config); - bind_q = get_bind_queue_index(devname); - /* With xsk_socket__create_shared(), as only one bind queue index can - * be passed, NIC in use needs to be configured accordingly to have - * only a single combined TX-RX queue, otherwise traffic may not end up - * on the socket. For now, always bind to the first queue (overridable - * with environment variable). */ - ret = xsk_socket__create_shared(&priv->sock_info.xsk, devname, bind_q, - priv->sock_info.umem_info->umem, &priv->sock_info.rx, - &priv->sock_info.tx, &priv->sock_info.fill_q, - &priv->sock_info.compl_q, &config); + priv->umem_info = (xdp_umem_info_t *)pool->mem_src_data; + ret = umem_create(priv->umem_info, pool); if (ret) { - ODP_ERR("Error creating xdp socket for bind queue %u: %d\n", bind_q, ret); - goto xsk_err; + ODP_ERR("Error creating UMEM pool for xdp: %d\n", ret); + return -1; } - /* Ring setup/clean up routines seem to be asynchronous with some drivers and might not be - * ready yet after xsk_socket__create_shared(). */ - sleep(1U); - + /* Mark transitory kernel-owned packets with the pktio index, so that they can be freed on + * close. */ + priv->pktio_idx = 1 + odp_pktio_index(pktio); /* Querying with ioctl() via AF_XDP socket doesn't seem to work, so * create a helper socket for this. */ - priv->sock_info.helper_sock = -1; ret = socket(AF_INET, SOCK_DGRAM, 0); if (ret == -1) { @@ -232,48 +266,53 @@ static int sock_xdp_open(odp_pktio_t pktio, pktio_entry_t *pktio_entry, const ch goto sock_err; } - priv->sock_info.helper_sock = ret; - priv->sock_info.mtu = _odp_mtu_get_fd(priv->sock_info.helper_sock, devname); + priv->helper_sock = ret; + priv->mtu = _odp_mtu_get_fd(priv->helper_sock, devname); - if (priv->sock_info.mtu == 0U) - goto res_err; + if (priv->mtu == 0U) + goto mtu_err; - priv->sock_info.max_mtu = pool->seg_len; + priv->max_mtu = pool->seg_len; - if (!reserve_fill_queue_elements(&priv->sock_info, config.rx_size)) { - ODP_ERR("Unable to reserve fill queue descriptors.\n"); - goto res_err; + for (int i = 0; i < PKTIO_MAX_QUEUES; ++i) { + odp_ticketlock_init(&priv->qs[i].rx_lock); + odp_ticketlock_init(&priv->qs[i].tx_lock); } - odp_ticketlock_init(&priv->rx_lock); - odp_ticketlock_init(&priv->tx_lock); + priv->bind_q = get_bind_queue_index(pktio_entry->s.name); + + ODP_DBG("Socket xdp interface (%s):\n", pktio_entry->s.name); + ODP_DBG(" num_rx_desc: %d\n", priv->umem_info->num_rx_desc); + ODP_DBG(" num_tx_desc: %d\n", priv->umem_info->num_tx_desc); + ODP_DBG(" starting bind queue: %u\n", priv->bind_q); return 0; -res_err: - close(priv->sock_info.helper_sock); - priv->sock_info.helper_sock = -1; +mtu_err: + close(priv->helper_sock); sock_err: - xsk_socket__delete(priv->sock_info.xsk); - priv->sock_info.xsk = NULL; + umem_delete(priv->umem_info); -xsk_err: return -1; } static int sock_xdp_close(pktio_entry_t *pktio_entry) { - pkt_xdp_t *priv = pkt_priv(pktio_entry); - pool_t *pool = priv->sock_info.umem_info->pool; + xdp_sock_info_t *priv = pkt_priv(pktio_entry); + pool_t *pool = priv->umem_info->pool; odp_packet_hdr_t *pkt_hdr; - if (priv->sock_info.helper_sock != -1) - close(priv->sock_info.helper_sock); + close(priv->helper_sock); - if (priv->sock_info.xsk != NULL) - xsk_socket__delete(priv->sock_info.xsk); + for (uint32_t i = 0U; i < priv->num_q; ++i) { + if (priv->qs[i].xsk != NULL) { + xsk_socket__delete(priv->qs[i].xsk); + priv->qs[i].xsk = NULL; + } + } + umem_delete(priv->umem_info); /* Ring setup/clean up routines seem to be asynchronous with some drivers and might not be * ready yet after xsk_socket__delete(). */ sleep(1U); @@ -282,7 +321,7 @@ static int sock_xdp_close(pktio_entry_t *pktio_entry) for (uint32_t i = 0U; i < pool->num + pool->skipped_blocks; ++i) { pkt_hdr = packet_hdr(packet_from_event_hdr(event_hdr_from_index(pool, i))); - if (pkt_hdr->ms_pktio_idx == priv->sock_info.pktio_idx) { + if (pkt_hdr->ms_pktio_idx == priv->pktio_idx) { pkt_hdr->ms_pktio_idx = 0U; odp_packet_free(packet_handle(pkt_hdr)); } @@ -291,6 +330,149 @@ static int sock_xdp_close(pktio_entry_t *pktio_entry) return 0; } +static int sock_xdp_stats(pktio_entry_t *pktio_entry, odp_pktio_stats_t *stats) +{ + xdp_sock_info_t *priv = pkt_priv(pktio_entry); + xdp_sock_t *sock; + odp_pktin_queue_stats_t qi_stats; + odp_pktout_queue_stats_t qo_stats; + struct xdp_statistics xdp_stats; + socklen_t optlen = sizeof(struct xdp_statistics); + + memset(stats, 0, sizeof(odp_pktio_stats_t)); + + for (uint32_t i = 0U; i < priv->num_q; ++i) { + sock = &priv->qs[i]; + qi_stats = sock->qi_stats; + qo_stats = sock->qo_stats; + stats->in_octets += qi_stats.octets; + stats->in_packets += qi_stats.packets; + stats->in_errors += qi_stats.errors; + stats->out_octets += qo_stats.octets; + stats->out_packets += qo_stats.packets; + + if (!getsockopt(xsk_socket__fd(sock->xsk), SOL_XDP, XDP_STATISTICS, &xdp_stats, + &optlen)) { + stats->in_errors += (xdp_stats.rx_dropped - sock->xdp_stats.rx_dropped); + stats->in_discards += + (xdp_stats.rx_invalid_descs - sock->xdp_stats.rx_inv_descs); + stats->out_discards += + (xdp_stats.tx_invalid_descs - sock->xdp_stats.tx_inv_descs); + } + } + + return 0; +} + +static int sock_xdp_stats_reset(pktio_entry_t *pktio_entry) +{ + xdp_sock_info_t *priv = pkt_priv(pktio_entry); + xdp_sock_t *sock; + struct xdp_statistics xdp_stats; + socklen_t optlen = sizeof(struct xdp_statistics); + + for (uint32_t i = 0U; i < priv->num_q; ++i) { + sock = &priv->qs[i]; + memset(&sock->qi_stats, 0, sizeof(odp_pktin_queue_stats_t)); + memset(&sock->qo_stats, 0, sizeof(odp_pktout_queue_stats_t)); + memset(sock->i_stats, 0, sizeof(sock->i_stats)); + + if (!getsockopt(xsk_socket__fd(sock->xsk), SOL_XDP, XDP_STATISTICS, &xdp_stats, + &optlen)) { + sock->xdp_stats.rx_dropped = xdp_stats.rx_dropped; + sock->xdp_stats.rx_inv_descs = xdp_stats.rx_invalid_descs; + sock->xdp_stats.tx_inv_descs = xdp_stats.tx_invalid_descs; + } + } + + return 0; +} + +static int sock_xdp_pktin_queue_stats(pktio_entry_t *pktio_entry, uint32_t index, + odp_pktin_queue_stats_t *pktin_stats) +{ + xdp_sock_info_t *priv = pkt_priv(pktio_entry); + xdp_sock_t *sock; + struct xdp_statistics xdp_stats; + socklen_t optlen = sizeof(struct xdp_statistics); + + sock = &priv->qs[index]; + *pktin_stats = sock->qi_stats; + + if (!getsockopt(xsk_socket__fd(sock->xsk), SOL_XDP, XDP_STATISTICS, &xdp_stats, &optlen)) { + pktin_stats->errors += (xdp_stats.rx_dropped - sock->xdp_stats.rx_dropped); + pktin_stats->discards += + (xdp_stats.rx_invalid_descs - sock->xdp_stats.rx_inv_descs); + } + + return 0; +} + +static int sock_xdp_pktout_queue_stats(pktio_entry_t *pktio_entry, uint32_t index, + odp_pktout_queue_stats_t *pktout_stats) +{ + xdp_sock_info_t *priv = pkt_priv(pktio_entry); + xdp_sock_t *sock; + struct xdp_statistics xdp_stats; + socklen_t optlen = sizeof(struct xdp_statistics); + + sock = &priv->qs[index]; + *pktout_stats = sock->qo_stats; + + if (!getsockopt(xsk_socket__fd(sock->xsk), SOL_XDP, XDP_STATISTICS, &xdp_stats, &optlen)) + pktout_stats->discards += + (xdp_stats.tx_invalid_descs - sock->xdp_stats.tx_inv_descs); + + return 0; +} + +static int sock_xdp_extra_stat_info(pktio_entry_t *pktio_entry, odp_pktio_extra_stat_info_t info[], + int num) +{ + xdp_sock_info_t *priv = pkt_priv(pktio_entry); + const int total_stats = MAX_INTERNAL_STATS * priv->num_q; + + if (info != NULL && num > 0) { + for (int i = 0; i < _ODP_MIN(num, total_stats); ++i) + snprintf(info[i].name, ODP_PKTIO_STATS_EXTRA_NAME_LEN - 1, + "q%" PRIu64 "_%s", i / MAX_INTERNAL_STATS, + internal_stats_strs[i % MAX_INTERNAL_STATS]); + } + + return total_stats; +} + +static int sock_xdp_extra_stats(pktio_entry_t *pktio_entry, uint64_t stats[], int num) +{ + xdp_sock_info_t *priv = pkt_priv(pktio_entry); + const int total_stats = MAX_INTERNAL_STATS * priv->num_q; + uint64_t *i_stats; + + if (stats != NULL && num > 0) { + for (int i = 0; i < _ODP_MIN(num, total_stats); ++i) { + i_stats = priv->qs[i / MAX_INTERNAL_STATS].i_stats; + stats[i] = i_stats[i % MAX_INTERNAL_STATS]; + } + } + + return total_stats; +} + +static int sock_xdp_extra_stat_counter(pktio_entry_t *pktio_entry, uint32_t id, uint64_t *stat) +{ + xdp_sock_info_t *priv = pkt_priv(pktio_entry); + const uint32_t total_stats = MAX_INTERNAL_STATS * priv->num_q; + + if (id >= total_stats) { + ODP_ERR("Invalid counter id: %u (allowed range: 0-%u)\n", id, total_stats - 1U); + return -1; + } + + *stat = priv->qs[id / MAX_INTERNAL_STATS].i_stats[id % MAX_INTERNAL_STATS]; + + return 0; +} + static inline void extract_data(const struct xdp_desc *rx_desc, uint8_t *pool_base_addr, pkt_data_t *pkt_data) { @@ -311,17 +493,16 @@ static inline void extract_data(const struct xdp_desc *rx_desc, uint8_t *pool_ba pkt_data->len = rx_desc->len; } -static uint32_t process_received(pktio_entry_t *pktio_entry, xdp_sock_info_t *sock_info, +static uint32_t process_received(pktio_entry_t *pktio_entry, xdp_sock_t *sock, pool_t *pool, uint32_t start_idx, odp_packet_t packets[], int num) { + struct xsk_ring_cons *rx = &sock->rx; + uint8_t *base_addr = pool->base_addr; pkt_data_t pkt_data; - struct xsk_ring_cons *rx = &sock_info->rx; - uint8_t *base_addr = sock_info->umem_info->pool->base_addr; const odp_proto_layer_t layer = pktio_entry->s.parse_layer; - const odp_proto_chksums_t in_chksums = pktio_entry->s.in_chksums; + int ret; const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; - uint64_t l4_part_sum = 0U; - odp_pool_t *pool_hdl = &sock_info->umem_info->pool->pool_hdl; + uint64_t errors = 0U, octets = 0U; odp_pktio_t pktio_hdl = pktio_entry->s.handle; uint32_t num_rx = 0U; @@ -329,92 +510,146 @@ static uint32_t process_received(pktio_entry_t *pktio_entry, xdp_sock_info_t *so extract_data(xsk_ring_cons__rx_desc(rx, start_idx++), base_addr, &pkt_data); pkt_data.pkt_hdr->ms_pktio_idx = 0U; packet_init(pkt_data.pkt_hdr, pkt_data.len); + pkt_data.pkt_hdr->seg_data = pkt_data.data; + pkt_data.pkt_hdr->event_hdr.base_data = pkt_data.data; if (layer) { - if (_odp_packet_parse_common(&pkt_data.pkt_hdr->p, pkt_data.data, - pkt_data.len, pkt_data.len, - layer, in_chksums, &l4_part_sum, opt) < 0) { + ret = _odp_packet_parse_common(pkt_data.pkt_hdr, pkt_data.data, + pkt_data.len, pkt_data.len, + layer, opt); + + if (ret) + ++errors; + + if (ret < 0) { odp_packet_free(pkt_data.pkt); continue; } - if (pktio_cls_enabled(pktio_entry) && - _odp_cls_classify_packet(pktio_entry, pkt_data.data, pool_hdl, - pkt_data.pkt_hdr)) { - odp_packet_free(pkt_data.pkt); - continue; + if (pktio_cls_enabled(pktio_entry)) { + odp_pool_t new_pool; + + ret = _odp_cls_classify_packet(pktio_entry, pkt_data.data, + &new_pool, pkt_data.pkt_hdr); + if (ret) { + odp_packet_free(pkt_data.pkt); + continue; + } + + if (odp_unlikely(_odp_pktio_packet_to_pool( + &pkt_data.pkt, &pkt_data.pkt_hdr, new_pool))) { + odp_packet_free(pkt_data.pkt); + continue; + } } } - pkt_data.pkt_hdr->seg_data = pkt_data.data; - pkt_data.pkt_hdr->event_hdr.base_data = pkt_data.data; pkt_data.pkt_hdr->input = pktio_hdl; packets[num_rx++] = pkt_data.pkt; + octets += pkt_data.len; } + sock->qi_stats.octets += octets; + sock->qi_stats.packets += num_rx; + sock->qi_stats.errors += errors; + return num_rx; } -static int sock_xdp_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_packet_t packets[], - int num) +static odp_bool_t reserve_fill_queue_elements(xdp_sock_info_t *sock_info, xdp_sock_t *sock, + int num) { - pkt_xdp_t *priv; + pool_t *pool; + odp_packet_t packets[num]; + int count; + struct xsk_ring_prod *fill_q; + uint32_t start_idx; + int pktio_idx; + uint32_t block_size; + odp_packet_hdr_t *pkt_hdr; + + pool = sock_info->umem_info->pool; + count = odp_packet_alloc_multi(pool->pool_hdl, sock_info->mtu, packets, num); + + if (count <= 0) { + ++sock->i_stats[RX_PKT_ALLOC_ERR]; + return false; + } + + fill_q = &sock->fill_q; + + if (xsk_ring_prod__reserve(fill_q, count, &start_idx) == 0U) { + odp_packet_free_multi(packets, count); + ++sock->i_stats[RX_DESC_RSV_ERR]; + return false; + } + + pktio_idx = sock_info->pktio_idx; + block_size = pool->block_size; + + for (int i = 0; i < count; ++i) { + pkt_hdr = packet_hdr(packets[i]); + pkt_hdr->ms_pktio_idx = pktio_idx; + *xsk_ring_prod__fill_addr(fill_q, start_idx++) = + pkt_hdr->event_hdr.index.event * block_size; + } + + xsk_ring_prod__submit(&sock->fill_q, count); + + return true; +} + +static int sock_xdp_recv(pktio_entry_t *pktio_entry, int index, odp_packet_t packets[], int num) +{ + xdp_sock_info_t *priv; + xdp_sock_t *sock; struct pollfd fd; uint32_t start_idx = 0U, recvd, procd; priv = pkt_priv(pktio_entry); - odp_ticketlock_lock(&priv->rx_lock); + sock = &priv->qs[index]; + + if (!priv->lockless_rx) + odp_ticketlock_lock(&sock->rx_lock); - if (odp_unlikely(xsk_ring_prod__needs_wakeup(&priv->sock_info.fill_q))) { - fd.fd = xsk_socket__fd(priv->sock_info.xsk); + if (odp_unlikely(xsk_ring_prod__needs_wakeup(&sock->fill_q))) { + fd.fd = xsk_socket__fd(sock->xsk); fd.events = POLLIN; (void)poll(&fd, 1U, 0); } - recvd = xsk_ring_cons__peek(&priv->sock_info.rx, num, &start_idx); + recvd = xsk_ring_cons__peek(&sock->rx, num, &start_idx); if (recvd == 0U) { - odp_ticketlock_unlock(&priv->rx_lock); + if (!priv->lockless_rx) + odp_ticketlock_unlock(&sock->rx_lock); return 0; } - procd = process_received(pktio_entry, &priv->sock_info, start_idx, packets, recvd); - xsk_ring_cons__release(&priv->sock_info.rx, recvd); - (void)reserve_fill_queue_elements(&priv->sock_info, recvd); - odp_ticketlock_unlock(&priv->rx_lock); - - return procd; -} + procd = process_received(pktio_entry, sock, priv->umem_info->pool, start_idx, packets, + recvd); + xsk_ring_cons__release(&sock->rx, recvd); + (void)reserve_fill_queue_elements(priv, sock, recvd); -static inline void populate_tx_desc(pool_t *pool, odp_packet_hdr_t *pkt_hdr, - struct xdp_desc *tx_desc) -{ - uint64_t frame_off; - uint64_t pkt_off; + if (!priv->lockless_rx) + odp_ticketlock_unlock(&sock->rx_lock); - frame_off = pkt_hdr->event_hdr.index.event * pool->block_size; - pkt_off = (uint64_t)(uintptr_t)pkt_hdr->event_hdr.base_data - - (uint64_t)(uintptr_t)pool->base_addr - frame_off; - pkt_off <<= XSK_UNALIGNED_BUF_OFFSET_SHIFT; - tx_desc->addr = frame_off | pkt_off; - tx_desc->len = pkt_hdr->frame_len; + return procd; } -static void handle_pending_tx(xdp_sock_info_t *sock_info, int num) +static void handle_pending_tx(xdp_sock_t *sock, uint8_t *base_addr, int num) { struct xsk_ring_cons *compl_q; uint32_t sent; - uint8_t *base_addr; uint32_t start_idx; uint64_t frame_off; odp_packet_t pkt; - if (odp_unlikely(xsk_ring_prod__needs_wakeup(&sock_info->tx))) - (void)sendto(xsk_socket__fd(sock_info->xsk), NULL, 0U, MSG_DONTWAIT, NULL, 0U); + if (odp_unlikely(xsk_ring_prod__needs_wakeup(&sock->tx))) + (void)sendto(xsk_socket__fd(sock->xsk), NULL, 0U, MSG_DONTWAIT, NULL, 0U); - compl_q = &sock_info->compl_q; + compl_q = &sock->compl_q; sent = xsk_ring_cons__peek(compl_q, num, &start_idx); - base_addr = sock_info->umem_info->pool->base_addr; odp_packet_t packets[sent]; @@ -432,57 +667,95 @@ static void handle_pending_tx(xdp_sock_info_t *sock_info, int num) } } -static int sock_xdp_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, - const odp_packet_t packets[], int num) +static inline void populate_tx_desc(odp_packet_hdr_t *pkt_hdr, pool_t *pool, + struct xdp_desc *tx_desc, uint32_t len) { - pkt_xdp_t *priv; - xdp_sock_info_t *sock_info; + uint64_t frame_off; + uint64_t pkt_off; + + frame_off = pkt_hdr->event_hdr.index.event * pool->block_size; + pkt_off = (uint64_t)(uintptr_t)pkt_hdr->event_hdr.base_data + - (uint64_t)(uintptr_t)pool->base_addr - frame_off; + pkt_off <<= XSK_UNALIGNED_BUF_OFFSET_SHIFT; + tx_desc->addr = frame_off | pkt_off; + tx_desc->len = len; +} + +static inline void populate_tx_descs(odp_packet_hdr_t *pkt_hdr, pool_t *pool, + struct xsk_ring_prod *tx, int seg_cnt, uint32_t start_idx, + int pktio_idx) +{ + if (odp_likely(seg_cnt == 1)) { + populate_tx_desc(pkt_hdr, pool, xsk_ring_prod__tx_desc(tx, start_idx), + pkt_hdr->frame_len); + pkt_hdr->ms_pktio_idx = pktio_idx; + } else { + for (int i = 0; i < seg_cnt; ++i) { + populate_tx_desc(pkt_hdr, pool, xsk_ring_prod__tx_desc(tx, start_idx++), + pkt_hdr->seg_len); + pkt_hdr->ms_pktio_idx = pktio_idx; + pkt_hdr = pkt_hdr->seg_next; + } + } +} + +static int sock_xdp_send(pktio_entry_t *pktio_entry, int index, const odp_packet_t packets[], + int num) +{ + xdp_sock_info_t *priv; + xdp_sock_t *sock; pool_t *pool; odp_pool_t pool_hdl; - int pktio_idx, i; + int pktio_idx, i, seg_cnt; struct xsk_ring_prod *tx; + uint8_t *base_addr; odp_packet_t pkt; odp_packet_hdr_t *pkt_hdr; - uint32_t start_idx; + uint32_t tx_descs, start_idx, sent = 0U; + uint64_t octets = 0U; if (odp_unlikely(num == 0)) return 0; priv = pkt_priv(pktio_entry); - odp_ticketlock_lock(&priv->tx_lock); - sock_info = &priv->sock_info; - pool = sock_info->umem_info->pool; + sock = &priv->qs[index]; + + if (!priv->lockless_tx) + odp_ticketlock_lock(&sock->tx_lock); + + pool = priv->umem_info->pool; pool_hdl = pool->pool_hdl; - pktio_idx = sock_info->pktio_idx; - tx = &sock_info->tx; + pktio_idx = priv->pktio_idx; + tx = &sock->tx; + base_addr = priv->umem_info->pool->base_addr; + tx_descs = priv->umem_info->num_tx_desc; for (i = 0; i < num; ++i) { pkt = ODP_PACKET_INVALID; - - if (odp_unlikely(odp_packet_num_segs(packets[i])) > 1) { - /* TODO: handle segmented packets */ - ODP_ERR("Only single-segment packets supported\n"); - break; - } - pkt_hdr = packet_hdr(packets[i]); + seg_cnt = pkt_hdr->seg_count; if (pkt_hdr->event_hdr.pool_ptr != pool) { pkt = odp_packet_copy(packets[i], pool_hdl); - if (odp_unlikely(pkt == ODP_PACKET_INVALID)) + if (odp_unlikely(pkt == ODP_PACKET_INVALID)) { + ++sock->i_stats[TX_PKT_ALLOC_ERR]; break; + } pkt_hdr = packet_hdr(pkt); + seg_cnt = pkt_hdr->seg_count; } - if (xsk_ring_prod__reserve(tx, 1U, &start_idx) == 0U) { - handle_pending_tx(sock_info, NUM_XDP_DESCS); + if (xsk_ring_prod__reserve(tx, seg_cnt, &start_idx) == 0U) { + handle_pending_tx(sock, base_addr, tx_descs); - if (xsk_ring_prod__reserve(tx, 1U, &start_idx) == 0U) { + if (xsk_ring_prod__reserve(tx, seg_cnt, &start_idx) == 0U) { if (pkt != ODP_PACKET_INVALID) odp_packet_free(pkt); + ++sock->i_stats[TX_DESC_RSV_ERR]; + break; } } @@ -490,94 +763,212 @@ static int sock_xdp_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, if (pkt != ODP_PACKET_INVALID) odp_packet_free(packets[i]); - pkt_hdr->ms_pktio_idx = pktio_idx; - populate_tx_desc(pool, pkt_hdr, xsk_ring_prod__tx_desc(tx, start_idx)); + populate_tx_descs(pkt_hdr, pool, tx, seg_cnt, start_idx, pktio_idx); + sent += seg_cnt; + octets += pkt_hdr->frame_len; } - xsk_ring_prod__submit(tx, i); - handle_pending_tx(sock_info, NUM_XDP_DESCS); - odp_ticketlock_unlock(&priv->tx_lock); + xsk_ring_prod__submit(tx, sent); + handle_pending_tx(sock, base_addr, tx_descs); + sock->qo_stats.octets += octets; + sock->qo_stats.packets += i; + + if (!priv->lockless_tx) + odp_ticketlock_unlock(&sock->tx_lock); return i; } static uint32_t sock_xdp_mtu_get(pktio_entry_t *pktio_entry) { - return pkt_priv(pktio_entry)->sock_info.mtu; + return pkt_priv(pktio_entry)->mtu; } static int sock_xdp_mtu_set(pktio_entry_t *pktio_entry, uint32_t maxlen_input, uint32_t maxlen_output ODP_UNUSED) { - pkt_xdp_t *priv = pkt_priv(pktio_entry); + xdp_sock_info_t *priv = pkt_priv(pktio_entry); int ret; - ret = _odp_mtu_set_fd(priv->sock_info.helper_sock, pktio_entry->s.name, maxlen_input); + ret = _odp_mtu_set_fd(priv->helper_sock, pktio_entry->s.name, maxlen_input); if (ret) return ret; - priv->sock_info.mtu = maxlen_input; + priv->mtu = maxlen_input; return 0; } static int sock_xdp_promisc_mode_set(pktio_entry_t *pktio_entry, int enable) { - return _odp_promisc_mode_set_fd(pkt_priv(pktio_entry)->sock_info.helper_sock, + return _odp_promisc_mode_set_fd(pkt_priv(pktio_entry)->helper_sock, pktio_entry->s.name, enable); } static int sock_xdp_promisc_mode_get(pktio_entry_t *pktio_entry) { - return _odp_promisc_mode_get_fd(pkt_priv(pktio_entry)->sock_info.helper_sock, + return _odp_promisc_mode_get_fd(pkt_priv(pktio_entry)->helper_sock, pktio_entry->s.name); } static int sock_xdp_mac_addr_get(pktio_entry_t *pktio_entry ODP_UNUSED, void *mac_addr) { - return _odp_mac_addr_get_fd(pkt_priv(pktio_entry)->sock_info.helper_sock, + return _odp_mac_addr_get_fd(pkt_priv(pktio_entry)->helper_sock, pktio_entry->s.name, mac_addr) ? -1 : ETH_ALEN; } static int sock_xdp_link_status(pktio_entry_t *pktio_entry) { - return _odp_link_status_fd(pkt_priv(pktio_entry)->sock_info.helper_sock, + return _odp_link_status_fd(pkt_priv(pktio_entry)->helper_sock, pktio_entry->s.name); } static int sock_xdp_link_info(pktio_entry_t *pktio_entry, odp_pktio_link_info_t *info) { - return _odp_link_info_fd(pkt_priv(pktio_entry)->sock_info.helper_sock, + return _odp_link_info_fd(pkt_priv(pktio_entry)->helper_sock, pktio_entry->s.name, info); } +static int set_queue_capability(int fd, const char *devname, odp_pktio_capability_t *capa) +{ + struct ifreq ifr; + struct ethtool_channels channels; + uint32_t max_channels; + int ret; + + memset(&channels, 0, sizeof(struct ethtool_channels)); + channels.cmd = ETHTOOL_GCHANNELS; + snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", devname); + ifr.ifr_data = (char *)&channels; + ret = ioctl(fd, SIOCETHTOOL, &ifr); + + if (ret == -1 || channels.max_combined == 0U) { + if (ret == -1 && errno != EOPNOTSUPP) { + ODP_ERR("Unable to query NIC channel capabilities: %s\n", strerror(errno)); + return -1; + } + + channels.max_combined = 1U; + } + + max_channels = _ODP_MIN((uint32_t)PKTIO_MAX_QUEUES, channels.max_combined); + capa->max_input_queues = max_channels; + capa->max_output_queues = max_channels; + + return 0; +} + static int sock_xdp_capability(pktio_entry_t *pktio_entry, odp_pktio_capability_t *capa) { - pkt_xdp_t *priv = pkt_priv(pktio_entry); + xdp_sock_info_t *priv = pkt_priv(pktio_entry); memset(capa, 0, sizeof(odp_pktio_capability_t)); - capa->max_input_queues = 1U; - capa->max_output_queues = 1U; + + if (set_queue_capability(priv->helper_sock, pktio_entry->s.name, capa)) + return -1; + capa->set_op.op.promisc_mode = 1U; capa->set_op.op.maxlen = 1U; capa->maxlen.equal = true; capa->maxlen.min_input = _ODP_SOCKET_MTU_MIN; - capa->maxlen.max_input = priv->sock_info.max_mtu; + capa->maxlen.max_input = priv->max_mtu; capa->maxlen.min_output = _ODP_SOCKET_MTU_MIN; - capa->maxlen.max_output = priv->sock_info.max_mtu; + capa->maxlen.max_output = priv->max_mtu; capa->config.parser.layer = ODP_PROTO_LAYER_ALL; - capa->stats.pktio.all_counters = 0U; - capa->stats.pktin_queue.all_counters = 0U; - capa->stats.pktout_queue.all_counters = 0U; + capa->stats.pktio.counter.in_octets = 1U; + capa->stats.pktio.counter.in_packets = 1U; + capa->stats.pktio.counter.in_errors = 1U; + capa->stats.pktio.counter.in_discards = 1U; + capa->stats.pktio.counter.out_octets = 1U; + capa->stats.pktio.counter.out_packets = 1U; + capa->stats.pktio.counter.out_discards = 1U; + + capa->stats.pktin_queue.counter.octets = 1U; + capa->stats.pktin_queue.counter.packets = 1U; + capa->stats.pktin_queue.counter.errors = 1U; + capa->stats.pktin_queue.counter.discards = 1U; + capa->stats.pktout_queue.counter.octets = 1U; + capa->stats.pktout_queue.counter.packets = 1U; + capa->stats.pktout_queue.counter.discards = 1U; + + return 0; +} + +static int sock_xdp_input_queues_config(pktio_entry_t *pktio_entry, + const odp_pktin_queue_param_t *param) +{ + xdp_sock_info_t *priv = pkt_priv(pktio_entry); + + priv->lockless_rx = pktio_entry->s.param.in_mode == ODP_PKTIN_MODE_SCHED || + param->op_mode == ODP_PKTIO_OP_MT_UNSAFE; + + return 0; +} + +static void fill_socket_config(struct xsk_socket_config *config, xdp_umem_info_t *umem_info) +{ + config->rx_size = umem_info->num_rx_desc; + config->tx_size = umem_info->num_tx_desc; + config->libxdp_flags = 0U; + config->xdp_flags = 0U; + config->bind_flags = XDP_ZEROCOPY; /* TODO: XDP_COPY */ +} + +static int sock_xdp_output_queues_config(pktio_entry_t *pktio_entry, + const odp_pktout_queue_param_t *param) +{ + xdp_sock_info_t *priv = pkt_priv(pktio_entry); + struct xsk_socket_config config; + const char *devname = pktio_entry->s.name; + uint32_t bind_q, i; + struct xsk_umem *umem; + xdp_sock_t *sock; + int ret; + + priv->lockless_tx = param->op_mode == ODP_PKTIO_OP_MT_UNSAFE; + fill_socket_config(&config, priv->umem_info); + bind_q = priv->bind_q; + umem = priv->umem_info->umem; + + for (i = 0U; i < param->num_queues; ++i) { + sock = &priv->qs[i]; + ret = xsk_socket__create_shared(&sock->xsk, devname, bind_q, umem, &sock->rx, + &sock->tx, &sock->fill_q, &sock->compl_q, &config); + + if (ret) { + ODP_ERR("Error creating xdp socket for bind queue %u: %d\n", bind_q, ret); + goto err; + } + + if (!reserve_fill_queue_elements(priv, sock, config.rx_size)) { + ODP_ERR("Unable to reserve fill queue descriptors for queue: %u.\n", + bind_q); + goto err; + } + + ++bind_q; + } + + priv->num_q = i; + /* Ring setup/clean up routines seem to be asynchronous with some drivers and might not be + * ready yet after xsk_socket__create_shared(). */ + sleep(1U); return 0; + +err: + for (uint32_t j = 0U; j < i; ++j) { + xsk_socket__delete(priv->qs[j].xsk); + priv->qs[j].xsk = NULL; + } + + return -1; } const pktio_if_ops_t _odp_sock_xdp_pktio_ops = { - /* TODO: at least stats */ .name = "socket_xdp", .print = NULL, .init_global = sock_xdp_init_global, @@ -587,13 +978,13 @@ const pktio_if_ops_t _odp_sock_xdp_pktio_ops = { .close = sock_xdp_close, .start = NULL, .stop = NULL, - .stats = NULL, - .stats_reset = NULL, - .pktin_queue_stats = NULL, - .pktout_queue_stats = NULL, - .extra_stat_info = NULL, - .extra_stats = NULL, - .extra_stat_counter = NULL, + .stats = sock_xdp_stats, + .stats_reset = sock_xdp_stats_reset, + .pktin_queue_stats = sock_xdp_pktin_queue_stats, + .pktout_queue_stats = sock_xdp_pktout_queue_stats, + .extra_stat_info = sock_xdp_extra_stat_info, + .extra_stats = sock_xdp_extra_stats, + .extra_stat_counter = sock_xdp_extra_stat_counter, .pktio_ts_res = NULL, .pktio_ts_from_ns = NULL, .pktio_time = NULL, @@ -612,8 +1003,8 @@ const pktio_if_ops_t _odp_sock_xdp_pktio_ops = { .link_info = sock_xdp_link_info, .capability = sock_xdp_capability, .config = NULL, - .input_queues_config = NULL, - .output_queues_config = NULL + .input_queues_config = sock_xdp_input_queues_config, + .output_queues_config = sock_xdp_output_queues_config }; static odp_bool_t sock_xdp_is_mem_src_active(void) @@ -647,39 +1038,13 @@ static void sock_xdp_adjust_block_size(uint8_t *data ODP_UNUSED, uint32_t *block *block_size = _ODP_MAX(size, MIN_FRAME_SIZE); } -static int sock_xdp_umem_create(uint8_t *data, pool_t *pool) -{ - struct xsk_umem_config cfg; - xdp_umem_info_t *umem_info = (xdp_umem_info_t *)data; - - umem_info->pool = pool; - /* Fill queue size is recommended to be >= HW RX ring size + AF_XDP RX - * ring size, so use size twice the size of AF_XDP RX ring. */ - cfg.fill_size = NUM_XDP_DESCS * 2U; /* TODO: num descs vs pool size */ - cfg.comp_size = NUM_XDP_DESCS; - cfg.frame_size = pool->block_size; - cfg.frame_headroom = sizeof(odp_packet_hdr_t) + pool->headroom; - cfg.flags = XDP_UMEM_UNALIGNED_CHUNK_FLAG; - - return xsk_umem__create(&umem_info->umem, pool->base_addr, pool->shm_size, - &umem_info->fill_q, &umem_info->compl_q, &cfg); -} - -static void sock_xdp_umem_delete(uint8_t *data) -{ - xdp_umem_info_t *umem_info = (xdp_umem_info_t *)data; - - while (xsk_umem__delete(umem_info->umem) == -EBUSY) - continue; -} - const _odp_pool_mem_src_ops_t _odp_pool_sock_xdp_mem_src_ops = { .name = "xdp_zc", .is_active = sock_xdp_is_mem_src_active, .force_disable = sock_xdp_force_mem_src_disable, .adjust_size = sock_xdp_adjust_block_size, - .bind = sock_xdp_umem_create, - .unbind = sock_xdp_umem_delete + .bind = NULL, + .unbind = NULL }; #else diff --git a/platform/linux-generic/pktio/tap.c b/platform/linux-generic/pktio/tap.c index 40c46b43c..c61e05575 100644 --- a/platform/linux-generic/pktio/tap.c +++ b/platform/linux-generic/pktio/tap.c @@ -285,31 +285,11 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data, { odp_packet_t pkt; odp_packet_hdr_t *pkt_hdr; - odp_packet_hdr_t parsed_hdr; int num; - uint64_t l4_part_sum = 0; uint16_t frame_offset = pktio_entry->s.pktin_frame_offset; - const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums; const odp_proto_layer_t layer = pktio_entry->s.parse_layer; const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin; - if (layer) { - packet_parse_reset(&parsed_hdr, 1); - packet_set_len(&parsed_hdr, len); - if (_odp_packet_parse_common(&parsed_hdr.p, data, len, len, layer, - chksums, &l4_part_sum, opt) < 0) { - return ODP_PACKET_INVALID; - } - - if (pktio_cls_enabled(pktio_entry)) { - if (_odp_cls_classify_packet(pktio_entry, data, - &pkt_priv(pktio_entry)->pool, - &parsed_hdr)) { - return ODP_PACKET_INVALID; - } - } - } - num = _odp_packet_alloc_multi(pkt_priv(pktio_entry)->pool, len + frame_offset, &pkt, 1); if (num != 1) @@ -327,10 +307,29 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data, } if (layer) { - _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr); + if (_odp_packet_parse_common(pkt_hdr, data, len, len, layer, + opt) < 0) { + odp_packet_free(pkt); + return ODP_PACKET_INVALID; + } + + if (pktio_cls_enabled(pktio_entry)) { + odp_pool_t new_pool; - if (layer >= ODP_PROTO_LAYER_L4) - _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum); + if (_odp_cls_classify_packet(pktio_entry, data, + &new_pool, pkt_hdr)) { + odp_packet_free(pkt); + return ODP_PACKET_INVALID; + } + + if (odp_unlikely(_odp_pktio_packet_to_pool( + &pkt, &pkt_hdr, new_pool))) { + odp_packet_free(pkt); + odp_atomic_inc_u64( + &pktio_entry->s.stats_extra.in_discards); + return ODP_PACKET_INVALID; + } + } } packet_set_ts(pkt_hdr, ts); diff --git a/platform/linux-generic/test/example/ipsec_api/pktio_env b/platform/linux-generic/test/example/ipsec_api/pktio_env index 5140b1e8f..f8cadb5cb 100644 --- a/platform/linux-generic/test/example/ipsec_api/pktio_env +++ b/platform/linux-generic/test/example/ipsec_api/pktio_env @@ -35,6 +35,11 @@ if [ -n "$WITH_OPENSSL_CRYPTO" ] && [ ${WITH_OPENSSL_CRYPTO} -eq 0 ]; then exit 77 fi +if [ ${ODPH_PROC_MODE} -eq 1 ]; then + echo "Process mode not supported. Skipping." + exit 77 +fi + # Skip live and router mode tests. if [ ${IPSEC_APP_MODE} -eq 1 ] || [ ${IPSEC_APP_MODE} -eq 2 ]; then echo "IPsec Live / Router mode test. Skipping." diff --git a/platform/linux-generic/test/example/ipsec_crypto/pktio_env b/platform/linux-generic/test/example/ipsec_crypto/pktio_env index c287b08c2..d2550063e 100644 --- a/platform/linux-generic/test/example/ipsec_crypto/pktio_env +++ b/platform/linux-generic/test/example/ipsec_crypto/pktio_env @@ -35,6 +35,11 @@ if [ -n "$WITH_OPENSSL_CRYPTO" ] && [ ${WITH_OPENSSL_CRYPTO} -eq 0 ]; then exit 77 fi +if [ ${ODPH_PROC_MODE} -eq 1 ]; then + echo "Process mode not supported. Skipping." + exit 77 +fi + # Skip live and router mode tests. if [ ${IPSEC_APP_MODE} -eq 1 ] || [ ${IPSEC_APP_MODE} -eq 2 ]; then echo "Live / Router mode test. Skipping." diff --git a/platform/linux-generic/test/inline-timer.conf b/platform/linux-generic/test/inline-timer.conf index 2ade68c32..c7379e38a 100644 --- a/platform/linux-generic/test/inline-timer.conf +++ b/platform/linux-generic/test/inline-timer.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.20" +config_file_version = "0.1.21" timer: { # Enable inline timer implementation diff --git a/platform/linux-generic/test/packet_align.conf b/platform/linux-generic/test/packet_align.conf index f0370c7a2..433899017 100644 --- a/platform/linux-generic/test/packet_align.conf +++ b/platform/linux-generic/test/packet_align.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.20" +config_file_version = "0.1.21" pool: { pkt: { diff --git a/platform/linux-generic/test/process-mode.conf b/platform/linux-generic/test/process-mode.conf index 3ec28dce9..a36c9fc3d 100644 --- a/platform/linux-generic/test/process-mode.conf +++ b/platform/linux-generic/test/process-mode.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.20" +config_file_version = "0.1.21" # Shared memory options shm: { diff --git a/platform/linux-generic/test/sched-basic.conf b/platform/linux-generic/test/sched-basic.conf index 25e8b33e0..16654595b 100644 --- a/platform/linux-generic/test/sched-basic.conf +++ b/platform/linux-generic/test/sched-basic.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.20" +config_file_version = "0.1.21" # Test scheduler with an odd spread value and without dynamic load balance sched_basic: { diff --git a/test/common/test_packet_ipv4.h b/test/common/test_packet_ipv4.h index 2bccf7b5f..8dd98d60d 100644 --- a/test/common/test_packet_ipv4.h +++ b/test/common/test_packet_ipv4.h @@ -1,5 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited - * Copyright (c) 2021, Nokia + * Copyright (c) 2021-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -212,6 +212,54 @@ static const uint8_t test_packet_ipv4_rr_nop_icmp[] = { 0x00, 0x00 }; +/* Ethernet/IPv4/UDP packet. Ethernet frame length 325 bytes (+ CRC). + * - source IP addr: 192.168.1.2 + * - destination IP addr: 192.168.1.1 + */ +static const uint8_t test_packet_ipv4_udp_325[] = { + 0x00, 0x00, 0x09, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x09, 0x00, 0x04, 0x00, 0x08, 0x00, 0x45, 0x00, + 0x01, 0x37, 0x00, 0x00, 0x00, 0x00, 0x40, 0x11, + 0xF6, 0x62, 0xC0, 0xA8, 0x01, 0x02, 0xC0, 0xA8, + 0x01, 0x01, 0x00, 0x3F, 0x00, 0x3F, 0x01, 0x23, + 0x02, 0xED, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, + 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, + 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, + 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, + 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, + 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, + 0x2E, 0x2F, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, + 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, + 0x3E, 0x3F, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, + 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, + 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, + 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, + 0x5E, 0x5F, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, + 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, + 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, + 0x76, 0x77, 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, + 0x7E, 0x7F, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, + 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, + 0x8E, 0x8F, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, + 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, + 0x9E, 0x9F, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, + 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, + 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, + 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, + 0xBE, 0xBF, 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, + 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, + 0xCE, 0xCF, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, + 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, + 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, + 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, + 0xEE, 0xEF, 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, + 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, + 0xFE, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, + 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, + 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, + 0x16, 0x17, 0x18, 0x19, 0x1A +}; + /* Ethernet/IPv4/UDP packet. Ethernet frame length 1500 bytes (+ CRC). */ static const uint8_t test_packet_ipv4_udp_1500[] = { 0x00, 0x00, 0x09, 0x00, 0x05, 0x00, 0x00, 0x00, diff --git a/test/validation/api/classification/odp_classification_basic.c b/test/validation/api/classification/odp_classification_basic.c index fb5ec4ed0..4fecba1e2 100644 --- a/test/validation/api/classification/odp_classification_basic.c +++ b/test/validation/api/classification/odp_classification_basic.c @@ -1,4 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2021-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -10,23 +11,29 @@ #define PMR_SET_NUM 5 -static void classification_test_default_values(void) +static void test_defaults(uint8_t fill) { odp_cls_cos_param_t cos_param; odp_pmr_param_t pmr_param; - memset(&cos_param, 0x55, sizeof(cos_param)); + memset(&cos_param, fill, sizeof(cos_param)); odp_cls_cos_param_init(&cos_param); CU_ASSERT_EQUAL(cos_param.num_queue, 1); CU_ASSERT_EQUAL(cos_param.red.enable, false); CU_ASSERT_EQUAL(cos_param.bp.enable, false); CU_ASSERT_EQUAL(cos_param.vector.enable, false); - memset(&pmr_param, 0x55, sizeof(pmr_param)); + memset(&pmr_param, fill, sizeof(pmr_param)); odp_cls_pmr_param_init(&pmr_param); CU_ASSERT_EQUAL(pmr_param.range_term, false); } +static void classification_test_default_values(void) +{ + test_defaults(0); + test_defaults(0xff); +} + static void classification_test_create_cos(void) { odp_cos_t cos; diff --git a/test/validation/api/crypto/odp_crypto_test_inp.c b/test/validation/api/crypto/odp_crypto_test_inp.c index e3eff88b9..0112bf27a 100644 --- a/test/validation/api/crypto/odp_crypto_test_inp.c +++ b/test/validation/api/crypto/odp_crypto_test_inp.c @@ -28,11 +28,11 @@ struct suite_context_s { static struct suite_context_s suite_context; -static void test_default_values(void) +static void test_defaults(uint8_t fill) { odp_crypto_session_param_t param; - memset(¶m, 0x55, sizeof(param)); + memset(¶m, fill, sizeof(param)); odp_crypto_session_param_init(¶m); CU_ASSERT_EQUAL(param.op, ODP_CRYPTO_OP_ENCODE); @@ -55,6 +55,12 @@ static void test_default_values(void) #endif } +static void test_default_values(void) +{ + test_defaults(0); + test_defaults(0xff); +} + static int packet_cmp_mem_bits(odp_packet_t pkt, uint32_t offset, uint8_t *s, uint32_t len) { @@ -689,7 +695,8 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op, * In some cases an individual algorithm cannot be used alone, * i.e. with the null cipher/auth algorithm. */ - if (rc == ODP_CRYPTO_SES_ERR_ALG_COMBO) { + if (rc < 0 && + status == ODP_CRYPTO_SES_ERR_ALG_COMBO) { printf("\n Unsupported algorithm combination: %s, %s\n", cipher_alg_name(cipher_alg), auth_alg_name(auth_alg)); @@ -1177,11 +1184,11 @@ static void test_capability(void) * operation with hash_result_offset outside the auth_range and by * copying the hash in the ciphertext packet. */ -static void create_hash_test_reference(odp_auth_alg_t auth, - const odp_crypto_auth_capability_t *capa, - crypto_test_reference_t *ref, - uint32_t digest_offset, - uint8_t digest_fill_byte) +static int create_hash_test_reference(odp_auth_alg_t auth, + const odp_crypto_auth_capability_t *capa, + crypto_test_reference_t *ref, + uint32_t digest_offset, + uint8_t digest_fill) { odp_crypto_session_t session; int rc; @@ -1209,7 +1216,7 @@ static void create_hash_test_reference(odp_auth_alg_t auth, fill_with_pattern(ref->auth_iv, ref->auth_iv_length); fill_with_pattern(ref->plaintext, auth_bytes); - memset(ref->plaintext + digest_offset, digest_fill_byte, ref->digest_length); + memset(ref->plaintext + digest_offset, digest_fill, ref->digest_length); pkt = odp_packet_alloc(suite_context.pool, auth_bytes + ref->digest_length); CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); @@ -1219,12 +1226,12 @@ static void create_hash_test_reference(odp_auth_alg_t auth, session = session_create(ODP_CRYPTO_OP_ENCODE, ODP_CIPHER_ALG_NULL, auth, ref, PACKET_IV, HASH_NO_OVERLAP); + if (session == ODP_CRYPTO_SESSION_INVALID) + return -1; - if (crypto_op(pkt, &ok, session, - ref->cipher_iv, ref->auth_iv, - &cipher_range, &auth_range, - ref->aad, enc_digest_offset)) - return; + rc = crypto_op(pkt, &ok, session, ref->cipher_iv, ref->auth_iv, + &cipher_range, &auth_range, ref->aad, enc_digest_offset); + CU_ASSERT(rc == 0); CU_ASSERT(ok); rc = odp_crypto_session_destroy(session); @@ -1245,6 +1252,8 @@ static void create_hash_test_reference(odp_auth_alg_t auth, CU_ASSERT(rc == 0); odp_packet_free(pkt); + + return 0; } static void test_auth_hash_in_auth_range(odp_auth_alg_t auth, @@ -1257,7 +1266,8 @@ static void test_auth_hash_in_auth_range(odp_auth_alg_t auth, * Create test packets with auth hash in the authenticated range and * zeroes in the hash location in the plaintext packet. */ - create_hash_test_reference(auth, capa, &ref, digest_offset, 0); + if (create_hash_test_reference(auth, capa, &ref, digest_offset, 0)) + return; /* * Decode the ciphertext packet. @@ -1279,7 +1289,8 @@ static void test_auth_hash_in_auth_range(odp_auth_alg_t auth, * Create test packets with auth hash in the authenticated range and * ones in the hash location in the plaintext packet. */ - create_hash_test_reference(auth, capa, &ref, digest_offset, 1); + if (create_hash_test_reference(auth, capa, &ref, digest_offset, 1)) + return; /* * Encode the plaintext packet. diff --git a/test/validation/api/dma/dma.c b/test/validation/api/dma/dma.c index 4df81ca7a..8eb75b172 100644 --- a/test/validation/api/dma/dma.c +++ b/test/validation/api/dma/dma.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2021, Nokia +/* Copyright (c) 2021-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -199,33 +199,43 @@ static void test_dma_capability(void) } } -static void test_dma_param(void) +static void test_dma_param(uint8_t fill) { odp_dma_param_t dma_param; odp_dma_transfer_param_t trs_param; odp_dma_compl_param_t compl_param; odp_dma_pool_param_t dma_pool_param; + memset(&dma_param, fill, sizeof(dma_param)); odp_dma_param_init(&dma_param); CU_ASSERT(dma_param.direction == ODP_DMA_MAIN_TO_MAIN); CU_ASSERT(dma_param.type == ODP_DMA_TYPE_COPY); CU_ASSERT(dma_param.mt_mode == ODP_DMA_MT_SAFE); CU_ASSERT(dma_param.order == ODP_DMA_ORDER_NONE); + memset(&trs_param, fill, sizeof(trs_param)); odp_dma_transfer_param_init(&trs_param); CU_ASSERT(trs_param.src_format == ODP_DMA_FORMAT_ADDR); CU_ASSERT(trs_param.dst_format == ODP_DMA_FORMAT_ADDR); CU_ASSERT(trs_param.num_src == 1); CU_ASSERT(trs_param.num_dst == 1); + memset(&compl_param, fill, sizeof(compl_param)); odp_dma_compl_param_init(&compl_param); CU_ASSERT(compl_param.user_ptr == NULL); + memset(&dma_pool_param, fill, sizeof(dma_pool_param)); odp_dma_pool_param_init(&dma_pool_param); CU_ASSERT(dma_pool_param.cache_size <= global.dma_capa.pool.max_cache_size); CU_ASSERT(dma_pool_param.cache_size >= global.dma_capa.pool.min_cache_size); } +static void test_dma_param_init(void) +{ + test_dma_param(0); + test_dma_param(0xff); +} + static void test_dma_debug(void) { odp_dma_param_t dma_param; @@ -284,6 +294,30 @@ static void test_dma_compl_pool(void) odp_dma_compl_free(compl); } +static void test_dma_compl_pool_same_name(void) +{ + odp_dma_pool_param_t dma_pool_param; + odp_pool_t pool, pool_a, pool_b; + const char *name = COMPL_POOL_NAME; + + pool_a = global.compl_pool; + + pool = odp_pool_lookup(name); + CU_ASSERT(pool == pool_a); + + odp_dma_pool_param_init(&dma_pool_param); + dma_pool_param.num = NUM_COMPL; + + /* Second pool with the same name */ + pool_b = odp_dma_pool_create(name, &dma_pool_param); + CU_ASSERT_FATAL(pool_b != ODP_POOL_INVALID); + + pool = odp_pool_lookup(name); + CU_ASSERT(pool == pool_a || pool == pool_b); + + CU_ASSERT_FATAL(odp_pool_destroy(pool_b) == 0); +} + static void init_source(uint8_t *src, uint32_t len) { uint32_t i; @@ -1139,9 +1173,10 @@ static void test_dma_multi_pkt_to_pkt_event(void) odp_testinfo_t dma_suite[] = { ODP_TEST_INFO(test_dma_capability), - ODP_TEST_INFO_CONDITIONAL(test_dma_param, check_sync), + ODP_TEST_INFO_CONDITIONAL(test_dma_param_init, check_sync), ODP_TEST_INFO_CONDITIONAL(test_dma_debug, check_sync), ODP_TEST_INFO_CONDITIONAL(test_dma_compl_pool, check_event), + ODP_TEST_INFO_CONDITIONAL(test_dma_compl_pool_same_name, check_event), ODP_TEST_INFO_CONDITIONAL(test_dma_addr_to_addr_sync, check_sync), ODP_TEST_INFO_CONDITIONAL(test_dma_addr_to_addr_sync_mtrs, check_sync), ODP_TEST_INFO_CONDITIONAL(test_dma_addr_to_addr_sync_mseg, check_sync), diff --git a/test/validation/api/init/init_main.c b/test/validation/api/init/init_main.c index f1716db25..f8fd96aeb 100644 --- a/test/validation/api/init/init_main.c +++ b/test/validation/api/init/init_main.c @@ -1,5 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2019-2021, Nokia + * Copyright (c) 2019-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -48,6 +48,22 @@ static int my_log_thread_func(odp_log_level_t level, const char *fmt, ...) return 0; } +static void test_param_init(uint8_t fill) +{ + odp_init_t param; + + memset(¶m, fill, sizeof(param)); + odp_init_param_init(¶m); + CU_ASSERT(param.mem_model == ODP_MEM_MODEL_THREAD); + CU_ASSERT(param.shm.max_memory == 0); +} + +static void init_test_param_init(void) +{ + test_param_init(0); + test_param_init(0xff); +} + static void init_test_defaults(void) { int ret; @@ -226,6 +242,7 @@ static void init_test_feature_disabled(void) } odp_testinfo_t testinfo[] = { + ODP_TEST_INFO(init_test_param_init), ODP_TEST_INFO(init_test_defaults), ODP_TEST_INFO(init_test_abort), ODP_TEST_INFO(init_test_log), diff --git a/test/validation/api/ipsec/ipsec_test_out.c b/test/validation/api/ipsec/ipsec_test_out.c index 4e3230844..9604e65e8 100644 --- a/test/validation/api/ipsec/ipsec_test_out.c +++ b/test/validation/api/ipsec/ipsec_test_out.c @@ -1,6 +1,6 @@ /* Copyright (c) 2017-2018, Linaro Limited * Copyright (c) 2020, Marvell - * Copyright (c) 2020-2021, Nokia + * Copyright (c) 2020-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -1598,14 +1598,12 @@ static void ipsec_test_capability(void) CU_ASSERT(odp_ipsec_capability(&capa) == 0); } -static void ipsec_test_default_values(void) +static void test_defaults(uint8_t fill) { odp_ipsec_config_t config; odp_ipsec_sa_param_t sa_param; - memset(&config, 0x55, sizeof(config)); - memset(&sa_param, 0x55, sizeof(sa_param)); - + memset(&config, fill, sizeof(config)); odp_ipsec_config_init(&config); CU_ASSERT(config.inbound.lookup.min_spi == 0); CU_ASSERT(config.inbound.lookup.max_spi == UINT32_MAX); @@ -1623,6 +1621,7 @@ static void ipsec_test_default_values(void) CU_ASSERT(!config.stats_en); CU_ASSERT(!config.vector.enable); + memset(&sa_param, fill, sizeof(sa_param)); odp_ipsec_sa_param_init(&sa_param); CU_ASSERT(sa_param.proto == ODP_IPSEC_ESP); CU_ASSERT(sa_param.crypto.cipher_alg == ODP_CIPHER_ALG_NULL); @@ -1654,6 +1653,12 @@ static void ipsec_test_default_values(void) CU_ASSERT(sa_param.outbound.frag_mode == ODP_IPSEC_FRAG_DISABLED); } +static void ipsec_test_default_values(void) +{ + test_defaults(0); + test_defaults(0xff); +} + static void test_ipsec_stats(void) { ipsec_test_flags flags; diff --git a/test/validation/api/pktio/lso.c b/test/validation/api/pktio/lso.c index bde94816e..3e033ee8b 100644 --- a/test/validation/api/pktio/lso.c +++ b/test/validation/api/pktio/lso.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2020, Nokia +/* Copyright (c) 2020-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -20,10 +20,6 @@ /* Maximum number of segments test is prepared to receive per outgoing packet */ #define MAX_NUM_SEG 256 -/* Max payload bytes per LSO segment */ -#define CUSTOM_MAX_PAYLOAD 288 -#define IPV4_MAX_PAYLOAD 700 - /* Pktio interface info */ typedef struct { @@ -59,6 +55,8 @@ odp_pool_t lso_pool = ODP_POOL_INVALID; /* Check test packet size */ ODP_STATIC_ASSERT(sizeof(test_packet_ipv4_udp_1500) == 1500, "error: size is not 1500"); +ODP_STATIC_ASSERT(sizeof(test_packet_ipv4_udp_325) == 325, "error: size is not 325"); +ODP_STATIC_ASSERT(sizeof(test_packet_custom_eth_1) == 723, "error: size is not 723"); static inline void wait_linkup(odp_pktio_t pktio) { @@ -465,11 +463,14 @@ static int check_lso_custom(void) return ODP_TEST_ACTIVE; } -static int check_lso_custom_restart(void) +static int check_lso_custom_segs(uint32_t num) { if (check_lso_custom() == ODP_TEST_INACTIVE) return ODP_TEST_INACTIVE; + if (num > pktio_a->capa.lso.max_segments) + return ODP_TEST_INACTIVE; + if (disable_restart && num_starts > 0) return ODP_TEST_INACTIVE; @@ -479,6 +480,21 @@ static int check_lso_custom_restart(void) return ODP_TEST_ACTIVE; } +static int check_lso_custom_segs_1(void) +{ + return check_lso_custom_segs(1); +} + +static int check_lso_custom_segs_2(void) +{ + return check_lso_custom_segs(2); +} + +static int check_lso_custom_segs_3(void) +{ + return check_lso_custom_segs(3); +} + static int check_lso_ipv4(void) { if (pktio_a->capa.lso.max_profiles == 0 || pktio_a->capa.lso.max_profiles_per_pktio == 0) @@ -490,11 +506,14 @@ static int check_lso_ipv4(void) return ODP_TEST_ACTIVE; } -static int check_lso_ipv4_restart(void) +static int check_lso_ipv4_segs(uint32_t num) { if (check_lso_ipv4() == ODP_TEST_INACTIVE) return ODP_TEST_INACTIVE; + if (num > pktio_a->capa.lso.max_segments) + return ODP_TEST_INACTIVE; + if (disable_restart && num_starts > 0) return ODP_TEST_INACTIVE; @@ -503,6 +522,21 @@ static int check_lso_ipv4_restart(void) return ODP_TEST_ACTIVE; } +static int check_lso_ipv4_segs_1(void) +{ + return check_lso_ipv4_segs(1); +} + +static int check_lso_ipv4_segs_2(void) +{ + return check_lso_ipv4_segs(2); +} + +static int check_lso_ipv4_segs_3(void) +{ + return check_lso_ipv4_segs(3); +} + static void lso_capability(void) { /* LSO not supported when max_profiles is zero */ @@ -620,7 +654,8 @@ static void test_lso_request_clear(odp_lso_profile_t lso_profile, const uint8_t odp_packet_free(pkt); } -static void lso_send_custom_eth(int use_opt) +static void lso_send_custom_eth(const uint8_t *test_packet, uint32_t pkt_len, uint32_t max_payload, + int use_opt) { int i, ret, num; odp_lso_profile_param_t param; @@ -632,9 +667,7 @@ static void lso_send_custom_eth(int use_opt) uint32_t hdr_len = 22; /* Offset to "segment number" field */ uint32_t segnum_offset = 16; - uint32_t pkt_len = sizeof(test_packet_custom_eth_1); uint32_t sent_payload = pkt_len - hdr_len; - uint32_t max_payload = CUSTOM_MAX_PAYLOAD; odp_lso_profile_param_init(¶m); param.lso_proto = ODP_LSO_PROTO_CUSTOM; @@ -648,10 +681,10 @@ static void lso_send_custom_eth(int use_opt) CU_ASSERT_FATAL(start_interfaces() == 0); - test_lso_request_clear(profile, test_packet_custom_eth_1, pkt_len, hdr_len, max_payload); + test_lso_request_clear(profile, test_packet, pkt_len, hdr_len, max_payload); - ret = send_packets(profile, pktio_a, pktio_b, test_packet_custom_eth_1, - pkt_len, hdr_len, max_payload, 0, use_opt); + ret = send_packets(profile, pktio_a, pktio_b, test_packet, pkt_len, hdr_len, + max_payload, 0, use_opt); CU_ASSERT_FATAL(ret == 0); ODPH_DBG("\n Sent payload length: %u bytes\n", sent_payload); @@ -686,7 +719,7 @@ static void lso_send_custom_eth(int use_opt) ODPH_DBG(" LSO segment[%u] payload: %u bytes\n", segnum, payload_len); - CU_ASSERT(payload_len <= CUSTOM_MAX_PAYLOAD); + CU_ASSERT(payload_len <= max_payload); if (compare_data(pkt_out[i], hdr_len, test_packet_custom_eth_1 + offset, payload_len) >= 0) { @@ -710,17 +743,51 @@ static void lso_send_custom_eth(int use_opt) CU_ASSERT_FATAL(odp_lso_profile_destroy(profile) == 0); } -static void lso_send_custom_eth_use_pkt_meta(void) +static void lso_send_custom_eth_723(uint32_t max_payload, int use_opt) { - lso_send_custom_eth(0); + uint32_t pkt_len = sizeof(test_packet_custom_eth_1); + + if (max_payload > pktio_a->capa.lso.max_payload_len) + max_payload = pktio_a->capa.lso.max_payload_len; + + lso_send_custom_eth(test_packet_custom_eth_1, pkt_len, max_payload, use_opt); +} + +/* No segmentation needed: packet size 723 bytes, LSO segment payload 800 bytes */ +static void lso_send_custom_eth_723_800_pkt_meta(void) +{ + lso_send_custom_eth_723(800, 0); } -static void lso_send_custom_eth_use_opt(void) +static void lso_send_custom_eth_723_800_opt(void) { - lso_send_custom_eth(1); + lso_send_custom_eth_723(800, 1); } -static void lso_send_ipv4(int use_opt) +/* At least 2 segments: packet size 723 bytes, LSO segment payload 500 bytes */ +static void lso_send_custom_eth_723_500_pkt_meta(void) +{ + lso_send_custom_eth_723(500, 0); +} + +static void lso_send_custom_eth_723_500_opt(void) +{ + lso_send_custom_eth_723(500, 1); +} + +/* At least 3 segments: packet size 723 bytes, LSO segment payload 288 bytes */ +static void lso_send_custom_eth_723_288_pkt_meta(void) +{ + lso_send_custom_eth_723(288, 0); +} + +static void lso_send_custom_eth_723_288_opt(void) +{ + lso_send_custom_eth_723(288, 1); +} + +static void lso_send_ipv4(const uint8_t *test_packet, uint32_t pkt_len, uint32_t max_payload, + int use_opt) { int i, ret, num; odp_lso_profile_param_t param; @@ -729,9 +796,7 @@ static void lso_send_ipv4(int use_opt) odp_packet_t packet[MAX_NUM_SEG]; /* Ethernet 14B + IPv4 header 20B */ uint32_t hdr_len = 34; - uint32_t pkt_len = sizeof(test_packet_ipv4_udp_1500); uint32_t sent_payload = pkt_len - hdr_len; - uint32_t max_payload = IPV4_MAX_PAYLOAD; odp_lso_profile_param_init(¶m); param.lso_proto = ODP_LSO_PROTO_IPV4; @@ -741,10 +806,10 @@ static void lso_send_ipv4(int use_opt) CU_ASSERT_FATAL(start_interfaces() == 0); - test_lso_request_clear(profile, test_packet_ipv4_udp_1500, pkt_len, hdr_len, max_payload); + test_lso_request_clear(profile, test_packet, pkt_len, hdr_len, max_payload); - ret = send_packets(profile, pktio_a, pktio_b, test_packet_ipv4_udp_1500, - pkt_len, hdr_len, max_payload, 14, use_opt); + ret = send_packets(profile, pktio_a, pktio_b, test_packet, pkt_len, + hdr_len, max_payload, 14, use_opt); CU_ASSERT_FATAL(ret == 0); ODPH_DBG("\n Sent payload length: %u bytes\n", sent_payload); @@ -773,12 +838,13 @@ static void lso_send_ipv4(int use_opt) ODPH_DBG(" LSO segment[%i] payload: %u bytes\n", i, payload_len); - CU_ASSERT(odp_packet_has_ipfrag(packet[i])); CU_ASSERT(odp_packet_has_error(packet[i]) == 0); - CU_ASSERT(payload_len <= IPV4_MAX_PAYLOAD); + CU_ASSERT(payload_len <= max_payload); - if (compare_data(packet[i], hdr_len, - test_packet_ipv4_udp_1500 + offset, payload_len) >= 0) { + if (pkt_len > max_payload) + CU_ASSERT(odp_packet_has_ipfrag(packet[i])); + + if (compare_data(packet[i], hdr_len, test_packet + offset, payload_len) >= 0) { ODPH_ERR(" Payload compare failed at offset %u\n", offset); CU_FAIL("Payload compare failed\n"); } @@ -799,23 +865,74 @@ static void lso_send_ipv4(int use_opt) CU_ASSERT_FATAL(odp_lso_profile_destroy(profile) == 0); } -static void lso_send_ipv4_use_pkt_meta(void) +static void lso_send_ipv4_udp_325(uint32_t max_payload, int use_opt) +{ + uint32_t pkt_len = sizeof(test_packet_ipv4_udp_325); + + if (max_payload > pktio_a->capa.lso.max_payload_len) + max_payload = pktio_a->capa.lso.max_payload_len; + + lso_send_ipv4(test_packet_ipv4_udp_325, pkt_len, max_payload, use_opt); +} + +static void lso_send_ipv4_udp_1500(uint32_t max_payload, int use_opt) +{ + uint32_t pkt_len = sizeof(test_packet_ipv4_udp_1500); + + if (max_payload > pktio_a->capa.lso.max_payload_len) + max_payload = pktio_a->capa.lso.max_payload_len; + + lso_send_ipv4(test_packet_ipv4_udp_1500, pkt_len, max_payload, use_opt); +} + +/* No segmentation needed: packet size 325 bytes, LSO segment payload 700 bytes */ +static void lso_send_ipv4_325_700_pkt_meta(void) +{ + lso_send_ipv4_udp_325(700, 0); +} + +static void lso_send_ipv4_325_700_opt(void) +{ + lso_send_ipv4_udp_325(700, 1); +} + +/* At least 2 segments: packet size 1500 bytes, LSO segment payload 1000 bytes */ +static void lso_send_ipv4_1500_1000_pkt_meta(void) +{ + lso_send_ipv4_udp_1500(1000, 0); +} + +static void lso_send_ipv4_1500_1000_opt(void) +{ + lso_send_ipv4_udp_1500(1000, 1); +} + +/* At least 3 segments: packet size 1500 bytes, LSO segment payload 700 bytes */ +static void lso_send_ipv4_1500_700_pkt_meta(void) { - lso_send_ipv4(0); + lso_send_ipv4_udp_1500(700, 0); } -static void lso_send_ipv4_use_opt(void) +static void lso_send_ipv4_1500_700_opt(void) { - lso_send_ipv4(1); + lso_send_ipv4_udp_1500(700, 1); } odp_testinfo_t lso_suite[] = { ODP_TEST_INFO(lso_capability), ODP_TEST_INFO_CONDITIONAL(lso_create_ipv4_profile, check_lso_ipv4), ODP_TEST_INFO_CONDITIONAL(lso_create_custom_profile, check_lso_custom), - ODP_TEST_INFO_CONDITIONAL(lso_send_ipv4_use_pkt_meta, check_lso_ipv4_restart), - ODP_TEST_INFO_CONDITIONAL(lso_send_ipv4_use_opt, check_lso_ipv4_restart), - ODP_TEST_INFO_CONDITIONAL(lso_send_custom_eth_use_pkt_meta, check_lso_custom_restart), - ODP_TEST_INFO_CONDITIONAL(lso_send_custom_eth_use_opt, check_lso_custom_restart), + ODP_TEST_INFO_CONDITIONAL(lso_send_ipv4_325_700_pkt_meta, check_lso_ipv4_segs_1), + ODP_TEST_INFO_CONDITIONAL(lso_send_ipv4_325_700_opt, check_lso_ipv4_segs_1), + ODP_TEST_INFO_CONDITIONAL(lso_send_ipv4_1500_1000_pkt_meta, check_lso_ipv4_segs_2), + ODP_TEST_INFO_CONDITIONAL(lso_send_ipv4_1500_1000_opt, check_lso_ipv4_segs_2), + ODP_TEST_INFO_CONDITIONAL(lso_send_ipv4_1500_700_pkt_meta, check_lso_ipv4_segs_3), + ODP_TEST_INFO_CONDITIONAL(lso_send_ipv4_1500_700_opt, check_lso_ipv4_segs_3), + ODP_TEST_INFO_CONDITIONAL(lso_send_custom_eth_723_800_pkt_meta, check_lso_custom_segs_1), + ODP_TEST_INFO_CONDITIONAL(lso_send_custom_eth_723_800_opt, check_lso_custom_segs_1), + ODP_TEST_INFO_CONDITIONAL(lso_send_custom_eth_723_500_pkt_meta, check_lso_custom_segs_2), + ODP_TEST_INFO_CONDITIONAL(lso_send_custom_eth_723_500_opt, check_lso_custom_segs_2), + ODP_TEST_INFO_CONDITIONAL(lso_send_custom_eth_723_288_pkt_meta, check_lso_custom_segs_3), + ODP_TEST_INFO_CONDITIONAL(lso_send_custom_eth_723_288_opt, check_lso_custom_segs_3), ODP_TEST_INFO_NULL }; diff --git a/test/validation/api/pktio/pktio.c b/test/validation/api/pktio/pktio.c index d3ce41a2c..9a33fd983 100644 --- a/test/validation/api/pktio/pktio.c +++ b/test/validation/api/pktio/pktio.c @@ -1578,19 +1578,19 @@ static void pktio_test_mac(void) CU_ASSERT(0 == ret); } -static void pktio_test_default_values(void) +static void test_defaults(uint8_t fill) { odp_pktio_param_t pktio_p; odp_pktin_queue_param_t qp_in; odp_pktout_queue_param_t qp_out; odp_pktio_config_t pktio_conf; - memset(&pktio_p, 0x55, sizeof(pktio_p)); + memset(&pktio_p, fill, sizeof(pktio_p)); odp_pktio_param_init(&pktio_p); CU_ASSERT_EQUAL(pktio_p.in_mode, ODP_PKTIN_MODE_DIRECT); CU_ASSERT_EQUAL(pktio_p.out_mode, ODP_PKTOUT_MODE_DIRECT); - memset(&qp_in, 0x55, sizeof(qp_in)); + memset(&qp_in, fill, sizeof(qp_in)); odp_pktin_queue_param_init(&qp_in); CU_ASSERT_EQUAL(qp_in.op_mode, ODP_PKTIO_OP_MT); CU_ASSERT_EQUAL(qp_in.classifier_enable, 0); @@ -1609,13 +1609,13 @@ static void pktio_test_default_values(void) CU_ASSERT_EQUAL(qp_in.queue_param_ovr, NULL); CU_ASSERT_EQUAL(qp_in.vector.enable, false); - memset(&qp_out, 0x55, sizeof(qp_out)); + memset(&qp_out, fill, sizeof(qp_out)); odp_pktout_queue_param_init(&qp_out); CU_ASSERT_EQUAL(qp_out.op_mode, ODP_PKTIO_OP_MT); CU_ASSERT_EQUAL(qp_out.num_queues, 1); CU_ASSERT_EQUAL(qp_out.queue_size[0], 0); - memset(&pktio_conf, 0x55, sizeof(pktio_conf)); + memset(&pktio_conf, fill, sizeof(pktio_conf)); odp_pktio_config_init(&pktio_conf); CU_ASSERT_EQUAL(pktio_conf.pktin.all_bits, 0); CU_ASSERT_EQUAL(pktio_conf.pktout.all_bits, 0); @@ -1630,6 +1630,12 @@ static void pktio_test_default_values(void) CU_ASSERT_EQUAL(pktio_conf.reassembly.max_num_frags, 2); } +static void pktio_test_default_values(void) +{ + test_defaults(0); + test_defaults(0xff); +} + static void pktio_test_open(void) { odp_pktio_t pktio; diff --git a/test/validation/api/pool/pool.c b/test/validation/api/pool/pool.c index 8dad89d81..3befe4939 100644 --- a/test/validation/api/pool/pool.c +++ b/test/validation/api/pool/pool.c @@ -42,10 +42,11 @@ static odp_pool_capability_t global_pool_capa; static odp_pool_param_t default_pool_param; static odp_pool_ext_capability_t global_pool_ext_capa; -static void pool_test_param_init(void) +static void test_param_init(uint8_t fill) { odp_pool_param_t param; + memset(¶m, fill, sizeof(param)); odp_pool_param_init(¶m); CU_ASSERT(param.buf.cache_size >= global_pool_capa.buf.min_cache_size && @@ -63,6 +64,12 @@ static void pool_test_param_init(void) param.vector.cache_size <= global_pool_capa.vector.max_cache_size); } +static void pool_test_param_init(void) +{ + test_param_init(0); + test_param_init(0xff); +} + static void pool_create_destroy(odp_pool_param_t *param) { odp_pool_t pool; @@ -166,6 +173,79 @@ static void pool_test_lookup_info_print(void) CU_ASSERT(odp_pool_destroy(pool) == 0); } +static void pool_test_same_name(const odp_pool_param_t *param) +{ + odp_pool_t pool, pool_a, pool_b; + const char *name = "same_name"; + + pool_a = odp_pool_create(name, param); + CU_ASSERT_FATAL(pool_a != ODP_POOL_INVALID); + + pool = odp_pool_lookup(name); + CU_ASSERT(pool == pool_a); + + /* Second pool with the same name */ + pool_b = odp_pool_create(name, param); + CU_ASSERT_FATAL(pool_b != ODP_POOL_INVALID); + + pool = odp_pool_lookup(name); + CU_ASSERT(pool == pool_a || pool == pool_b); + + CU_ASSERT(odp_pool_destroy(pool_a) == 0); + CU_ASSERT(odp_pool_destroy(pool_b) == 0); +} + +static void pool_test_same_name_buf(void) +{ + odp_pool_param_t param; + + odp_pool_param_init(¶m); + + param.type = ODP_POOL_BUFFER; + param.buf.size = BUF_SIZE; + param.buf.num = BUF_NUM; + + pool_test_same_name(¶m); +} + +static void pool_test_same_name_pkt(void) +{ + odp_pool_param_t param; + + odp_pool_param_init(¶m); + + param.type = ODP_POOL_PACKET; + param.pkt.len = PKT_LEN; + param.pkt.num = PKT_NUM; + + pool_test_same_name(¶m); +} + +static void pool_test_same_name_tmo(void) +{ + odp_pool_param_t param; + + odp_pool_param_init(¶m); + + param.type = ODP_POOL_TIMEOUT; + param.tmo.num = TMO_NUM; + + pool_test_same_name(¶m); +} + +static void pool_test_same_name_vec(void) +{ + odp_pool_param_t param; + + odp_pool_param_init(¶m); + + param.type = ODP_POOL_VECTOR; + param.vector.num = 10; + param.vector.max_size = 2; + + pool_test_same_name(¶m); +} + static void alloc_buffer(uint32_t cache_size) { odp_pool_t pool; @@ -1291,13 +1371,13 @@ static void test_packet_pool_ext_capa(void) CU_ASSERT(capa.pkt.max_headroom_size > 0); CU_ASSERT(capa.pkt.max_headroom_size >= capa.pkt.max_headroom); CU_ASSERT(capa.pkt.max_segs_per_pkt > 0); - CU_ASSERT(capa.pkt.max_uarea_size > 0); } -static void test_packet_pool_ext_param_init(void) +static void test_ext_param_init(uint8_t fill) { odp_pool_ext_param_t param; + memset(¶m, fill, sizeof(param)); odp_pool_ext_param_init(ODP_POOL_PACKET, ¶m); CU_ASSERT(param.type == ODP_POOL_PACKET); @@ -1308,6 +1388,12 @@ static void test_packet_pool_ext_param_init(void) CU_ASSERT(param.pkt.uarea_size == 0); } +static void test_packet_pool_ext_param_init(void) +{ + test_ext_param_init(0); + test_ext_param_init(0xff); +} + static void test_packet_pool_ext_create(void) { odp_pool_t pool; @@ -1763,6 +1849,11 @@ odp_testinfo_t pool_suite[] = { ODP_TEST_INFO(pool_test_create_destroy_packet), ODP_TEST_INFO(pool_test_create_destroy_timeout), ODP_TEST_INFO(pool_test_create_destroy_vector), + ODP_TEST_INFO(pool_test_lookup_info_print), + ODP_TEST_INFO(pool_test_same_name_buf), + ODP_TEST_INFO(pool_test_same_name_pkt), + ODP_TEST_INFO(pool_test_same_name_tmo), + ODP_TEST_INFO(pool_test_same_name_vec), ODP_TEST_INFO(pool_test_alloc_buffer), ODP_TEST_INFO(pool_test_alloc_buffer_min_cache), ODP_TEST_INFO(pool_test_alloc_buffer_max_cache), @@ -1777,7 +1868,6 @@ odp_testinfo_t pool_suite[] = { ODP_TEST_INFO(pool_test_alloc_timeout_min_cache), ODP_TEST_INFO(pool_test_alloc_timeout_max_cache), ODP_TEST_INFO(pool_test_info_packet), - ODP_TEST_INFO(pool_test_lookup_info_print), ODP_TEST_INFO(pool_test_info_data_range), ODP_TEST_INFO(pool_test_buf_max_num), ODP_TEST_INFO(pool_test_pkt_max_num), diff --git a/test/validation/api/queue/queue.c b/test/validation/api/queue/queue.c index ec6863628..cd5e030d3 100644 --- a/test/validation/api/queue/queue.c +++ b/test/validation/api/queue/queue.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2021, Nokia + * Copyright (c) 2021-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -134,6 +134,32 @@ static void queue_test_capa(void) CU_ASSERT(capa.max_queues >= capa.plain.waitfree.max_num); } +static void test_defaults(uint8_t fill) +{ + odp_queue_param_t param; + + memset(¶m, fill, sizeof(param)); + odp_queue_param_init(¶m); + CU_ASSERT(param.type == ODP_QUEUE_TYPE_PLAIN); + CU_ASSERT(param.enq_mode == ODP_QUEUE_OP_MT); + CU_ASSERT(param.deq_mode == ODP_QUEUE_OP_MT); + CU_ASSERT(param.sched.prio == odp_schedule_default_prio()); + CU_ASSERT(param.sched.sync == ODP_SCHED_SYNC_PARALLEL); + CU_ASSERT(param.sched.group == ODP_SCHED_GROUP_ALL); + CU_ASSERT(param.sched.lock_count == 0); + CU_ASSERT(param.order == ODP_QUEUE_ORDER_KEEP); + CU_ASSERT(param.nonblocking == ODP_BLOCKING); + CU_ASSERT(param.context == NULL); + CU_ASSERT(param.context_len == 0); + CU_ASSERT(param.size == 0); +} + +static void queue_test_param_init(void) +{ + test_defaults(0); + test_defaults(0xff); +} + static void queue_test_max_plain(void) { odp_queue_capability_t capa; @@ -607,20 +633,7 @@ static void queue_test_param(void) odp_queue_param_t qparams; odp_buffer_t enbuf; - /* Defaults */ odp_queue_param_init(&qparams); - CU_ASSERT(qparams.type == ODP_QUEUE_TYPE_PLAIN); - CU_ASSERT(qparams.enq_mode == ODP_QUEUE_OP_MT); - CU_ASSERT(qparams.deq_mode == ODP_QUEUE_OP_MT); - CU_ASSERT(qparams.sched.prio == odp_schedule_default_prio()); - CU_ASSERT(qparams.sched.sync == ODP_SCHED_SYNC_PARALLEL); - CU_ASSERT(qparams.sched.group == ODP_SCHED_GROUP_ALL); - CU_ASSERT(qparams.sched.lock_count == 0); - CU_ASSERT(qparams.order == ODP_QUEUE_ORDER_KEEP); - CU_ASSERT(qparams.nonblocking == ODP_BLOCKING); - CU_ASSERT(qparams.context == NULL); - CU_ASSERT(qparams.context_len == 0); - CU_ASSERT(qparams.size == 0); /* Schedule type queue */ qparams.type = ODP_QUEUE_TYPE_SCHED; @@ -714,6 +727,44 @@ static void queue_test_param(void) CU_ASSERT(odp_queue_destroy(queue) == 0); } +static void queue_test_same_name(int sched) +{ + odp_queue_t queue, queue_a, queue_b; + odp_queue_param_t param; + const char *name = "same_name"; + + odp_queue_param_init(¶m); + + if (sched) + param.type = ODP_QUEUE_TYPE_SCHED; + + queue_a = odp_queue_create(name, ¶m); + CU_ASSERT_FATAL(queue_a != ODP_QUEUE_INVALID); + + queue = odp_queue_lookup(name); + CU_ASSERT(queue == queue_a); + + /* Second queue with the same name */ + queue_b = odp_queue_create(name, ¶m); + CU_ASSERT_FATAL(queue_b != ODP_QUEUE_INVALID); + + queue = odp_queue_lookup(name); + CU_ASSERT(queue == queue_a || queue == queue_b); + + CU_ASSERT_FATAL(odp_queue_destroy(queue_a) == 0); + CU_ASSERT_FATAL(odp_queue_destroy(queue_b) == 0); +} + +static void queue_test_same_name_plain(void) +{ + queue_test_same_name(0); +} + +static void queue_test_same_name_sched(void) +{ + queue_test_same_name(1); +} + static void queue_test_info(void) { odp_queue_t q_plain, q_order; @@ -756,6 +807,10 @@ static void queue_test_info(void) CU_ASSERT(strcmp(nq_plain, info.name) == 0); CU_ASSERT(info.param.type == ODP_QUEUE_TYPE_PLAIN); CU_ASSERT(info.param.type == odp_queue_type(q_plain)); + CU_ASSERT(info.param.enq_mode == ODP_QUEUE_OP_MT); + CU_ASSERT(info.param.deq_mode == ODP_QUEUE_OP_MT); + CU_ASSERT(info.param.order == ODP_QUEUE_ORDER_KEEP); + CU_ASSERT(info.param.nonblocking == ODP_BLOCKING); ctx = info.param.context; /* 'char' context ptr */ CU_ASSERT(ctx == q_plain_ctx); CU_ASSERT(info.param.context == odp_queue_context(q_plain)); @@ -766,6 +821,10 @@ static void queue_test_info(void) CU_ASSERT(strcmp(nq_order, info.name) == 0); CU_ASSERT(info.param.type == ODP_QUEUE_TYPE_SCHED); CU_ASSERT(info.param.type == odp_queue_type(q_order)); + CU_ASSERT(info.param.enq_mode == ODP_QUEUE_OP_MT); + CU_ASSERT(info.param.deq_mode == ODP_QUEUE_OP_DISABLED); + CU_ASSERT(info.param.order == ODP_QUEUE_ORDER_KEEP); + CU_ASSERT(info.param.nonblocking == ODP_BLOCKING); ctx = info.param.context; /* 'char' context ptr */ CU_ASSERT(ctx == q_order_ctx); CU_ASSERT(info.param.context == odp_queue_context(q_order)); @@ -985,6 +1044,7 @@ static void queue_test_mt_plain_nonblock_lf(void) odp_testinfo_t queue_suite[] = { ODP_TEST_INFO(queue_test_capa), + ODP_TEST_INFO(queue_test_param_init), ODP_TEST_INFO(queue_test_mode), ODP_TEST_INFO(queue_test_max_plain), ODP_TEST_INFO(queue_test_burst), @@ -1004,6 +1064,8 @@ odp_testinfo_t queue_suite[] = { ODP_TEST_INFO(queue_test_pair_lf_mpsc), ODP_TEST_INFO(queue_test_pair_lf_spsc), ODP_TEST_INFO(queue_test_param), + ODP_TEST_INFO(queue_test_same_name_plain), + ODP_TEST_INFO(queue_test_same_name_sched), ODP_TEST_INFO(queue_test_info), ODP_TEST_INFO(queue_test_mt_plain_block), ODP_TEST_INFO(queue_test_mt_plain_nonblock_lf), diff --git a/test/validation/api/scheduler/scheduler.c b/test/validation/api/scheduler/scheduler.c index 490ac9fea..a9c94f5a8 100644 --- a/test/validation/api/scheduler/scheduler.c +++ b/test/validation/api/scheduler/scheduler.c @@ -89,6 +89,10 @@ typedef struct { odp_queue_t sched; odp_queue_t plain; } sched_and_plain_q; + struct { + odp_atomic_u32_t helper_ready; + odp_atomic_u32_t helper_active; + } order_wait; } test_globals_t; typedef struct { @@ -145,10 +149,11 @@ static void release_context(odp_schedule_sync_t sync) odp_schedule_release_ordered(); } -static void scheduler_test_init(void) +static void test_init(uint8_t fill) { odp_schedule_config_t default_config; + memset(&default_config, fill, sizeof(default_config)); odp_schedule_config_init(&default_config); CU_ASSERT(default_config.max_flow_id == 0); @@ -158,6 +163,12 @@ static void scheduler_test_init(void) CU_ASSERT(default_config.sched_group.worker); } +static void scheduler_test_init(void) +{ + test_init(0); + test_init(0xff); +} + static void scheduler_test_capa(void) { odp_schedule_capability_t sched_capa; @@ -2316,13 +2327,10 @@ static void enqueue_event(odp_queue_t queue) static void scheduler_test_order_wait_1_thread(void) { - odp_schedule_capability_t sched_capa; odp_queue_param_t queue_param; odp_queue_t queue; odp_event_t ev; - CU_ASSERT(!odp_schedule_capability(&sched_capa)); - sched_queue_param_init(&queue_param); queue_param.sched.sync = ODP_SCHED_SYNC_ORDERED; queue = odp_queue_create("ordered queue", &queue_param); @@ -2336,8 +2344,6 @@ static void scheduler_test_order_wait_1_thread(void) CU_ASSERT_FATAL(ev != ODP_EVENT_INVALID); odp_event_free(ev); - /* Fail build if the capability field does not exist */ - printf(" (capa=%d) ", sched_capa.order_wait); /* Check that order wait does not get stuck or crash */ odp_schedule_order_wait(); @@ -2348,6 +2354,137 @@ static void scheduler_test_order_wait_1_thread(void) CU_ASSERT(odp_queue_destroy(queue) == 0); } +static int order_wait_helper(void *arg ODP_UNUSED) +{ + odp_event_t ev; + + ev = odp_schedule(NULL, odp_schedule_wait_time(ODP_TIME_SEC_IN_NS)); + + if (ev != ODP_EVENT_INVALID) { + odp_event_free(ev); + + odp_atomic_store_rel_u32(&globals->order_wait.helper_active, 1); + odp_atomic_store_rel_u32(&globals->order_wait.helper_ready, 1); + + /* Wait that the main thread can attempt to overtake us */ + odp_time_wait_ns(ODP_TIME_SEC_IN_NS); + + odp_atomic_store_rel_u32(&globals->order_wait.helper_active, 0); + } + + /* We are not interested in further events */ + odp_schedule_pause(); + /* Release context */ + while ((ev = odp_schedule(NULL, ODP_SCHED_NO_WAIT)) + != ODP_EVENT_INVALID) { + /* We got an event that was meant for the main thread */ + odp_event_free(ev); + } + + return 0; +} + +static void scheduler_test_order_wait_2_threads(void) +{ + odp_schedule_capability_t sched_capa; + odp_queue_param_t queue_param; + odp_queue_t queue; + pthrd_arg thr_arg = {.numthrds = 1}; + int ret; + odp_time_t start; + odp_event_t ev; + + CU_ASSERT(!odp_schedule_capability(&sched_capa)); + + sched_queue_param_init(&queue_param); + queue_param.sched.sync = ODP_SCHED_SYNC_ORDERED; + queue = odp_queue_create("ordered queue", &queue_param); + CU_ASSERT_FATAL(queue != ODP_QUEUE_INVALID); + CU_ASSERT_FATAL(odp_queue_type(queue) == ODP_QUEUE_TYPE_SCHED); + CU_ASSERT_FATAL(odp_queue_sched_type(queue) == ODP_SCHED_SYNC_ORDERED); + + odp_atomic_init_u32(&globals->order_wait.helper_ready, 0); + odp_atomic_init_u32(&globals->order_wait.helper_active, 0); + + ret = odp_cunit_thread_create(order_wait_helper, &thr_arg); + CU_ASSERT_FATAL(ret == thr_arg.numthrds); + + /* Send an event to the helper thread */ + enqueue_event(queue); + + /* Wait that the helper thread gets the event */ + start = odp_time_local(); + while (!odp_atomic_load_acq_u32(&globals->order_wait.helper_ready)) { + odp_time_t now = odp_time_local(); + + if (odp_time_diff_ns(now, start) > ODP_TIME_SEC_IN_NS) { + CU_FAIL("Timeout waiting for helper\n"); + break; + } + } + + /* Try to send an event to ourselves */ + enqueue_event(queue); + /* + * If ordered queues are implemented as atomic queues, the schedule + * call here will not return anything until the helper thread has + * released the scheduling context of the first event. So we have + * to wait long enough before giving up. + */ + ev = odp_schedule(NULL, odp_schedule_wait_time(2 * ODP_TIME_SEC_IN_NS)); + if (ev == ODP_EVENT_INVALID) { + /* Helper thread got the event. Give up. */ + printf("SKIPPED..."); + goto out; + } + odp_event_free(ev); + + /* + * We are now in an ordered scheduling context and behind the helper + * thread in source queue order if the helper thread has not released + * the scheuduling context. + */ + + if (!odp_atomic_load_acq_u32(&globals->order_wait.helper_active)) { + /* + * Helper thread has released the context already. + * We cannot test order wait fully. + */ + printf("reduced test..."); + } + + /* + * The function we are testing: Wait until there are no scheduling + * contexts that precede ours. + */ + odp_schedule_order_wait(); + + /* + * If order wait is supported, we are now first in the source queue + * order, so the helper thread must have released its context. + */ + if (sched_capa.order_wait) + CU_ASSERT(!odp_atomic_load_acq_u32(&globals->order_wait.helper_active)); + + /* Release the context */ + ev = odp_schedule(NULL, ODP_SCHED_NO_WAIT); + CU_ASSERT(ev == ODP_EVENT_INVALID); + +out: + CU_ASSERT(odp_cunit_thread_exit(&thr_arg) == 0); + CU_ASSERT(odp_queue_destroy(queue) == 0); +} + +static int check_2_workers(void) +{ + if (globals->num_workers < 2) { + printf("\nTest: scheduler_test_order_wait_2_threads: SKIPPED\n"); + return ODP_TEST_INACTIVE; + } + + return ODP_TEST_ACTIVE; +} + static int sched_and_plain_thread(void *arg) { odp_event_t ev1, ev2; @@ -3278,6 +3415,7 @@ odp_testinfo_t scheduler_basic_suite[] = { ODP_TEST_INFO(scheduler_test_pause_enqueue), ODP_TEST_INFO(scheduler_test_ordered_lock), ODP_TEST_INFO(scheduler_test_order_wait_1_thread), + ODP_TEST_INFO_CONDITIONAL(scheduler_test_order_wait_2_threads, check_2_workers), ODP_TEST_INFO_CONDITIONAL(scheduler_test_flow_aware, check_flow_aware_support), ODP_TEST_INFO(scheduler_test_parallel), diff --git a/test/validation/api/stash/stash.c b/test/validation/api/stash/stash.c index 49da49dfd..4057156c7 100644 --- a/test/validation/api/stash/stash.c +++ b/test/validation/api/stash/stash.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2020-2021, Nokia +/* Copyright (c) 2020-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -129,11 +129,11 @@ static void stash_capability(void) } } -static void stash_param_defaults(void) +static void param_defaults(uint8_t fill) { odp_stash_param_t param; - memset(¶m, 0xff, sizeof(odp_stash_param_t)); + memset(¶m, fill, sizeof(param)); odp_stash_param_init(¶m); CU_ASSERT(param.type == ODP_STASH_TYPE_DEFAULT); CU_ASSERT(param.put_mode == ODP_STASH_OP_MT); @@ -141,6 +141,12 @@ static void stash_param_defaults(void) CU_ASSERT(param.cache_size == 0); } +static void stash_param_defaults(void) +{ + param_defaults(0); + param_defaults(0xff); +} + static void stash_create_u64(void) { odp_stash_t stash, lookup; diff --git a/test/validation/api/timer/timer.c b/test/validation/api/timer/timer.c index ccfbf5558..00ef13c89 100644 --- a/test/validation/api/timer/timer.c +++ b/test/validation/api/timer/timer.c @@ -331,11 +331,11 @@ static void timer_test_capa(void) } } -static void timer_test_param_init(void) +static void test_param_init(uint8_t fill) { odp_timer_pool_param_t tp_param; - memset(&tp_param, 0x55, sizeof(odp_timer_pool_param_t)); + memset(&tp_param, fill, sizeof(tp_param)); odp_timer_pool_param_init(&tp_param); CU_ASSERT(tp_param.res_ns == 0); @@ -350,6 +350,12 @@ static void timer_test_param_init(void) CU_ASSERT(tp_param.periodic.base_freq_hz.denom == 0); } +static void timer_test_param_init(void) +{ + test_param_init(0); + test_param_init(0xff); +} + static void timer_test_timeout_pool_alloc(void) { odp_pool_t pool; diff --git a/test/validation/api/traffic_mngr/traffic_mngr.c b/test/validation/api/traffic_mngr/traffic_mngr.c index 65f91192c..0679e215c 100644 --- a/test/validation/api/traffic_mngr/traffic_mngr.c +++ b/test/validation/api/traffic_mngr/traffic_mngr.c @@ -471,7 +471,8 @@ static int test_overall_capabilities(void) return -1; } - if (per_level->tm_node_shaper_supported) { + if (per_level->tm_node_shaper_supported || + per_level->tm_node_rate_limiter_supported) { CU_ASSERT(per_level->max_burst > 0); CU_ASSERT(per_level->min_rate > 0); CU_ASSERT(per_level->max_rate > 0); @@ -1852,7 +1853,8 @@ set_reqs_based_on_capas(odp_tm_requirements_t *req) req->marking_colors_needed[color] = true; } - if (tm_capabilities.tm_queue_shaper_supported) + if (tm_capabilities.tm_queue_shaper_supported || + tm_capabilities.tm_queue_rate_limiter_supported) req->tm_queue_shaper_needed = true; /* We can use any packet priority mode since it does not affect @@ -4451,7 +4453,7 @@ static void test_packet_aging(uint64_t tmo_ns, uint32_t pkt_len, odp_bool_t is_d CU_ASSERT(odp_tm_is_idle(odp_tm_systems[0])); } -static void traffic_mngr_test_default_values(void) +static void test_defaults(uint8_t fill) { odp_tm_requirements_t req; odp_tm_shaper_params_t shaper; @@ -4462,7 +4464,7 @@ static void traffic_mngr_test_default_values(void) odp_tm_queue_params_t queue; int n; - memset(&req, 0xff, sizeof(req)); + memset(&req, fill, sizeof(req)); odp_tm_requirements_init(&req); CU_ASSERT_EQUAL(req.num_levels, 0); CU_ASSERT(!req.tm_queue_shaper_needed); @@ -4486,8 +4488,9 @@ static void traffic_mngr_test_default_values(void) CU_ASSERT(!l_req->tm_node_threshold_needed); } - memset(&shaper, 0xff, sizeof(shaper)); + memset(&shaper, fill, sizeof(shaper)); odp_tm_shaper_params_init(&shaper); + CU_ASSERT(shaper.packet_mode == ODP_TM_SHAPER_RATE_SHAPE); CU_ASSERT_EQUAL(shaper.shaper_len_adjust, 0); CU_ASSERT(!shaper.dual_rate); CU_ASSERT(!shaper.packet_mode); @@ -4497,24 +4500,24 @@ static void traffic_mngr_test_default_values(void) for (n = 0; n < ODP_TM_MAX_PRIORITIES; n++) CU_ASSERT_EQUAL(sched.sched_modes[n], ODP_TM_BYTE_BASED_WEIGHTS); - memset(&threshold, 0xff, sizeof(threshold)); + memset(&threshold, fill, sizeof(threshold)); odp_tm_threshold_params_init(&threshold); CU_ASSERT(!threshold.enable_max_pkts); CU_ASSERT(!threshold.enable_max_bytes); - memset(&wred, 0xff, sizeof(wred)); + memset(&wred, fill, sizeof(wred)); odp_tm_wred_params_init(&wred); CU_ASSERT(!wred.enable_wred); CU_ASSERT(!wred.use_byte_fullness); - memset(&node, 0xff, sizeof(node)); + memset(&node, fill, sizeof(node)); odp_tm_node_params_init(&node); CU_ASSERT_EQUAL(node.shaper_profile, ODP_TM_INVALID); CU_ASSERT_EQUAL(node.threshold_profile, ODP_TM_INVALID); for (n = 0; n < ODP_NUM_PACKET_COLORS; n++) CU_ASSERT_EQUAL(node.wred_profile[n], ODP_TM_INVALID); - memset(&queue, 0xff, sizeof(queue)); + memset(&queue, fill, sizeof(queue)); odp_tm_queue_params_init(&queue); CU_ASSERT_EQUAL(queue.shaper_profile, ODP_TM_INVALID); CU_ASSERT_EQUAL(queue.threshold_profile, ODP_TM_INVALID); @@ -4522,10 +4525,12 @@ static void traffic_mngr_test_default_values(void) CU_ASSERT_EQUAL(queue.wred_profile[n], ODP_TM_INVALID); CU_ASSERT_EQUAL(queue.priority, 0); CU_ASSERT(queue.ordered_enqueue); - /* re-check ordered_enqueue to notice if it is not set at all */ - memset(&queue, 0, sizeof(queue)); - odp_tm_queue_params_init(&queue); - CU_ASSERT(queue.ordered_enqueue); +} + +static void traffic_mngr_test_default_values(void) +{ + test_defaults(0); + test_defaults(0xff); } static void traffic_mngr_test_capabilities(void) |