diff options
Diffstat (limited to 'platform')
59 files changed, 1067 insertions, 514 deletions
diff --git a/platform/Makefile.inc b/platform/Makefile.inc index c820727c2..ed161d83d 100644 --- a/platform/Makefile.inc +++ b/platform/Makefile.inc @@ -14,7 +14,7 @@ endif VPATH = $(srcdir) $(builddir) lib_LTLIBRARIES = -AM_LDFLAGS = -version-number '$(ODP_LIBSO_VERSION)' +AM_LDFLAGS += -version-number '$(ODP_LIBSO_VERSION)' if ODP_ABI_COMPAT AM_LDFLAGS += -export-symbols-regex '^(odp_|_deprecated_odp_)' @@ -22,7 +22,7 @@ else AM_LDFLAGS += -export-symbols-regex '^(odp_|_odp_|_deprecated_odp_)' endif -AM_CFLAGS = "-DODP_VERSION_BUILD=$(VERSION)" +AM_CFLAGS += "-DODP_VERSION_BUILD=$(VERSION)" AM_CFLAGS += $(VISIBILITY_CFLAGS) -AM_CFLAGS += @PTHREAD_CFLAGS@ +AM_CFLAGS += $(PTHREAD_CFLAGS) diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 4fa7b7255..a676380dd 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -129,6 +129,7 @@ noinst_HEADERS = \ ${top_srcdir}/platform/linux-generic/include/odp_name_table_internal.h \ include/odp_packet_io_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_parse_internal.h \ + ${top_srcdir}/platform/linux-generic/include/odp_print_internal.h \ include/odp_errno_define.h \ include/odp_event_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_packet_dpdk.h \ @@ -204,6 +205,7 @@ __LIB__libodp_dpdk_la_SOURCES = \ ../linux-generic/pktio/loop.c \ ../linux-generic/pktio/null.c \ ../linux-generic/odp_pkt_queue.c \ + ../linux-generic/odp_print.c \ odp_pool.c \ odp_queue_basic.c \ odp_queue_eventdev.c \ diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h b/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h index e0169579a..26ada3655 100644 --- a/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h @@ -26,6 +26,9 @@ extern "C" { #define _odp_pkt_get(pkt, cast, field) \ (*(cast *)(uintptr_t)((uint8_t *)pkt + _odp_packet_inline.field)) +#define _odp_pkt_get_ptr(pkt, cast, field) \ + ((cast *)(uintptr_t)((uint8_t *)pkt + _odp_packet_inline.field)) + /* Packet header field offsets for inline functions */ typedef struct _odp_packet_inline_offset_t { uint16_t mb; @@ -113,12 +116,13 @@ typedef union { uint32_t all_flags; struct { - uint32_t reserved1: 7; + uint32_t reserved1: 6; /* * Init flags */ uint32_t user_ptr_set: 1; /* User has set a non-NULL value */ + uint32_t user_flag: 1; /* * Packet output flags @@ -148,8 +152,8 @@ typedef union { /* Flag groups */ struct { - uint32_t reserved2: 7; - uint32_t other: 18; /* All other flags */ + uint32_t reserved2: 6; + uint32_t other: 19; /* All other flags */ uint32_t error: 7; /* All error flags */ } all; diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h index 5b853abb2..9856d1dd7 100644 --- a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h @@ -66,11 +66,14 @@ extern "C" { #define odp_packet_tailroom __odp_packet_tailroom #define odp_packet_pool __odp_packet_pool #define odp_packet_input __odp_packet_input + #define odp_packet_input_set __odp_packet_input_set #define odp_packet_input_index __odp_packet_input_index #define odp_packet_num_segs __odp_packet_num_segs #define odp_packet_user_ptr __odp_packet_user_ptr #define odp_packet_user_area __odp_packet_user_area #define odp_packet_user_area_size __odp_packet_user_area_size + #define odp_packet_user_flag __odp_packet_user_flag + #define odp_packet_user_flag_set __odp_packet_user_flag_set #define odp_packet_l2_offset __odp_packet_l2_offset #define odp_packet_l3_offset __odp_packet_l3_offset #define odp_packet_l4_offset __odp_packet_l4_offset @@ -105,6 +108,12 @@ extern "C" { #define odp_packet_color __odp_packet_color #define odp_packet_drop_eligible __odp_packet_drop_eligible #define odp_packet_shaper_len_adjust __odp_packet_shaper_len_adjust + #define odp_packet_buf_data_len __odp_packet_buf_data_len + #define odp_packet_buf_size __odp_packet_buf_size + #define odp_packet_buf_head __odp_packet_buf_head + #define odp_packet_buf_data_offset __odp_packet_buf_data_offset + #define odp_packet_buf_data_set __odp_packet_buf_data_set + #define odp_packet_buf_from_head __odp_packet_buf_from_head #else #undef _ODP_INLINE #define _ODP_INLINE @@ -198,6 +207,13 @@ _ODP_INLINE odp_pktio_t odp_packet_input(odp_packet_t pkt) return _odp_pkt_get(pkt, odp_pktio_t, input); } +_ODP_INLINE void odp_packet_input_set(odp_packet_t pkt, odp_pktio_t pktio) +{ + odp_pktio_t *pktio_ptr = _odp_pkt_get_ptr(pkt, odp_pktio_t, input); + + *pktio_ptr = pktio; +} + _ODP_INLINE int odp_packet_input_index(odp_packet_t pkt) { odp_pktio_t pktio = odp_packet_input(pkt); @@ -234,6 +250,22 @@ _ODP_INLINE uint32_t odp_packet_user_area_size(odp_packet_t pkt) return _odp_pool_get(pool, uint32_t, uarea_size); } +_ODP_INLINE int odp_packet_user_flag(odp_packet_t pkt) +{ + _odp_packet_flags_t flags; + + flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + + return flags.user_flag; +} + +_ODP_INLINE void odp_packet_user_flag_set(odp_packet_t pkt, int val) +{ + _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); + + flags->user_flag = !!val; +} + _ODP_INLINE uint32_t odp_packet_l2_offset(odp_packet_t pkt) { return _odp_pkt_get(pkt, uint16_t, l2_offset); @@ -516,6 +548,58 @@ _ODP_INLINE int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt) return (int8_t)flags.shaper_len_adj; } +_ODP_INLINE uint32_t odp_packet_buf_data_len(odp_packet_buf_t pkt_buf) +{ + return odp_packet_seg_data_len(ODP_PACKET_INVALID, (odp_packet_seg_t)pkt_buf); +} + +_ODP_INLINE uint32_t odp_packet_buf_size(odp_packet_buf_t pkt_buf) +{ + odp_pool_t pool = _odp_pkt_get((odp_packet_buf_t)pkt_buf, odp_pool_t, pool); + + return _odp_pool_get(pool, uint32_t, seg_len); +} + +_ODP_INLINE void *odp_packet_buf_head(odp_packet_buf_t pkt_buf) +{ + odp_pool_t pool = _odp_pkt_get(pkt_buf, odp_pool_t, pool); + const uint32_t head_offset = _odp_pool_get(pool, uint32_t, ext_head_offset); + + /* Check that pool is external */ + if (odp_unlikely(!head_offset)) + return NULL; + + return (uint8_t *)(uintptr_t)pkt_buf + head_offset; +} + +_ODP_INLINE uint32_t odp_packet_buf_data_offset(odp_packet_buf_t pkt_buf) +{ + void *data = odp_packet_seg_data(ODP_PACKET_INVALID, (odp_packet_seg_t)pkt_buf); + void *head = odp_packet_buf_head(pkt_buf); + + return (uint32_t)((uintptr_t)data - (uintptr_t)head); +} + +_ODP_INLINE void odp_packet_buf_data_set(odp_packet_buf_t pkt_buf, uint32_t data_offset, + uint32_t data_len) +{ + struct rte_mbuf *mb = (struct rte_mbuf *)pkt_buf; + + mb->data_off = (uint16_t)data_offset; + mb->data_len = (uint16_t)data_len; +} + +_ODP_INLINE odp_packet_buf_t odp_packet_buf_from_head(odp_pool_t pool, void *head) +{ + const uint32_t head_offset = _odp_pool_get(pool, uint32_t, ext_head_offset); + + /* Check that pool is external */ + if (odp_unlikely(!head_offset)) + return ODP_PACKET_BUF_INVALID; + + return (odp_packet_buf_t)((uintptr_t)head - head_offset); +} + #ifdef __cplusplus } #endif diff --git a/platform/linux-dpdk/include/odp/api/plat/timer_inline_types.h b/platform/linux-dpdk/include/odp/api/plat/timer_inline_types.h index e3397c4df..ec6804c72 100644 --- a/platform/linux-dpdk/include/odp/api/plat/timer_inline_types.h +++ b/platform/linux-dpdk/include/odp/api/plat/timer_inline_types.h @@ -25,6 +25,7 @@ typedef struct _odp_timeout_inline_offset_t { uint16_t expiration; uint16_t timer; uint16_t user_ptr; + uint16_t uarea_addr; } _odp_timeout_inline_offset_t; diff --git a/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h b/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h index 270a6769b..48154a26f 100644 --- a/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h @@ -24,6 +24,7 @@ extern const _odp_timeout_inline_offset_t _odp_timeout_inline_offset; #define odp_timeout_timer __odp_timeout_timer #define odp_timeout_tick __odp_timeout_tick #define odp_timeout_user_ptr __odp_timeout_user_ptr + #define odp_timeout_user_area __odp_timeout_user_area #define odp_timeout_from_event __odp_timeout_from_event #define odp_timeout_to_event __odp_timeout_to_event #else @@ -45,6 +46,11 @@ _ODP_INLINE void *odp_timeout_user_ptr(odp_timeout_t tmo) return _odp_timeout_hdr_field(tmo, void *, user_ptr); } +_ODP_INLINE void *odp_timeout_user_area(odp_timeout_t tmo) +{ + return _odp_timeout_hdr_field(tmo, void *, uarea_addr); +} + _ODP_INLINE odp_timeout_t odp_timeout_from_event(odp_event_t ev) { return (odp_timeout_t)ev; diff --git a/platform/linux-dpdk/include/odp_buffer_internal.h b/platform/linux-dpdk/include/odp_buffer_internal.h index e7b1d215d..dfffdc2be 100644 --- a/platform/linux-dpdk/include/odp_buffer_internal.h +++ b/platform/linux-dpdk/include/odp_buffer_internal.h @@ -52,6 +52,9 @@ typedef struct ODP_ALIGNED_CACHE odp_buffer_hdr_t { /* Common event header */ _odp_event_hdr_t event_hdr; + /* User area pointer */ + void *uarea_addr; + } odp_buffer_hdr_t; /* diff --git a/platform/linux-dpdk/include/odp_event_vector_internal.h b/platform/linux-dpdk/include/odp_event_vector_internal.h index c866d9036..5fa8c31c6 100644 --- a/platform/linux-dpdk/include/odp_event_vector_internal.h +++ b/platform/linux-dpdk/include/odp_event_vector_internal.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2020-2021, Nokia +/* Copyright (c) 2020-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -17,6 +17,8 @@ #include <odp/api/debug.h> #include <odp/api/packet.h> +#include <odp/api/plat/event_vector_inline_types.h> + #include <odp_event_internal.h> #include <stdint.h> @@ -28,9 +30,15 @@ typedef struct ODP_ALIGNED_CACHE odp_event_vector_hdr_t { /* Common event header */ _odp_event_hdr_t event_hdr; + /* User area pointer */ + void *uarea_addr; + /* Event vector size */ uint32_t size; + /* Flags */ + _odp_event_vector_flags_t flags; + /* Vector of packet handles */ odp_packet_t packet[]; diff --git a/platform/linux-dpdk/include/odp_pool_internal.h b/platform/linux-dpdk/include/odp_pool_internal.h index 1d1ab02e2..a5ffcf517 100644 --- a/platform/linux-dpdk/include/odp_pool_internal.h +++ b/platform/linux-dpdk/include/odp_pool_internal.h @@ -73,7 +73,7 @@ typedef struct ODP_ALIGNED_CACHE { uint8_t memset_mark; struct rte_mempool *rte_mempool; uint32_t seg_len; - uint32_t hdr_size; + uint32_t ext_head_offset; uint32_t num; uint32_t num_populated; odp_pool_type_t type_2; diff --git a/platform/linux-dpdk/include/odp_timer_internal.h b/platform/linux-dpdk/include/odp_timer_internal.h index 2e8487689..f504a20aa 100644 --- a/platform/linux-dpdk/include/odp_timer_internal.h +++ b/platform/linux-dpdk/include/odp_timer_internal.h @@ -35,6 +35,9 @@ typedef struct ODP_ALIGNED_CACHE odp_timeout_hdr_t { /* User ptr inherited from parent timer */ const void *user_ptr; + /* User area pointer */ + void *uarea_addr; + /* Parent timer */ odp_timer_t timer; diff --git a/platform/linux-dpdk/m4/odp_libconfig.m4 b/platform/linux-dpdk/m4/odp_libconfig.m4 index b4bfdd719..84d331b8a 100644 --- a/platform/linux-dpdk/m4/odp_libconfig.m4 +++ b/platform/linux-dpdk/m4/odp_libconfig.m4 @@ -3,7 +3,7 @@ ########################################################################## m4_define([_odp_config_version_generation], [0]) m4_define([_odp_config_version_major], [1]) -m4_define([_odp_config_version_minor], [17]) +m4_define([_odp_config_version_minor], [18]) m4_define([_odp_config_version], [_odp_config_version_generation._odp_config_version_major._odp_config_version_minor]) diff --git a/platform/linux-dpdk/odp_buffer.c b/platform/linux-dpdk/odp_buffer.c index 8a2cbb949..758782563 100644 --- a/platform/linux-dpdk/odp_buffer.c +++ b/platform/linux-dpdk/odp_buffer.c @@ -40,6 +40,13 @@ int odp_buffer_is_valid(odp_buffer_t buf) return 1; } +void *odp_buffer_user_area(odp_buffer_t buf) +{ + odp_buffer_hdr_t *hdr = _odp_buf_hdr(buf); + + return hdr->uarea_addr; +} + void odp_buffer_print(odp_buffer_t buf) { odp_buffer_hdr_t *hdr; diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index 6e740da77..99dbe07f6 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -626,13 +626,6 @@ void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ptr) pkt_hdr->p.flags.user_ptr_set = 1; } -void odp_packet_input_set(odp_packet_t pkt, odp_pktio_t pktio) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - pkt_hdr->input = pktio; -} - int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); @@ -1979,73 +1972,6 @@ int odp_packet_reass_partial_state(odp_packet_t pkt, odp_packet_t frags[], return -ENOTSUP; } -static inline odp_packet_hdr_t *packet_buf_to_hdr(odp_packet_buf_t pkt_buf) -{ - return (odp_packet_hdr_t *)(uintptr_t)pkt_buf; -} - -void *odp_packet_buf_head(odp_packet_buf_t pkt_buf) -{ - odp_packet_hdr_t *pkt_hdr = packet_buf_to_hdr(pkt_buf); - pool_t *pool = _odp_pool_entry(pkt_hdr->event_hdr.pool); - - if (odp_unlikely(pool->pool_ext == 0)) { - ODP_ERR("Not an external memory pool\n"); - return NULL; - } - - return (uint8_t *)pkt_hdr + pool->hdr_size; -} - -uint32_t odp_packet_buf_size(odp_packet_buf_t pkt_buf) -{ - odp_packet_hdr_t *pkt_hdr = packet_buf_to_hdr(pkt_buf); - pool_t *pool = _odp_pool_entry(pkt_hdr->event_hdr.pool); - - return pool->seg_len; -} - -uint32_t odp_packet_buf_data_offset(odp_packet_buf_t pkt_buf) -{ - void *data = odp_packet_seg_data(ODP_PACKET_INVALID, - (odp_packet_seg_t)pkt_buf); - void *head = odp_packet_buf_head(pkt_buf); - - return (uintptr_t)data - (uintptr_t)head; -} - -uint32_t odp_packet_buf_data_len(odp_packet_buf_t pkt_buf) -{ - return odp_packet_seg_data_len(ODP_PACKET_INVALID, - (odp_packet_seg_t)pkt_buf); -} - -void odp_packet_buf_data_set(odp_packet_buf_t pkt_buf, uint32_t data_offset, - uint32_t data_len) -{ - odp_packet_hdr_t *pkt_hdr = packet_buf_to_hdr(pkt_buf); - - pkt_hdr->event_hdr.mb.data_off = data_offset; - pkt_hdr->event_hdr.mb.data_len = data_len; -} - -odp_packet_buf_t odp_packet_buf_from_head(odp_pool_t pool_hdl, void *head) -{ - pool_t *pool = _odp_pool_entry(pool_hdl); - - if (odp_unlikely(pool->type != ODP_POOL_PACKET)) { - ODP_ERR("Not a packet pool\n"); - return ODP_PACKET_BUF_INVALID; - } - - if (odp_unlikely(pool->pool_ext == 0)) { - ODP_ERR("Not an external memory pool\n"); - return ODP_PACKET_BUF_INVALID; - } - - return (odp_packet_buf_t)((uintptr_t)head - pool->hdr_size); -} - uint32_t odp_packet_disassemble(odp_packet_t pkt, odp_packet_buf_t pkt_buf[], uint32_t num) { diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index 49d6185a2..e400540d9 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -69,7 +69,9 @@ pool_global_t *_odp_pool_glb; /* Fill in pool header field offsets for inline functions */ const _odp_pool_inline_offset_t _odp_pool_inline ODP_ALIGNED_CACHE = { .index = offsetof(pool_t, pool_idx), - .uarea_size = offsetof(pool_t, params.pkt.uarea_size) + .seg_len = offsetof(pool_t, seg_len), + .uarea_size = offsetof(pool_t, params.pkt.uarea_size), + .ext_head_offset = offsetof(pool_t, ext_head_offset) }; #include <odp/visibility_end.h> @@ -250,6 +252,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->buf.max_align = ODP_CONFIG_BUFFER_ALIGN_MAX; capa->buf.max_size = MAX_SIZE; capa->buf.max_num = CONFIG_POOL_MAX_NUM; + capa->buf.max_uarea_size = MAX_UAREA_SIZE; capa->buf.min_cache_size = 0; capa->buf.max_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; capa->buf.stats.all = supported_stats.all; @@ -273,6 +276,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) /* Timeout pools */ capa->tmo.max_pools = max_pools; capa->tmo.max_num = CONFIG_POOL_MAX_NUM; + capa->tmo.max_uarea_size = MAX_UAREA_SIZE; capa->tmo.min_cache_size = 0; capa->tmo.max_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; capa->tmo.stats.all = supported_stats.all; @@ -280,6 +284,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) /* Vector pools */ capa->vector.max_pools = max_pools; capa->vector.max_num = CONFIG_POOL_MAX_NUM; + capa->vector.max_uarea_size = MAX_UAREA_SIZE; capa->vector.max_size = CONFIG_PACKET_VECTOR_MAX_SIZE; capa->vector.min_cache_size = 0; capa->vector.max_cache_size = RTE_MEMPOOL_CACHE_MAX_SIZE; @@ -309,10 +314,12 @@ odp_dpdk_mbuf_ctor(struct rte_mempool *mp, struct mbuf_ctor_arg *mb_ctor_arg; struct rte_mbuf *mb = raw_mbuf; _odp_event_hdr_t *event_hdr; + void *uarea; - /* The rte_mbuf is at the begninning in all cases */ + /* The rte_mbuf is at the beginning in all cases */ mb_ctor_arg = (struct mbuf_ctor_arg *)opaque_arg; mb = (struct rte_mbuf *)raw_mbuf; + uarea = mb_ctor_arg->pool->uarea_base_addr + (i * mb_ctor_arg->pool->uarea_size); RTE_ASSERT(mp->elt_size >= sizeof(struct rte_mbuf)); @@ -359,20 +366,32 @@ odp_dpdk_mbuf_ctor(struct rte_mempool *mp, event_hdr->type = mb_ctor_arg->type; event_hdr->event_type = mb_ctor_arg->event_type; + /* Initialize buffer metadata */ + if (mb_ctor_arg->type == ODP_POOL_BUFFER) { + odp_buffer_hdr_t *buf_hdr = (void *)raw_mbuf; + + buf_hdr->uarea_addr = uarea; + } + /* Initialize packet metadata */ if (mb_ctor_arg->type == ODP_POOL_PACKET) { - odp_packet_hdr_t *pkt_hdr = (odp_packet_hdr_t *)raw_mbuf; + odp_packet_hdr_t *pkt_hdr = (void *)raw_mbuf; - pkt_hdr->uarea_addr = mb_ctor_arg->pool->uarea_base_addr + - i * mb_ctor_arg->pool->uarea_size; + pkt_hdr->uarea_addr = uarea; } /* Initialize event vector metadata */ if (mb_ctor_arg->type == ODP_POOL_VECTOR) { - odp_event_vector_hdr_t *vect_hdr; + odp_event_vector_hdr_t *vect_hdr = (void *)raw_mbuf; + + vect_hdr->uarea_addr = uarea; + } + + /* Initialize timeout metadata */ + if (mb_ctor_arg->type == ODP_POOL_TIMEOUT) { + odp_timeout_hdr_t *tmo_hdr = (void *)raw_mbuf; - vect_hdr = (odp_event_vector_hdr_t *)raw_mbuf; - vect_hdr->size = 0; + tmo_hdr->uarea_addr = uarea; } } @@ -424,6 +443,11 @@ static int check_params(const odp_pool_param_t *params) return -1; } + if (params->buf.uarea_size > capa.buf.max_uarea_size) { + ODP_ERR("buf.uarea_size too large %u\n", params->buf.uarea_size); + return -1; + } + if (params->stats.all & ~capa.buf.stats.all) { ODP_ERR("Unsupported pool statistics counter\n"); return -1; @@ -506,6 +530,11 @@ static int check_params(const odp_pool_param_t *params) return -1; } + if (params->tmo.uarea_size > capa.tmo.max_uarea_size) { + ODP_ERR("tmo.uarea_size too large %u\n", params->tmo.uarea_size); + return -1; + } + if (params->stats.all & ~capa.tmo.stats.all) { ODP_ERR("Unsupported pool statistics counter\n"); return -1; @@ -539,6 +568,11 @@ static int check_params(const odp_pool_param_t *params) return -1; } + if (params->vector.uarea_size > capa.vector.max_uarea_size) { + ODP_ERR("vector.uarea_size too large %u\n", params->vector.uarea_size); + return -1; + } + if (params->stats.all & ~capa.vector.stats.all) { ODP_ERR("Unsupported pool statistics counter\n"); return -1; @@ -684,6 +718,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, buf_align = params->buf.align; blk_size = params->buf.size; cache_size = params->buf.cache_size; + uarea_size = params->buf.uarea_size; /* Set correct alignment based on input request */ if (buf_align == 0) @@ -754,6 +789,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, mbp_ctor_arg.mbuf_data_room_size = 0; num = params->tmo.num; cache_size = params->tmo.cache_size; + uarea_size = params->tmo.uarea_size; event_type = ODP_EVENT_TIMEOUT; ODP_DBG("type: tmo, name: %s, num: %u\n", pool_name, num); @@ -764,6 +800,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, mbp_ctor_arg.mbuf_data_room_size = 0; num = params->vector.num; cache_size = params->vector.cache_size; + uarea_size = params->vector.uarea_size; event_type = ODP_EVENT_PACKET_VECTOR; ODP_DBG("type: vector, name: %s, num: %u\n", pool_name, num); @@ -1300,7 +1337,7 @@ odp_pool_t odp_pool_ext_create(const char *name, } pool->ext_param = *params; - pool->hdr_size = hdr_size; + pool->ext_head_offset = hdr_size; pool->num = num; pool->num_populated = 0; pool->params.pkt.uarea_size = params->pkt.uarea_size; @@ -1430,8 +1467,7 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size, mb_ctor_arg.type = params->type; mb_ctor_arg.event_type = pool->type; mb_ctor_arg.pool = pool; - odp_dpdk_mbuf_ctor(mp, (void *)&mb_ctor_arg, (void *)mb, - mp->populated_size); + odp_dpdk_mbuf_ctor(mp, (void *)&mb_ctor_arg, (void *)mb, num_populated); pool->num_populated++; } diff --git a/platform/linux-dpdk/odp_timer.c b/platform/linux-dpdk/odp_timer.c index f09a2f72c..7e59775d3 100644 --- a/platform/linux-dpdk/odp_timer.c +++ b/platform/linux-dpdk/odp_timer.c @@ -183,7 +183,8 @@ const _odp_timeout_inline_offset_t _odp_timeout_inline_offset ODP_ALIGNED_CACHE = { .expiration = offsetof(odp_timeout_hdr_t, expiration), .timer = offsetof(odp_timeout_hdr_t, timer), - .user_ptr = offsetof(odp_timeout_hdr_t, user_ptr) + .user_ptr = offsetof(odp_timeout_hdr_t, user_ptr), + .uarea_addr = offsetof(odp_timeout_hdr_t, uarea_addr) }; #include <odp/visibility_end.h> diff --git a/platform/linux-dpdk/test/alternate-timer.conf b/platform/linux-dpdk/test/alternate-timer.conf index bb884013c..0b326f259 100644 --- a/platform/linux-dpdk/test/alternate-timer.conf +++ b/platform/linux-dpdk/test/alternate-timer.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.17" +config_file_version = "0.1.18" timer: { # Enable alternate DPDK timer implementation diff --git a/platform/linux-dpdk/test/crypto.conf b/platform/linux-dpdk/test/crypto.conf index 413e7a043..f3d642963 100644 --- a/platform/linux-dpdk/test/crypto.conf +++ b/platform/linux-dpdk/test/crypto.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.17" +config_file_version = "0.1.18" system: { # One crypto queue pair is required per thread for lockless operation diff --git a/platform/linux-dpdk/test/sched-basic.conf b/platform/linux-dpdk/test/sched-basic.conf index 7093cd810..181136c33 100644 --- a/platform/linux-dpdk/test/sched-basic.conf +++ b/platform/linux-dpdk/test/sched-basic.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.17" +config_file_version = "0.1.18" # Test scheduler with an odd spread value and without dynamic load balance sched_basic: { diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 56a37544c..708526558 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -135,6 +135,7 @@ noinst_HEADERS = \ include/odp_packet_internal.h \ include/odp_packet_io_internal.h \ include/odp_parse_internal.h \ + include/odp_print_internal.h \ include/odp_socket_common.h \ include/odp_packet_io_stats_common.h \ include/odp_packet_io_stats.h \ @@ -213,6 +214,7 @@ __LIB__libodp_linux_la_SOURCES = \ odp_packet_io.c \ odp_parse.c \ odp_pkt_queue.c \ + odp_print.c \ odp_pool.c \ odp_pool_mem_src_ops.c \ odp_queue_basic.c \ diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h index f530afd4f..e8f33f09e 100644 --- a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic_inlines.h @@ -162,7 +162,7 @@ static inline void _odp_atomic_add_u32(odp_atomic_u32_t *atom, uint32_t val) static inline void _odp_atomic_sub_u32(odp_atomic_u32_t *atom, uint32_t val) { - int32_t neg_val = -val; + int32_t neg_val = (int32_t)-val; __asm__ volatile("stadd %w[neg_val], %[atom]" : [atom] "+Q" (atom->v) @@ -188,7 +188,7 @@ static inline void _odp_atomic_add_u64(odp_atomic_u64_t *atom, uint64_t val) static inline void _odp_atomic_sub_u64(odp_atomic_u64_t *atom, uint64_t val) { - int64_t neg_val = -val; + int64_t neg_val = (int64_t)-val; __asm__ volatile("stadd %[neg_val], %[atom]" : [atom] "+Q" (atom->v) @@ -215,7 +215,7 @@ static inline void _odp_atomic_add_rel_u32(odp_atomic_u32_t *atom, uint32_t val) static inline void _odp_atomic_sub_rel_u32(odp_atomic_u32_t *atom, uint32_t val) { - int32_t neg_val = -val; + int32_t neg_val = (int32_t)-val; __asm__ volatile("staddl %w[neg_val], %[atom]" : [atom] "+Q" (atom->v) @@ -233,7 +233,7 @@ static inline void _odp_atomic_add_rel_u64(odp_atomic_u64_t *atom, uint64_t val) static inline void _odp_atomic_sub_rel_u64(odp_atomic_u64_t *atom, uint64_t val) { - int64_t neg_val = -val; + int64_t neg_val = (int64_t)-val; __asm__ volatile("staddl %[neg_val], %[atom]" : [atom] "+Q" (atom->v) diff --git a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c index 4531ebc28..32341dd02 100644 --- a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c +++ b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c @@ -904,53 +904,71 @@ int odp_crypto_result(odp_crypto_packet_result_t *result, return 0; } -static -int crypto_int(odp_packet_t pkt_in, - odp_packet_t *pkt_out, - const odp_crypto_packet_op_param_t *param) +static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src) { - odp_crypto_generic_session_t *session; - odp_bool_t allocated = false; - odp_packet_t out_pkt = *pkt_out; - - session = (odp_crypto_generic_session_t *)(intptr_t)param->session; + int md_copy; + int rc; - /* Resolve output buffer */ - if (odp_unlikely(ODP_PACKET_INVALID == out_pkt) && - ODP_POOL_INVALID != session->p.output_pool) { - out_pkt = odp_packet_alloc(session->p.output_pool, - odp_packet_len(pkt_in)); - allocated = true; + md_copy = _odp_packet_copy_md_possible(odp_packet_pool(dst), + odp_packet_pool(src)); + if (odp_unlikely(md_copy < 0)) { + ODP_ERR("Unable to copy packet metadata\n"); + return -1; } - if (odp_unlikely(ODP_PACKET_INVALID == out_pkt)) { - ODP_DBG("Alloc failed.\n"); + rc = odp_packet_copy_from_pkt(dst, 0, src, 0, odp_packet_len(src)); + if (odp_unlikely(rc < 0)) { + ODP_ERR("Unable to copy packet data\n"); return -1; } - if (odp_unlikely(pkt_in != out_pkt)) { - int ret; - int md_copy; + _odp_packet_copy_md(packet_hdr(dst), packet_hdr(src), md_copy); + return 0; +} - md_copy = _odp_packet_copy_md_possible(session->p.output_pool, - odp_packet_pool(pkt_in)); - if (odp_unlikely(md_copy < 0)) { - ODP_ERR("Unable to copy packet metadata\n"); - goto err; - } +static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *session, + odp_packet_t pkt_in, + odp_packet_t pkt_out) +{ + int rc; - ret = odp_packet_copy_from_pkt(out_pkt, - 0, - pkt_in, - 0, - odp_packet_len(pkt_in)); - if (odp_unlikely(ret < 0)) - goto err; + if (odp_likely(pkt_in == pkt_out)) + return pkt_out; - _odp_packet_copy_md(packet_hdr(out_pkt), packet_hdr(pkt_in), md_copy); - odp_packet_free(pkt_in); - pkt_in = ODP_PACKET_INVALID; + if (pkt_out == ODP_PACKET_INVALID) { + odp_pool_t pool = session->p.output_pool; + + ODP_ASSERT(pool != ODP_POOL_INVALID); + if (pool == odp_packet_pool(pkt_in)) { + pkt_out = pkt_in; + } else { + pkt_out = odp_packet_copy(pkt_in, pool); + if (odp_likely(pkt_out != ODP_PACKET_INVALID)) + odp_packet_free(pkt_in); + } + return pkt_out; } + rc = copy_data_and_metadata(pkt_out, pkt_in); + if (odp_unlikely(rc < 0)) + return ODP_PACKET_INVALID; + + odp_packet_free(pkt_in); + return pkt_out; +} + +static +int crypto_int(odp_packet_t pkt_in, + odp_packet_t *pkt_out, + const odp_crypto_packet_op_param_t *param) +{ + odp_crypto_generic_session_t *session; + odp_packet_t out_pkt; + + session = (odp_crypto_generic_session_t *)(intptr_t)param->session; + + out_pkt = get_output_packet(session, pkt_in, *pkt_out); + if (odp_unlikely(out_pkt == ODP_PACKET_INVALID)) + return -1; /* Invoke the crypto function */ session->func(out_pkt, param, session); @@ -961,14 +979,6 @@ int crypto_int(odp_packet_t pkt_in, *pkt_out = out_pkt; return 0; - -err: - if (allocated) { - odp_packet_free(out_pkt); - *pkt_out = ODP_PACKET_INVALID; - } - - return -1; } int odp_crypto_op(const odp_packet_t pkt_in[], diff --git a/platform/linux-generic/include/odp/api/plat/byteorder_inlines.h b/platform/linux-generic/include/odp/api/plat/byteorder_inlines.h index d466a51ad..31d2f1db9 100644 --- a/platform/linux-generic/include/odp/api/plat/byteorder_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/byteorder_inlines.h @@ -56,7 +56,11 @@ extern "C" { * Don't use this function directly, instead see odp_byteorder.h */ #if GCC_VERSION < 40800 -#define __odp_builtin_bswap16(u16) ((((u16)&0x00ff) << 8) | \ +/* + * We have to explicitly cast back to uint16_t because clang promotes the + * left side of << operator to int. + */ +#define __odp_builtin_bswap16(u16) ((uint16_t)(((u16)&0x00ff) << 8) | \ (((u16)&0xff00) >> 8)) #else #define __odp_builtin_bswap16(u16) __builtin_bswap16(u16) diff --git a/platform/linux-generic/include/odp/api/plat/event_vector_inline_types.h b/platform/linux-generic/include/odp/api/plat/event_vector_inline_types.h index 547620df6..723e1a3d1 100644 --- a/platform/linux-generic/include/odp/api/plat/event_vector_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/event_vector_inline_types.h @@ -15,6 +15,15 @@ extern "C" { /** @cond _ODP_HIDE_FROM_DOXYGEN_ */ +typedef union { + uint32_t all_flags; + + struct { + uint32_t user_flag : 1; + }; + +} _odp_event_vector_flags_t; + /* Event vector field accessors */ #define _odp_event_vect_get(vect, cast, field) \ (*(cast *)(uintptr_t)((uint8_t *)vect + _odp_event_vector_inline.field)) @@ -26,6 +35,9 @@ typedef struct _odp_event_vector_inline_offset_t { uint16_t packet; uint16_t pool; uint16_t size; + uint16_t uarea_addr; + uint16_t flags; + } _odp_event_vector_inline_offset_t; /** @endcond */ diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index c5293fc86..b00173aca 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -27,6 +27,9 @@ extern "C" { #define _odp_pkt_get(pkt, cast, field) \ (*(cast *)(uintptr_t)((uint8_t *)pkt + _odp_packet_inline.field)) +#define _odp_pkt_get_ptr(pkt, cast, field) \ + ((cast *)(uintptr_t)((uint8_t *)pkt + _odp_packet_inline.field)) + /* Packet header field offsets for inline functions */ typedef struct _odp_packet_inline_offset_t { uint16_t seg_data; @@ -115,12 +118,13 @@ typedef union { uint32_t all_flags; struct { - uint32_t reserved1: 7; + uint32_t reserved1: 6; /* * Init flags */ uint32_t user_ptr_set: 1; /* User has set a non-NULL value */ + uint32_t user_flag: 1; /* * Packet output flags @@ -150,8 +154,8 @@ typedef union { /* Flag groups */ struct { - uint32_t reserved2: 7; - uint32_t other: 18; /* All other flags */ + uint32_t reserved2: 6; + uint32_t other: 19; /* All other flags */ uint32_t error: 7; /* All error flags */ } all; diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 950ede8d7..da4eabe2e 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -44,11 +44,14 @@ #define odp_packet_tailroom __odp_packet_tailroom #define odp_packet_pool __odp_packet_pool #define odp_packet_input __odp_packet_input + #define odp_packet_input_set __odp_packet_input_set #define odp_packet_input_index __odp_packet_input_index #define odp_packet_num_segs __odp_packet_num_segs #define odp_packet_user_ptr __odp_packet_user_ptr #define odp_packet_user_area __odp_packet_user_area #define odp_packet_user_area_size __odp_packet_user_area_size + #define odp_packet_user_flag __odp_packet_user_flag + #define odp_packet_user_flag_set __odp_packet_user_flag_set #define odp_packet_l2_offset __odp_packet_l2_offset #define odp_packet_l3_offset __odp_packet_l3_offset #define odp_packet_l4_offset __odp_packet_l4_offset @@ -76,6 +79,13 @@ #define odp_packet_color __odp_packet_color #define odp_packet_drop_eligible __odp_packet_drop_eligible #define odp_packet_shaper_len_adjust __odp_packet_shaper_len_adjust + #define odp_packet_buf_data_len __odp_packet_buf_data_len + #define odp_packet_buf_size __odp_packet_buf_size + #define odp_packet_buf_head __odp_packet_buf_head + #define odp_packet_buf_data_offset __odp_packet_buf_data_offset + #define odp_packet_buf_data_set __odp_packet_buf_data_set + #define odp_packet_buf_from_head __odp_packet_buf_from_head + #else #undef _ODP_INLINE #define _ODP_INLINE @@ -135,6 +145,13 @@ _ODP_INLINE odp_pktio_t odp_packet_input(odp_packet_t pkt) return _odp_pkt_get(pkt, odp_pktio_t, input); } +_ODP_INLINE void odp_packet_input_set(odp_packet_t pkt, odp_pktio_t pktio) +{ + odp_pktio_t *pktio_ptr = _odp_pkt_get_ptr(pkt, odp_pktio_t, input); + + *pktio_ptr = pktio; +} + _ODP_INLINE int odp_packet_input_index(odp_packet_t pkt) { odp_pktio_t pktio = odp_packet_input(pkt); @@ -171,6 +188,22 @@ _ODP_INLINE uint32_t odp_packet_user_area_size(odp_packet_t pkt) return _odp_pool_get(pool, uint32_t, uarea_size); } +_ODP_INLINE int odp_packet_user_flag(odp_packet_t pkt) +{ + _odp_packet_flags_t flags; + + flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags); + + return flags.user_flag; +} + +_ODP_INLINE void odp_packet_user_flag_set(odp_packet_t pkt, int val) +{ + _odp_packet_flags_t *flags = _odp_pkt_get_ptr(pkt, _odp_packet_flags_t, flags); + + flags->user_flag = !!val; +} + _ODP_INLINE uint32_t odp_packet_l2_offset(odp_packet_t pkt) { return _odp_pkt_get(pkt, uint16_t, l2_offset); @@ -400,6 +433,59 @@ _ODP_INLINE int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt) return (int8_t)flags.shaper_len_adj; } +_ODP_INLINE uint32_t odp_packet_buf_data_len(odp_packet_buf_t pkt_buf) +{ + return _odp_pkt_get(pkt_buf, uint32_t, seg_len); +} + +_ODP_INLINE uint32_t odp_packet_buf_size(odp_packet_buf_t pkt_buf) +{ + odp_pool_t pool = _odp_pkt_get(pkt_buf, odp_pool_t, pool); + + return _odp_pool_get(pool, uint32_t, ext_pkt_buf_size) - + _odp_pool_get(pool, uint32_t, ext_head_offset); +} + +_ODP_INLINE void *odp_packet_buf_head(odp_packet_buf_t pkt_buf) +{ + odp_pool_t pool = _odp_pkt_get(pkt_buf, odp_pool_t, pool); + const uint32_t head_offset = _odp_pool_get(pool, uint32_t, ext_head_offset); + + /* Check that pool is external */ + if (odp_unlikely(!head_offset)) + return NULL; + + return (uint8_t *)(uintptr_t)pkt_buf + head_offset; +} + +_ODP_INLINE uint32_t odp_packet_buf_data_offset(odp_packet_buf_t pkt_buf) +{ + return (uint32_t)((uintptr_t)_odp_pkt_get(pkt_buf, void *, seg_data) - + (uintptr_t)odp_packet_buf_head(pkt_buf)); +} + +_ODP_INLINE void odp_packet_buf_data_set(odp_packet_buf_t pkt_buf, uint32_t data_offset, + uint32_t data_len) +{ + uint8_t *head = (uint8_t *)odp_packet_buf_head(pkt_buf); + uint32_t *seg_len = _odp_pkt_get_ptr(pkt_buf, uint32_t, seg_len); + void **seg_data = _odp_pkt_get_ptr(pkt_buf, void *, seg_data); + + *seg_len = data_len; + *seg_data = head + data_offset; +} + +_ODP_INLINE odp_packet_buf_t odp_packet_buf_from_head(odp_pool_t pool, void *head) +{ + const uint32_t head_offset = _odp_pool_get(pool, uint32_t, ext_head_offset); + + /* Check that pool is external */ + if (odp_unlikely(!head_offset)) + return ODP_PACKET_BUF_INVALID; + + return (odp_packet_buf_t)((uintptr_t)head - head_offset); +} + /** @endcond */ #endif diff --git a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h index c8da1b77a..76604dc4f 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h @@ -33,6 +33,9 @@ #define odp_packet_vector_pool __odp_packet_vector_pool #define odp_packet_vector_size __odp_packet_vector_size #define odp_packet_vector_size_set __odp_packet_vector_size_set + #define odp_packet_vector_user_area __odp_packet_vector_user_area + #define odp_packet_vector_user_flag __odp_packet_vector_user_flag + #define odp_packet_vector_user_flag_set __odp_packet_vector_user_flag_set #else #undef _ODP_INLINE #define _ODP_INLINE @@ -74,6 +77,28 @@ _ODP_INLINE void odp_packet_vector_size_set(odp_packet_vector_t pktv, uint32_t s *vector_size = size; } +_ODP_INLINE void *odp_packet_vector_user_area(odp_packet_vector_t pktv) +{ + return _odp_event_vect_get(pktv, void *, uarea_addr); +} + +_ODP_INLINE int odp_packet_vector_user_flag(odp_packet_vector_t pktv) +{ + _odp_event_vector_flags_t flags; + + flags.all_flags = _odp_event_vect_get(pktv, uint32_t, flags); + + return flags.user_flag; +} + +_ODP_INLINE void odp_packet_vector_user_flag_set(odp_packet_vector_t pktv, int val) +{ + _odp_event_vector_flags_t *flags = _odp_event_vect_get_ptr(pktv, _odp_event_vector_flags_t, + flags); + + flags->user_flag = !!val; +} + /** @endcond */ #endif diff --git a/platform/linux-generic/include/odp/api/plat/pool_inline_types.h b/platform/linux-generic/include/odp/api/plat/pool_inline_types.h index 0c356dbf2..9deec89a1 100644 --- a/platform/linux-generic/include/odp/api/plat/pool_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/pool_inline_types.h @@ -26,7 +26,13 @@ typedef struct _odp_pool_inline_offset_t { /** @internal field offset */ uint16_t index; /** @internal field offset */ + uint16_t seg_len; + /** @internal field offset */ uint16_t uarea_size; + /** @internal field offset */ + uint16_t ext_head_offset; + /** @internal field offset */ + uint16_t ext_pkt_buf_size; } _odp_pool_inline_offset_t; diff --git a/platform/linux-generic/include/odp/api/plat/timer_inline_types.h b/platform/linux-generic/include/odp/api/plat/timer_inline_types.h index e3397c4df..ec6804c72 100644 --- a/platform/linux-generic/include/odp/api/plat/timer_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/timer_inline_types.h @@ -25,6 +25,7 @@ typedef struct _odp_timeout_inline_offset_t { uint16_t expiration; uint16_t timer; uint16_t user_ptr; + uint16_t uarea_addr; } _odp_timeout_inline_offset_t; diff --git a/platform/linux-generic/include/odp/api/plat/timer_inlines.h b/platform/linux-generic/include/odp/api/plat/timer_inlines.h index 406aefdf6..7642376d0 100644 --- a/platform/linux-generic/include/odp/api/plat/timer_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/timer_inlines.h @@ -24,6 +24,7 @@ extern const _odp_timeout_inline_offset_t _odp_timeout_inline_offset; #define odp_timeout_timer __odp_timeout_timer #define odp_timeout_tick __odp_timeout_tick #define odp_timeout_user_ptr __odp_timeout_user_ptr + #define odp_timeout_user_area __odp_timeout_user_area #define odp_timer_tick_to_ns __odp_timer_tick_to_ns #define odp_timer_ns_to_tick __odp_timer_ns_to_tick #define odp_timeout_from_event __odp_timeout_from_event @@ -47,6 +48,11 @@ _ODP_INLINE void *odp_timeout_user_ptr(odp_timeout_t tmo) return _odp_timeout_hdr_field(tmo, void *, user_ptr); } +_ODP_INLINE void *odp_timeout_user_area(odp_timeout_t tmo) +{ + return _odp_timeout_hdr_field(tmo, void *, uarea_addr); +} + _ODP_INLINE uint64_t odp_timer_tick_to_ns(odp_timer_pool_t tp, uint64_t ticks) { (void)tp; diff --git a/platform/linux-generic/include/odp_atomic_internal.h b/platform/linux-generic/include/odp_atomic_internal.h index da79b3723..6de8cd485 100644 --- a/platform/linux-generic/include/odp_atomic_internal.h +++ b/platform/linux-generic/include/odp_atomic_internal.h @@ -159,31 +159,6 @@ static inline void _odp_atomic_u128_xchg_mm(_odp_atomic_u128_t *ptr, { __atomic_exchange(&ptr->v, val, old, mm); } - -/** - * Atomic compare and exchange (swap) of 16-byte atomic variable - * "Strong" semantics, will not fail spuriously. - * - * @param ptr Pointer to a 16-byte atomic variable - * @param exp Pointer to expected value (updated on failure) - * @param val Pointer to new value to write - * @param succ Memory model associated with a successful compare-and-swap - * operation - * @param fail Memory model associated with a failed compare-and-swap - * operation - * - * @retval 1 exchange successul - * @retval 0 exchange failed and '*exp' updated with current value - */ -static inline int _odp_atomic_u128_cmp_xchg_mm(_odp_atomic_u128_t *ptr, - _odp_u128_t *exp, - _odp_u128_t *val, - _odp_memmodel_t succ, - _odp_memmodel_t fail) -{ - return __atomic_compare_exchange(&ptr->v, exp, val, - false/*strong*/, succ, fail); -} #endif /** diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 8625fc5dd..1cececb99 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -36,6 +36,9 @@ typedef struct ODP_ALIGNED_CACHE odp_buffer_hdr_t { /* Common event header */ _odp_event_hdr_t event_hdr; + /* User area pointer */ + void *uarea_addr; + /* Data */ uint8_t data[]; } odp_buffer_hdr_t; diff --git a/platform/linux-generic/include/odp_debug_internal.h b/platform/linux-generic/include/odp_debug_internal.h index c5c1890c3..22dca3701 100644 --- a/platform/linux-generic/include/odp_debug_internal.h +++ b/platform/linux-generic/include/odp_debug_internal.h @@ -37,15 +37,8 @@ extern "C" { * level 0 to N. */ #define CONFIG_DEBUG_LEVEL 0 -ODP_PRINTF_FORMAT(1, 2) -static inline void check_printf_format(const char *fmt, ...) -{ - (void)fmt; -} - #define _ODP_LOG_FN(level, fmt, ...) \ do { \ - check_printf_format(fmt, ##__VA_ARGS__); \ if (_odp_this_thread && _odp_this_thread->log_fn) \ _odp_this_thread->log_fn(level, fmt, ##__VA_ARGS__); \ else \ diff --git a/platform/linux-generic/include/odp_event_vector_internal.h b/platform/linux-generic/include/odp_event_vector_internal.h index 33b26d711..55e33b913 100644 --- a/platform/linux-generic/include/odp_event_vector_internal.h +++ b/platform/linux-generic/include/odp_event_vector_internal.h @@ -17,6 +17,8 @@ #include <odp/api/debug.h> #include <odp/api/packet.h> +#include <odp/api/plat/event_vector_inline_types.h> + #include <odp_event_internal.h> #include <stdint.h> @@ -28,9 +30,15 @@ typedef struct ODP_ALIGNED_CACHE odp_event_vector_hdr_t { /* Common event header */ _odp_event_hdr_t event_hdr; + /* User area pointer */ + void *uarea_addr; + /* Event vector size */ uint32_t size; + /* Flags */ + _odp_event_vector_flags_t flags; + /* Vector of packet handles */ odp_packet_t packet[]; diff --git a/platform/linux-generic/include/odp_global_data.h b/platform/linux-generic/include/odp_global_data.h index 06e269b4d..462b8d639 100644 --- a/platform/linux-generic/include/odp_global_data.h +++ b/platform/linux-generic/include/odp_global_data.h @@ -61,7 +61,7 @@ typedef struct odp_global_data_ro_t { pid_t main_pid; pid_t fdserver_pid; char uid[UID_MAXLEN]; - odp_log_func_t log_fn; + odp_log_func_t ODP_PRINTF_FORMAT(2, 3) log_fn; odp_abort_func_t abort_fn; system_info_t system_info; hugepage_info_t hugepage_info; diff --git a/platform/linux-generic/include/odp_llqueue.h b/platform/linux-generic/include/odp_llqueue.h index 68325624a..6340d111a 100644 --- a/platform/linux-generic/include/odp_llqueue.h +++ b/platform/linux-generic/include/odp_llqueue.h @@ -41,6 +41,7 @@ static odp_bool_t llq_on_queue(struct llnode *node); *****************************************************************************/ #define SENTINEL ((void *)~(uintptr_t)0) +#define MAX_SPIN_COUNT 1000 #ifdef CONFIG_LLDSCD /* Implement queue operations using double-word LL/SC */ @@ -114,6 +115,7 @@ static inline struct llnode *llq_dequeue(struct llqueue *llq) (void)__atomic_load_n(&head->next, __ATOMIC_RELAXED); do { +restart_loop: old.ui = lld(&llq->u.ui, __ATOMIC_RELAXED); if (odp_unlikely(old.st.head == NULL)) { /* Empty list */ @@ -125,13 +127,18 @@ static inline struct llnode *llq_dequeue(struct llqueue *llq) } else { /* Multi-element list, dequeue head */ struct llnode *next; + int spin_count = 0; + /* Wait until llq_enqueue() has written true next * pointer */ while ((next = __atomic_load_n(&old.st.head->next, __ATOMIC_RELAXED)) == - SENTINEL) + SENTINEL) { odp_cpu_pause(); + if (++spin_count >= MAX_SPIN_COUNT) + goto restart_loop; + } neu.st.head = next; neu.st.tail = old.st.tail; } @@ -146,6 +153,7 @@ static inline odp_bool_t llq_dequeue_cond(struct llqueue *llq, union llht old, neu; do { +restart_loop: old.ui = lld(&llq->u.ui, __ATOMIC_ACQUIRE); if (odp_unlikely(old.st.head == NULL || old.st.head != exp)) { /* Empty list or wrong head */ @@ -157,13 +165,17 @@ static inline odp_bool_t llq_dequeue_cond(struct llqueue *llq, } else { /* Multi-element list, dequeue head */ struct llnode *next; + int spin_count = 0; /* Wait until llq_enqueue() has written true next * pointer */ while ((next = __atomic_load_n(&old.st.head->next, __ATOMIC_RELAXED)) == - SENTINEL) + SENTINEL) { odp_cpu_pause(); + if (++spin_count >= MAX_SPIN_COUNT) + goto restart_loop; + } neu.st.head = next; neu.st.tail = old.st.tail; diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index 201dbebf9..1c5b51c3d 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -18,6 +18,7 @@ extern "C" { #endif +#include <odp/api/atomic.h> #include <odp/api/shared_memory.h> #include <odp/api/ticketlock.h> #include <odp/api/align.h> @@ -32,7 +33,7 @@ extern "C" { typedef struct ODP_ALIGNED_CACHE pool_cache_t { /* Number of buffers in cache */ - uint32_t cache_num; + odp_atomic_u32_t cache_num; /* Cached buffers */ _odp_event_hdr_t *event_hdr[CONFIG_POOL_CACHE_MAX_SIZE]; @@ -91,6 +92,7 @@ typedef struct pool_t { uint8_t *uarea_base_addr; odp_pool_type_t type_2; odp_pool_ext_param_t ext_param; + uint32_t ext_head_offset; uint32_t skipped_blocks; uint8_t mem_from_huge_pages; const struct _odp_pool_mem_src_ops_t *mem_src_ops; diff --git a/platform/linux-generic/include/odp_print_internal.h b/platform/linux-generic/include/odp_print_internal.h new file mode 100644 index 000000000..949a1cc70 --- /dev/null +++ b/platform/linux-generic/include/odp_print_internal.h @@ -0,0 +1,22 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PRINT_INTERNAL_H_ +#define ODP_PRINT_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stddef.h> + +int _odp_snprint(char *str, size_t size, const char *format, ...); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp_ring_internal.h b/platform/linux-generic/include/odp_ring_internal.h index dcd190d07..296a87116 100644 --- a/platform/linux-generic/include/odp_ring_internal.h +++ b/platform/linux-generic/include/odp_ring_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited - * Copyright (c) 2019-2021, Nokia + * Copyright (c) 2019-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -80,6 +80,7 @@ static inline int cas_mo_u32(odp_atomic_u32_t *atom, uint32_t *old_val, #undef _RING_INIT #undef _RING_DEQ #undef _RING_DEQ_MULTI +#undef _RING_DEQ_BATCH #undef _RING_ENQ #undef _RING_ENQ_MULTI #undef _RING_LEN @@ -94,6 +95,7 @@ static inline int cas_mo_u32(odp_atomic_u32_t *atom, uint32_t *old_val, #define _RING_INIT ring_u32_init #define _RING_DEQ ring_u32_deq #define _RING_DEQ_MULTI ring_u32_deq_multi + #define _RING_DEQ_BATCH ring_u32_deq_batch #define _RING_ENQ ring_u32_enq #define _RING_ENQ_MULTI ring_u32_enq_multi #define _RING_LEN ring_u32_len @@ -104,6 +106,7 @@ static inline int cas_mo_u32(odp_atomic_u32_t *atom, uint32_t *old_val, #define _RING_INIT ring_u64_init #define _RING_DEQ ring_u64_deq #define _RING_DEQ_MULTI ring_u64_deq_multi + #define _RING_DEQ_BATCH ring_u64_deq_batch #define _RING_ENQ ring_u64_enq #define _RING_ENQ_MULTI ring_u64_enq_multi #define _RING_LEN ring_u64_len @@ -114,6 +117,7 @@ static inline int cas_mo_u32(odp_atomic_u32_t *atom, uint32_t *old_val, #define _RING_INIT ring_ptr_init #define _RING_DEQ ring_ptr_deq #define _RING_DEQ_MULTI ring_ptr_deq_multi + #define _RING_DEQ_BATCH ring_ptr_deq_batch #define _RING_ENQ ring_ptr_enq #define _RING_ENQ_MULTI ring_ptr_enq_multi #define _RING_LEN ring_ptr_len @@ -208,6 +212,45 @@ static inline uint32_t _RING_DEQ_MULTI(_ring_gen_t *ring, uint32_t mask, return num; } +/* Dequeue batch of data (0 or num) from the ring head. Num is smaller than ring size. */ +static inline uint32_t _RING_DEQ_BATCH(_ring_gen_t *ring, uint32_t mask, + _ring_data_t data[], uint32_t num) +{ + uint32_t head, tail, new_head, i; + + /* Load/CAS acquire of r_head ensures that w_tail load happens after + * r_head load, and thus head value is always behind or equal to tail + * value. */ + head = odp_atomic_load_acq_u32(&ring->r.r_head); + + /* Move reader head. This thread owns data at the new head. */ + do { + tail = odp_atomic_load_acq_u32(&ring->r.w_tail); + + /* Not enough data available */ + if ((tail - head) < num) + return 0; + + new_head = head + num; + + } while (odp_unlikely(cas_mo_u32(&ring->r.r_head, &head, new_head, + __ATOMIC_ACQUIRE, + __ATOMIC_ACQUIRE) == 0)); + + /* Read data. */ + for (i = 0; i < num; i++) + data[i] = ring->data[(head + 1 + i) & mask]; + + /* Wait until other readers have updated the tail */ + while (odp_unlikely(odp_atomic_load_u32(&ring->r.r_tail) != head)) + odp_cpu_pause(); + + /* Update the tail. Writers acquire it. */ + odp_atomic_store_rel_u32(&ring->r.r_tail, new_head); + + return num; +} + /* Enqueue data into the ring tail */ static inline void _RING_ENQ(_ring_gen_t *ring, uint32_t mask, _ring_data_t data) diff --git a/platform/linux-generic/include/odp_timer_internal.h b/platform/linux-generic/include/odp_timer_internal.h index b83591446..01ee4a0f3 100644 --- a/platform/linux-generic/include/odp_timer_internal.h +++ b/platform/linux-generic/include/odp_timer_internal.h @@ -35,6 +35,9 @@ typedef struct ODP_ALIGNED_CACHE odp_timeout_hdr_t { /* User ptr inherited from parent timer */ const void *user_ptr; + /* User area pointer */ + void *uarea_addr; + /* Parent timer */ odp_timer_t timer; diff --git a/platform/linux-generic/m4/odp_libconfig.m4 b/platform/linux-generic/m4/odp_libconfig.m4 index 886cc07e8..03dbc929d 100644 --- a/platform/linux-generic/m4/odp_libconfig.m4 +++ b/platform/linux-generic/m4/odp_libconfig.m4 @@ -3,7 +3,7 @@ ########################################################################## m4_define([_odp_config_version_generation], [0]) m4_define([_odp_config_version_major], [1]) -m4_define([_odp_config_version_minor], [21]) +m4_define([_odp_config_version_minor], [22]) m4_define([_odp_config_version], [_odp_config_version_generation._odp_config_version_major._odp_config_version_minor]) diff --git a/platform/linux-generic/odp_buffer.c b/platform/linux-generic/odp_buffer.c index 609b0b206..278aa8147 100644 --- a/platform/linux-generic/odp_buffer.c +++ b/platform/linux-generic/odp_buffer.c @@ -23,6 +23,13 @@ uint32_t odp_buffer_size(odp_buffer_t buf) return pool->seg_len; } +void *odp_buffer_user_area(odp_buffer_t buf) +{ + odp_buffer_hdr_t *hdr = _odp_buf_hdr(buf); + + return hdr->uarea_addr; +} + void odp_buffer_print(odp_buffer_t buf) { odp_buffer_hdr_t *hdr; diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index 6bb4fa3c2..ed0f6723d 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2019-2021, Nokia + * Copyright (c) 2019-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -181,6 +181,7 @@ int odp_cls_capability(odp_cls_capability_t *capability) capability->supported_terms.bit.dip_addr = 1; capability->supported_terms.bit.sip6_addr = 1; capability->supported_terms.bit.dip6_addr = 1; + capability->supported_terms.bit.ipsec_spi = 1; capability->supported_terms.bit.custom_frame = 1; capability->supported_terms.bit.custom_l3 = 1; capability->random_early_detection = ODP_SUPPORT_NO; @@ -188,6 +189,12 @@ int odp_cls_capability(odp_cls_capability_t *capability) capability->threshold_red.all_bits = 0; capability->threshold_bp.all_bits = 0; capability->max_hash_queues = CLS_COS_QUEUE_MAX; + capability->hash_protocols.proto.ipv4_udp = 1; + capability->hash_protocols.proto.ipv4_tcp = 1; + capability->hash_protocols.proto.ipv4 = 1; + capability->hash_protocols.proto.ipv6_udp = 1; + capability->hash_protocols.proto.ipv6_tcp = 1; + capability->hash_protocols.proto.ipv6 = 1; capability->max_mark = MAX_MARK; capability->stats.cos.counter.discards = 1; capability->stats.cos.counter.packets = 1; @@ -290,14 +297,17 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param_ cos->num_queue = param.num_queue; if (param.num_queue > 1) { - odp_queue_param_init(&cos->queue_param); + cos->queue_param = param.queue_param; cos->queue_group = true; cos->queue = ODP_QUEUE_INVALID; _odp_cls_update_hash_proto(cos, param.hash_proto); tbl_index = i * CLS_COS_QUEUE_MAX; for (j = 0; j < param.num_queue; j++) { - queue = odp_queue_create(NULL, &cos->queue_param); + char name[ODP_QUEUE_NAME_LEN]; + + snprintf(name, sizeof(name), "_odp_cos_hq_%u_%u", i, j); + queue = odp_queue_create(name, &cos->queue_param); if (queue == ODP_QUEUE_INVALID) { /* unwind the queues */ _cls_queue_unwind(tbl_index, j); @@ -309,6 +319,7 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param_ } } else { + cos->queue_group = false; cos->queue = param.queue; } @@ -398,6 +409,9 @@ int odp_cos_destroy(odp_cos_t cos_id) return -1; } + if (cos->queue_group) + _cls_queue_unwind(cos->index * CLS_COS_QUEUE_MAX, cos->num_queue); + cos->valid = 0; return 0; } @@ -1155,17 +1169,12 @@ static inline int verify_pmr_ipsec_spi(const uint8_t *pkt_addr, pkt_addr += pkt_hdr->p.l4_offset; - if (pkt_hdr->p.input_flags.ipsec_ah) { - const _odp_ahhdr_t *ahhdr = (const _odp_ahhdr_t *)pkt_addr; - - spi = odp_be_to_cpu_32(ahhdr->spi); - } else if (pkt_hdr->p.input_flags.ipsec_esp) { - const _odp_esphdr_t *esphdr = (const _odp_esphdr_t *)pkt_addr; - - spi = odp_be_to_cpu_32(esphdr->spi); - } else { + if (pkt_hdr->p.input_flags.ipsec_ah) + spi = ((const _odp_ahhdr_t *)pkt_addr)->spi; + else if (pkt_hdr->p.input_flags.ipsec_esp) + spi = ((const _odp_esphdr_t *)pkt_addr)->spi; + else return 0; - } if (term_value->match.value == (spi & term_value->match.mask)) return 1; @@ -1959,11 +1968,9 @@ void print_queue_ident(odp_queue_t q) odp_queue_info_t info; if (!odp_queue_info(q, &info) && strlen(info.name)) - ODP_PRINT("%s", info.name); + ODP_PRINT(" %s\n", info.name); else - ODP_PRINT("%" PRIx64, odp_queue_to_u64(q)); - - ODP_PRINT("\n"); + ODP_PRINT(" %" PRIx64 "\n", odp_queue_to_u64(q)); } static @@ -1978,13 +1985,20 @@ void print_hex(const void *vp, int len) static void cls_print_cos(cos_t *cos) { + uint32_t tbl_index = cos->index * CLS_COS_QUEUE_MAX; uint32_t num_rule = odp_atomic_load_u32(&cos->num_rule); bool first = true; ODP_PRINT("cos: "); print_cos_ident(cos); - ODP_PRINT(" queue: "); - print_queue_ident(cos->queue); + ODP_PRINT(" queues:\n"); + + if (!cos->queue_group) { + print_queue_ident(cos->queue); + } else { + for (uint32_t i = 0; i < cos->num_queue; i++) + print_queue_ident(queue_grp_tbl->queue[tbl_index + i]); + } for (uint32_t j = 0; j < num_rule; j++) { pmr_t *pmr = cos->pmr[j]; diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c index 8eb2332a1..981c9239c 100644 --- a/platform/linux-generic/odp_crypto_null.c +++ b/platform/linux-generic/odp_crypto_null.c @@ -486,54 +486,72 @@ int odp_crypto_result(odp_crypto_packet_result_t *result, return 0; } +static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src) +{ + int md_copy; + int rc; + + md_copy = _odp_packet_copy_md_possible(odp_packet_pool(dst), + odp_packet_pool(src)); + if (odp_unlikely(md_copy < 0)) { + ODP_ERR("Unable to copy packet metadata\n"); + return -1; + } + + rc = odp_packet_copy_from_pkt(dst, 0, src, 0, odp_packet_len(src)); + if (odp_unlikely(rc < 0)) { + ODP_ERR("Unable to copy packet data\n"); + return -1; + } + + _odp_packet_copy_md(packet_hdr(dst), packet_hdr(src), md_copy); + return 0; +} + +static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *session, + odp_packet_t pkt_in, + odp_packet_t pkt_out) +{ + int rc; + + if (odp_likely(pkt_in == pkt_out)) + return pkt_out; + + if (pkt_out == ODP_PACKET_INVALID) { + odp_pool_t pool = session->p.output_pool; + + ODP_ASSERT(pool != ODP_POOL_INVALID); + if (pool == odp_packet_pool(pkt_in)) { + pkt_out = pkt_in; + } else { + pkt_out = odp_packet_copy(pkt_in, pool); + if (odp_likely(pkt_out != ODP_PACKET_INVALID)) + odp_packet_free(pkt_in); + } + return pkt_out; + } + rc = copy_data_and_metadata(pkt_out, pkt_in); + if (odp_unlikely(rc < 0)) + return ODP_PACKET_INVALID; + + odp_packet_free(pkt_in); + return pkt_out; +} + static int crypto_int(odp_packet_t pkt_in, odp_packet_t *pkt_out, const odp_crypto_packet_op_param_t *param) { odp_crypto_generic_session_t *session; - odp_bool_t allocated = false; - odp_packet_t out_pkt = *pkt_out; + odp_packet_t out_pkt; odp_crypto_packet_result_t *op_result; session = (odp_crypto_generic_session_t *)(intptr_t)param->session; - /* Resolve output buffer */ - if (ODP_PACKET_INVALID == out_pkt && - ODP_POOL_INVALID != session->p.output_pool) { - out_pkt = odp_packet_alloc(session->p.output_pool, - odp_packet_len(pkt_in)); - allocated = true; - } - - if (odp_unlikely(ODP_PACKET_INVALID == out_pkt)) { - ODP_DBG("Alloc failed.\n"); + out_pkt = get_output_packet(session, pkt_in, *pkt_out); + if (odp_unlikely(out_pkt == ODP_PACKET_INVALID)) return -1; - } - - if (pkt_in != out_pkt) { - int ret; - int md_copy; - - md_copy = _odp_packet_copy_md_possible(session->p.output_pool, - odp_packet_pool(pkt_in)); - if (odp_unlikely(md_copy < 0)) { - ODP_ERR("Unable to copy packet metadata\n"); - goto err; - } - - ret = odp_packet_copy_from_pkt(out_pkt, - 0, - pkt_in, - 0, - odp_packet_len(pkt_in)); - if (odp_unlikely(ret < 0)) - goto err; - - _odp_packet_copy_md(packet_hdr(out_pkt), packet_hdr(pkt_in), md_copy); - odp_packet_free(pkt_in); - pkt_in = ODP_PACKET_INVALID; - } /* Fill in result */ packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO); @@ -548,14 +566,6 @@ int crypto_int(odp_packet_t pkt_in, *pkt_out = out_pkt; return 0; - -err: - if (allocated) { - odp_packet_free(out_pkt); - *pkt_out = ODP_PACKET_INVALID; - } - - return -1; } int odp_crypto_op(const odp_packet_t pkt_in[], diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c index 9402c805b..9b5ea4612 100644 --- a/platform/linux-generic/odp_crypto_openssl.c +++ b/platform/linux-generic/odp_crypto_openssl.c @@ -2815,6 +2815,58 @@ int odp_crypto_result(odp_crypto_packet_result_t *result, return 0; } +static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src) +{ + int md_copy; + int rc; + + md_copy = _odp_packet_copy_md_possible(odp_packet_pool(dst), + odp_packet_pool(src)); + if (odp_unlikely(md_copy < 0)) { + ODP_ERR("Unable to copy packet metadata\n"); + return -1; + } + + rc = odp_packet_copy_from_pkt(dst, 0, src, 0, odp_packet_len(src)); + if (odp_unlikely(rc < 0)) { + ODP_ERR("Unable to copy packet data\n"); + return -1; + } + + _odp_packet_copy_md(packet_hdr(dst), packet_hdr(src), md_copy); + return 0; +} + +static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *session, + odp_packet_t pkt_in, + odp_packet_t pkt_out) +{ + int rc; + + if (odp_likely(pkt_in == pkt_out)) + return pkt_out; + + if (pkt_out == ODP_PACKET_INVALID) { + odp_pool_t pool = session->p.output_pool; + + ODP_ASSERT(pool != ODP_POOL_INVALID); + if (pool == odp_packet_pool(pkt_in)) { + pkt_out = pkt_in; + } else { + pkt_out = odp_packet_copy(pkt_in, pool); + if (odp_likely(pkt_out != ODP_PACKET_INVALID)) + odp_packet_free(pkt_in); + } + return pkt_out; + } + rc = copy_data_and_metadata(pkt_out, pkt_in); + if (odp_unlikely(rc < 0)) + return ODP_PACKET_INVALID; + + odp_packet_free(pkt_in); + return pkt_out; +} + static int crypto_int(odp_packet_t pkt_in, odp_packet_t *pkt_out, @@ -2823,48 +2875,14 @@ int crypto_int(odp_packet_t pkt_in, odp_crypto_alg_err_t rc_cipher = ODP_CRYPTO_ALG_ERR_NONE; odp_crypto_alg_err_t rc_auth = ODP_CRYPTO_ALG_ERR_NONE; odp_crypto_generic_session_t *session; - odp_bool_t allocated = false; - odp_packet_t out_pkt = *pkt_out; + odp_packet_t out_pkt; odp_crypto_packet_result_t *op_result; session = (odp_crypto_generic_session_t *)(intptr_t)param->session; - /* Resolve output buffer */ - if (ODP_PACKET_INVALID == out_pkt && - ODP_POOL_INVALID != session->p.output_pool) { - out_pkt = odp_packet_alloc(session->p.output_pool, - odp_packet_len(pkt_in)); - allocated = true; - } - - if (odp_unlikely(ODP_PACKET_INVALID == out_pkt)) { - ODP_DBG("Alloc failed.\n"); + out_pkt = get_output_packet(session, pkt_in, *pkt_out); + if (odp_unlikely(out_pkt == ODP_PACKET_INVALID)) return -1; - } - - if (pkt_in != out_pkt) { - int ret; - int md_copy; - - md_copy = _odp_packet_copy_md_possible(session->p.output_pool, - odp_packet_pool(pkt_in)); - if (odp_unlikely(md_copy < 0)) { - ODP_ERR("Unable to copy packet metadata\n"); - goto err; - } - - ret = odp_packet_copy_from_pkt(out_pkt, - 0, - pkt_in, - 0, - odp_packet_len(pkt_in)); - if (odp_unlikely(ret < 0)) - goto err; - - _odp_packet_copy_md(packet_hdr(out_pkt), packet_hdr(pkt_in), md_copy); - odp_packet_free(pkt_in); - pkt_in = ODP_PACKET_INVALID; - } crypto_init(session); @@ -2892,14 +2910,6 @@ int crypto_int(odp_packet_t pkt_in, *pkt_out = out_pkt; return 0; - -err: - if (allocated) { - odp_packet_free(out_pkt); - *pkt_out = ODP_PACKET_INVALID; - } - - return -1; } int odp_crypto_op(const odp_packet_t pkt_in[], diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 04b4b6aeb..98fd2ac53 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -73,12 +73,13 @@ static void wait_for_order(ordering_mode_t mode) */ static int set_ipsec_crypto_capa(odp_ipsec_capability_t *capa) { - int rc; odp_crypto_capability_t crypto_capa; - rc = odp_crypto_capability(&crypto_capa); - if (rc < 0) - return rc; + crypto_capa.ciphers.all_bits = 0; + crypto_capa.auths.all_bits = 0; + + if (odp_crypto_capability(&crypto_capa)) + return -1; #define CHECK_CIPHER(field, alg) do { \ if (crypto_capa.ciphers.bit.field && \ diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 64d7b6fdf..e689089d3 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -168,6 +168,8 @@ int _odp_ipsec_sad_init_global(void) if (odp_global_ro.disable.ipsec) return 0; + crypto_capa.max_sessions = 0; + if (odp_crypto_capability(&crypto_capa)) { ODP_ERR("odp_crypto_capability() failed\n"); return -1; diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index d96eb8748..775836b66 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -1039,13 +1039,6 @@ void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ptr) pkt_hdr->p.flags.user_ptr_set = 1; } -void odp_packet_input_set(odp_packet_t pkt, odp_pktio_t pktio) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - pkt_hdr->input = pktio; -} - int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); @@ -2432,75 +2425,6 @@ odp_packet_reass_partial_state(odp_packet_t pkt, odp_packet_t frags[], return -ENOTSUP; } -static inline odp_packet_hdr_t *packet_buf_to_hdr(odp_packet_buf_t pkt_buf) -{ - return (odp_packet_hdr_t *)(uintptr_t)pkt_buf; -} - -void *odp_packet_buf_head(odp_packet_buf_t pkt_buf) -{ - odp_packet_hdr_t *pkt_hdr = packet_buf_to_hdr(pkt_buf); - pool_t *pool = _odp_pool_entry(pkt_hdr->event_hdr.pool); - uint32_t head_offset = sizeof(odp_packet_hdr_t) + pool->ext_param.pkt.app_header_size; - - if (odp_unlikely(pool->pool_ext == 0)) { - ODP_ERR("Not an external memory pool\n"); - return NULL; - } - - return (uint8_t *)pkt_hdr + head_offset; -} - -uint32_t odp_packet_buf_size(odp_packet_buf_t pkt_buf) -{ - odp_packet_hdr_t *pkt_hdr = packet_buf_to_hdr(pkt_buf); - pool_t *pool = _odp_pool_entry(pkt_hdr->event_hdr.pool); - uint32_t head_offset = sizeof(odp_packet_hdr_t) + pool->ext_param.pkt.app_header_size; - - return pool->ext_param.pkt.buf_size - head_offset; -} - -uint32_t odp_packet_buf_data_offset(odp_packet_buf_t pkt_buf) -{ - odp_packet_hdr_t *pkt_hdr = packet_buf_to_hdr(pkt_buf); - - return (uintptr_t)pkt_hdr->seg_data - (uintptr_t)odp_packet_buf_head(pkt_buf); -} - -uint32_t odp_packet_buf_data_len(odp_packet_buf_t pkt_buf) -{ - odp_packet_hdr_t *pkt_hdr = packet_buf_to_hdr(pkt_buf); - - return pkt_hdr->seg_len; -} - -void odp_packet_buf_data_set(odp_packet_buf_t pkt_buf, uint32_t data_offset, uint32_t data_len) -{ - odp_packet_hdr_t *pkt_hdr = packet_buf_to_hdr(pkt_buf); - uint8_t *head = odp_packet_buf_head(pkt_buf); - - pkt_hdr->seg_len = data_len; - pkt_hdr->seg_data = head + data_offset; -} - -odp_packet_buf_t odp_packet_buf_from_head(odp_pool_t pool_hdl, void *head) -{ - pool_t *pool = _odp_pool_entry(pool_hdl); - uint32_t head_offset = sizeof(odp_packet_hdr_t) + pool->ext_param.pkt.app_header_size; - - if (odp_unlikely(pool->type != ODP_POOL_PACKET)) { - ODP_ERR("Not a packet pool\n"); - return ODP_PACKET_BUF_INVALID; - } - - if (odp_unlikely(pool->pool_ext == 0)) { - ODP_ERR("Not an external memory pool\n"); - return ODP_PACKET_BUF_INVALID; - } - - return (odp_packet_buf_t)((uintptr_t)head - head_offset); -} - uint32_t odp_packet_disassemble(odp_packet_t pkt, odp_packet_buf_t pkt_buf[], uint32_t num) { uint32_t i; diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index d98449a4b..c9f43ab9c 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -2154,7 +2154,7 @@ void odp_pktio_extra_stats_print(odp_pktio_t pktio) return; } - printf("Pktio extra statistics\n----------------------\n"); + ODP_PRINT("Pktio extra statistics\n----------------------\n"); for (i = 0; i < num_stats; i++) ODP_PRINT(" %s=%" PRIu64 "\n", stats_info[i].name, extra_stats[i]); ODP_PRINT("\n"); diff --git a/platform/linux-generic/odp_packet_vector.c b/platform/linux-generic/odp_packet_vector.c index e0b99183f..b3edbf84b 100644 --- a/platform/linux-generic/odp_packet_vector.c +++ b/platform/linux-generic/odp_packet_vector.c @@ -24,7 +24,9 @@ const _odp_event_vector_inline_offset_t _odp_event_vector_inline ODP_ALIGNED_CACHE = { .packet = offsetof(odp_event_vector_hdr_t, packet), .pool = offsetof(odp_event_vector_hdr_t, event_hdr.pool), - .size = offsetof(odp_event_vector_hdr_t, size) + .size = offsetof(odp_event_vector_hdr_t, size), + .uarea_addr = offsetof(odp_event_vector_hdr_t, uarea_addr), + .flags = offsetof(odp_event_vector_hdr_t, flags) }; #include <odp/visibility_end.h> @@ -59,6 +61,7 @@ void odp_packet_vector_free(odp_packet_vector_t pktv) odp_event_vector_hdr_t *pktv_hdr = _odp_packet_vector_hdr(pktv); pktv_hdr->size = 0; + pktv_hdr->flags.all_flags = 0; _odp_event_free(odp_packet_vector_to_event(pktv)); } diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 90cdd6590..a92cc615d 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -5,12 +5,16 @@ * SPDX-License-Identifier: BSD-3-Clause */ +#include <odp/api/align.h> +#include <odp/api/atomic.h> #include <odp/api/pool.h> #include <odp/api/shared_memory.h> -#include <odp/api/align.h> -#include <odp/api/ticketlock.h> #include <odp/api/system_info.h> +#include <odp/api/ticketlock.h> + +#include <odp/api/plat/pool_inline_types.h> #include <odp/api/plat/thread_inlines.h> +#include <odp/api/plat/ticketlock_inlines.h> #include <odp_pool_internal.h> #include <odp_init_internal.h> @@ -31,8 +35,6 @@ #include <stddef.h> #include <inttypes.h> -#include <odp/api/plat/pool_inline_types.h> -#include <odp/api/plat/ticketlock_inlines.h> #define LOCK(a) odp_ticketlock_lock(a) #define UNLOCK(a) odp_ticketlock_unlock(a) #define LOCK_INIT(a) odp_ticketlock_init(a) @@ -73,7 +75,9 @@ static __thread pool_local_t local; /* Fill in pool header field offsets for inline functions */ const _odp_pool_inline_offset_t _odp_pool_inline ODP_ALIGNED_CACHE = { .index = offsetof(pool_t, pool_idx), - .uarea_size = offsetof(pool_t, param_uarea_size) + .uarea_size = offsetof(pool_t, param_uarea_size), + .ext_head_offset = offsetof(pool_t, ext_head_offset), + .ext_pkt_buf_size = offsetof(pool_t, ext_param.pkt.buf_size) }; #include <odp/visibility_end.h> @@ -81,12 +85,13 @@ const _odp_pool_inline_offset_t _odp_pool_inline ODP_ALIGNED_CACHE = { static inline void cache_init(pool_cache_t *cache) { memset(cache, 0, sizeof(pool_cache_t)); + odp_atomic_init_u32(&cache->cache_num, 0); } static inline uint32_t cache_pop(pool_cache_t *cache, _odp_event_hdr_t *event_hdr[], int max_num) { - uint32_t cache_num = cache->cache_num; + uint32_t cache_num = odp_atomic_load_u32(&cache->cache_num); uint32_t num_ch = max_num; uint32_t cache_begin; uint32_t i; @@ -100,7 +105,7 @@ static inline uint32_t cache_pop(pool_cache_t *cache, for (i = 0; i < num_ch; i++) event_hdr[i] = cache->event_hdr[cache_begin + i]; - cache->cache_num = cache_num - num_ch; + odp_atomic_store_u32(&cache->cache_num, cache_num - num_ch); return num_ch; } @@ -108,13 +113,13 @@ static inline uint32_t cache_pop(pool_cache_t *cache, static inline void cache_push(pool_cache_t *cache, _odp_event_hdr_t *event_hdr[], uint32_t num) { - uint32_t cache_num = cache->cache_num; + uint32_t cache_num = odp_atomic_load_u32(&cache->cache_num); uint32_t i; for (i = 0; i < num; i++) cache->event_hdr[cache_num + i] = event_hdr[i]; - cache->cache_num = cache_num + num; + odp_atomic_store_u32(&cache->cache_num, cache_num + num); } static void cache_flush(pool_cache_t *cache, pool_t *pool) @@ -152,8 +157,7 @@ static inline int cache_available(pool_t *pool, odp_pool_stats_t *stats) } for (int i = 0; i < ODP_THREAD_COUNT_MAX; i++) { - /* TODO: thread specific counters should be atomics */ - uint32_t cur = pool->local_cache[i].cache_num; + uint32_t cur = odp_atomic_load_u32(&pool->local_cache[i].cache_num); if (per_thread && i >= first && i <= last) stats->thread.cache_available[out_idx++] = cur; @@ -438,14 +442,26 @@ static pool_t *reserve_pool(uint32_t shmflags, uint8_t pool_ext, uint32_t num) } static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t event_index, - uint32_t hdr_len, uint8_t *data_ptr, void *uarea) + uint8_t *data_ptr, void *uarea) { + uint32_t hdr_len; odp_pool_type_t type = pool->type; + if (type == ODP_POOL_BUFFER) + hdr_len = sizeof(odp_buffer_hdr_t); + else if (type == ODP_POOL_PACKET) + hdr_len = sizeof(odp_packet_hdr_t); + else if (type == ODP_POOL_VECTOR) + hdr_len = sizeof(odp_event_vector_hdr_t); + else if (type == ODP_POOL_TIMEOUT) + hdr_len = sizeof(odp_timeout_hdr_t); + else + hdr_len = sizeof(_odp_event_hdr_t); + + /* Zero all event and type specific header fields */ memset(event_hdr, 0, hdr_len); /* Initialize common event metadata */ - event_hdr->index.u32 = 0; event_hdr->index.pool = pool->pool_idx; event_hdr->index.event = event_index; event_hdr->type = type; @@ -458,6 +474,12 @@ static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t e event_hdr->buf_end = data_ptr + pool->seg_len + pool->tailroom; } + if (type == ODP_POOL_BUFFER) { + odp_buffer_hdr_t *buf_hdr = (void *)event_hdr; + + buf_hdr->uarea_addr = uarea; + } + /* Initialize segmentation metadata */ if (type == ODP_POOL_PACKET) { odp_packet_hdr_t *pkt_hdr = (void *)event_hdr; @@ -476,8 +498,15 @@ static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t e if (type == ODP_POOL_VECTOR) { odp_event_vector_hdr_t *vect_hdr = (void *)event_hdr; - vect_hdr->size = 0; event_hdr->event_type = ODP_EVENT_PACKET_VECTOR; + vect_hdr->uarea_addr = uarea; + } + + /* Initialize timeout metadata */ + if (type == ODP_POOL_TIMEOUT) { + odp_timeout_hdr_t *tmo_hdr = (void *)event_hdr; + + tmo_hdr->uarea_addr = uarea; } } @@ -492,7 +521,7 @@ static void init_buffers(pool_t *pool) void *uarea = NULL; uint8_t *data = NULL; uint8_t *data_ptr = NULL; - uint32_t offset, hdr_len; + uint32_t offset; ring_ptr_t *ring; uint32_t mask; odp_pool_type_t type; @@ -550,16 +579,10 @@ static void init_buffers(pool_t *pool) while (((uintptr_t)&data[offset]) % pool->align != 0) offset++; - hdr_len = (uintptr_t)data - (uintptr_t)event_hdr; data_ptr = &data[offset]; - } else { - if (type == ODP_POOL_TIMEOUT) - hdr_len = sizeof(odp_timeout_hdr_t); - else - hdr_len = sizeof(odp_event_vector_hdr_t); } - init_event_hdr(pool, event_hdr, i, hdr_len, data_ptr, uarea); + init_event_hdr(pool, event_hdr, i, data_ptr, uarea); /* Store buffer into the global pool */ if (!skip) @@ -727,6 +750,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, case ODP_POOL_BUFFER: num = params->buf.num; seg_len = params->buf.size; + uarea_size = params->buf.uarea_size; cache_size = params->buf.cache_size; break; @@ -778,11 +802,13 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params, case ODP_POOL_TIMEOUT: num = params->tmo.num; + uarea_size = params->tmo.uarea_size; cache_size = params->tmo.cache_size; break; case ODP_POOL_VECTOR: num = params->vector.num; + uarea_size = params->vector.uarea_size; cache_size = params->vector.cache_size; seg_len = params->vector.max_size * sizeof(odp_packet_t); break; @@ -969,6 +995,11 @@ static int check_params(const odp_pool_param_t *params) return -1; } + if (params->buf.uarea_size > capa.buf.max_uarea_size) { + ODP_ERR("buf.uarea_size too large %u\n", params->buf.uarea_size); + return -1; + } + if (params->stats.all & ~capa.buf.stats.all) { ODP_ERR("Unsupported pool statistics counter\n"); return -1; @@ -1036,6 +1067,11 @@ static int check_params(const odp_pool_param_t *params) return -1; } + if (params->tmo.uarea_size > capa.tmo.max_uarea_size) { + ODP_ERR("tmo.uarea_size too large %u\n", params->tmo.uarea_size); + return -1; + } + if (params->stats.all & ~capa.tmo.stats.all) { ODP_ERR("Unsupported pool statistics counter\n"); return -1; @@ -1067,6 +1103,11 @@ static int check_params(const odp_pool_param_t *params) return -1; } + if (params->vector.uarea_size > capa.vector.max_uarea_size) { + ODP_ERR("vector.uarea_size too large %u\n", params->vector.uarea_size); + return -1; + } + if (params->stats.all & ~capa.vector.stats.all) { ODP_ERR("Unsupported pool statistics counter\n"); return -1; @@ -1278,7 +1319,7 @@ static inline void event_free_to_pool(pool_t *pool, /* Make room into local cache if needed. Do at least burst size * transfer. */ - cache_num = cache->cache_num; + cache_num = odp_atomic_load_u32(&cache->cache_num); if (odp_unlikely((int)(cache_size - cache_num) < num)) { int burst = pool->burst_size; @@ -1418,6 +1459,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->buf.max_align = ODP_CONFIG_BUFFER_ALIGN_MAX; capa->buf.max_size = MAX_SIZE; capa->buf.max_num = CONFIG_POOL_MAX_NUM; + capa->buf.max_uarea_size = MAX_UAREA_SIZE; capa->buf.min_cache_size = 0; capa->buf.max_cache_size = CONFIG_POOL_CACHE_MAX_SIZE; capa->buf.stats.all = supported_stats.all; @@ -1441,6 +1483,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) /* Timeout pools */ capa->tmo.max_pools = max_pools; capa->tmo.max_num = CONFIG_POOL_MAX_NUM; + capa->tmo.max_uarea_size = MAX_UAREA_SIZE; capa->tmo.min_cache_size = 0; capa->tmo.max_cache_size = CONFIG_POOL_CACHE_MAX_SIZE; capa->tmo.stats.all = supported_stats.all; @@ -1449,6 +1492,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->vector.max_pools = max_pools; capa->vector.max_num = CONFIG_POOL_MAX_NUM; capa->vector.max_size = CONFIG_PACKET_VECTOR_MAX_SIZE; + capa->vector.max_uarea_size = MAX_UAREA_SIZE; capa->vector.min_cache_size = 0; capa->vector.max_cache_size = CONFIG_POOL_CACHE_MAX_SIZE; capa->vector.stats.all = supported_stats.all; @@ -1844,6 +1888,9 @@ odp_pool_t odp_pool_ext_create(const char *name, const odp_pool_ext_param_t *par pool->seg_len = buf_size - head_offset - headroom - pool->tailroom; pool->max_seg_len = headroom + pool->seg_len + pool->tailroom; pool->max_len = PKT_MAX_SEGS * pool->seg_len; + pool->ext_head_offset = head_offset; + pool->base_addr = (uint8_t *)(uintptr_t)UINT64_MAX; + pool->max_addr = 0; ring_ptr_init(&pool->ring->hdr); @@ -1868,8 +1915,7 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size, u ring_ptr_t *ring; uint32_t i, ring_mask, buf_index, head_offset; uint32_t num_populated; - uint8_t *data_ptr; - uint32_t hdr_size = sizeof(odp_packet_hdr_t); + uint8_t *data_ptr, *min_addr, *max_addr; void *uarea = NULL; if (pool_hdl == ODP_POOL_INVALID) { @@ -1884,6 +1930,9 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size, u return -1; } + min_addr = pool->base_addr; + max_addr = pool->max_addr; + if (buf_size != pool->ext_param.pkt.buf_size) { ODP_ERR("Bad buffer size\n"); return -1; @@ -1909,11 +1958,17 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size, u ring = &pool->ring->hdr; ring_mask = pool->ring_mask; buf_index = pool->num_populated; - head_offset = sizeof(odp_packet_hdr_t) + pool->ext_param.pkt.app_header_size; + head_offset = pool->ext_head_offset; for (i = 0; i < num; i++) { event_hdr = buf[i]; + if ((uint8_t *)event_hdr < min_addr) + min_addr = (uint8_t *)event_hdr; + + if ((uint8_t *)event_hdr > max_addr) + max_addr = (uint8_t *)event_hdr; + if ((uintptr_t)event_hdr & (ODP_CACHE_LINE_SIZE - 1)) { ODP_ERR("Bad packet buffer align: buf[%u]\n", i); return -1; @@ -1928,7 +1983,7 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size, u uarea = &pool->uarea_base_addr[buf_index * pool->uarea_size]; data_ptr = (uint8_t *)event_hdr + head_offset + pool->headroom; - init_event_hdr(pool, event_hdr, buf_index, hdr_size, data_ptr, uarea); + init_event_hdr(pool, event_hdr, buf_index, data_ptr, uarea); pool->ring->event_hdr_by_index[buf_index] = event_hdr; buf_index++; @@ -1936,6 +1991,11 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size, u } pool->num_populated += num; + pool->base_addr = min_addr; + pool->max_addr = max_addr; + + if (flags & ODP_POOL_POPULATE_DONE) + pool->max_addr = max_addr + buf_size - 1; return 0; } diff --git a/platform/linux-generic/odp_print.c b/platform/linux-generic/odp_print.c new file mode 100644 index 000000000..30a06c2f4 --- /dev/null +++ b/platform/linux-generic/odp_print.c @@ -0,0 +1,47 @@ +/* Copyright (c) 2022, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/hints.h> +#include <odp_print_internal.h> +#include <stdarg.h> +#include <stdio.h> + +/* Helps with snprintf() return value checking + * + * Otherwise like snprintf(), but returns always the number of characters + * printed (without the end mark) or zero on error. Terminates the string + * always with the end mark. */ +ODP_PRINTF_FORMAT(3, 4) +int _odp_snprint(char *str, size_t size, const char *format, ...) +{ + va_list args; + int len; + + /* No space to print new characters */ + if (size < 1) + return 0; + + if (size < 2) { + str[0] = 0; + return 0; + } + + va_start(args, format); + len = vsnprintf(str, size, format, args); + va_end(args); + + /* Error. Ensure that string has the end mark */ + if (len < 0) { + str[0] = 0; + return 0; + } + + /* Print would have been longer. Return the number of characters printed. */ + if (len >= (int)size) + return (int)size - 1; + + return len; +} diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 81065a8d5..112c392ec 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -37,6 +37,7 @@ #include <odp_global_data.h> #include <odp_event_internal.h> #include <odp_macros_internal.h> +#include <odp_print_internal.h> #include <string.h> @@ -52,6 +53,14 @@ /* Spread balancing frequency. Balance every BALANCE_ROUNDS_M1 + 1 scheduling rounds. */ #define BALANCE_ROUNDS_M1 0xfffff +/* Number of scheduled queue synchronization types */ +#define NUM_SCHED_SYNC 3 + +/* Queue types used as array indices */ +ODP_STATIC_ASSERT(ODP_SCHED_SYNC_PARALLEL == 0, "ODP_SCHED_SYNC_PARALLEL_value_changed"); +ODP_STATIC_ASSERT(ODP_SCHED_SYNC_ATOMIC == 1, "ODP_SCHED_SYNC_ATOMIC_value_changed"); +ODP_STATIC_ASSERT(ODP_SCHED_SYNC_ORDERED == 2, "ODP_SCHED_SYNC_ORDERED_value_changed"); + /* Load of a queue */ #define QUEUE_LOAD 256 @@ -223,8 +232,8 @@ typedef struct ODP_ALIGNED_CACHE { typedef struct { struct { - uint8_t burst_default[NUM_PRIO]; - uint8_t burst_max[NUM_PRIO]; + uint8_t burst_default[NUM_SCHED_SYNC][NUM_PRIO]; + uint8_t burst_max[NUM_SCHED_SYNC][NUM_PRIO]; uint8_t num_spread; uint8_t prefer_ratio; } config; @@ -300,11 +309,46 @@ static sched_global_t *sched; /* Thread local scheduler context */ static __thread sched_local_t sched_local; +static int read_burst_size_conf(uint8_t out_tbl[], const char *conf_str, + int min_val, int max_val, int print) +{ + int burst_val[NUM_PRIO]; + const int max_len = 256; + const int n = max_len - 1; + char line[max_len]; + int len = 0; + + if (_odp_libconfig_lookup_array(conf_str, burst_val, NUM_PRIO) != + NUM_PRIO) { + ODP_ERR("Config option '%s' not found.\n", conf_str); + return -1; + } + + char str[strlen(conf_str) + 4]; + + snprintf(str, sizeof(str), "%s[]:", conf_str); + len += snprintf(&line[len], n - len, " %-38s", str); + + for (int i = 0; i < NUM_PRIO; i++) { + int val = burst_val[i]; + + if (val > max_val || val < min_val) { + ODP_ERR("Bad value for %s: %i\n", conf_str, val); + return -1; + } + len += snprintf(&line[len], n - len, " %3i", val); + if (val > 0) + out_tbl[i] = val; + } + if (print) + ODP_PRINT("%s\n", line); + + return 0; +} + static int read_config_file(sched_global_t *sched) { const char *str; - int i; - int burst_val[NUM_PRIO]; int val = 0; ODP_PRINT("Scheduler config:\n"); @@ -355,46 +399,48 @@ static int read_config_file(sched_global_t *sched) if (val == 0 || sched->config.num_spread == 1) sched->load_balance = 0; + /* Initialize default values for all queue types */ str = "sched_basic.burst_size_default"; - if (_odp_libconfig_lookup_array(str, burst_val, NUM_PRIO) != - NUM_PRIO) { - ODP_ERR("Config option '%s' not found.\n", str); + if (read_burst_size_conf(sched->config.burst_default[ODP_SCHED_SYNC_ATOMIC], str, 1, + STASH_SIZE, 1) || + read_burst_size_conf(sched->config.burst_default[ODP_SCHED_SYNC_PARALLEL], str, 1, + STASH_SIZE, 0) || + read_burst_size_conf(sched->config.burst_default[ODP_SCHED_SYNC_ORDERED], str, 1, + STASH_SIZE, 0)) return -1; - } - ODP_PRINT(" %s[] =", str); - for (i = 0; i < NUM_PRIO; i++) { - val = burst_val[i]; - sched->config.burst_default[i] = val; - ODP_PRINT(" %3i", val); + str = "sched_basic.burst_size_max"; + if (read_burst_size_conf(sched->config.burst_max[ODP_SCHED_SYNC_ATOMIC], str, 1, + BURST_MAX, 1) || + read_burst_size_conf(sched->config.burst_max[ODP_SCHED_SYNC_PARALLEL], str, 1, + BURST_MAX, 0) || + read_burst_size_conf(sched->config.burst_max[ODP_SCHED_SYNC_ORDERED], str, 1, + BURST_MAX, 0)) + return -1; - if (val > STASH_SIZE || val < 1) { - ODP_ERR("Bad value %i\n", val); - return -1; - } - } - ODP_PRINT("\n"); + if (read_burst_size_conf(sched->config.burst_default[ODP_SCHED_SYNC_ATOMIC], + "sched_basic.burst_size_atomic", 0, STASH_SIZE, 1)) + return -1; - str = "sched_basic.burst_size_max"; - if (_odp_libconfig_lookup_array(str, burst_val, NUM_PRIO) != - NUM_PRIO) { - ODP_ERR("Config option '%s' not found.\n", str); + if (read_burst_size_conf(sched->config.burst_max[ODP_SCHED_SYNC_ATOMIC], + "sched_basic.burst_size_max_atomic", 0, BURST_MAX, 1)) + return -1; + + if (read_burst_size_conf(sched->config.burst_default[ODP_SCHED_SYNC_PARALLEL], + "sched_basic.burst_size_parallel", 0, STASH_SIZE, 1)) return -1; - } - ODP_PRINT(" %s[] = ", str); - for (i = 0; i < NUM_PRIO; i++) { - val = burst_val[i]; - sched->config.burst_max[i] = val; - ODP_PRINT(" %3i", val); + if (read_burst_size_conf(sched->config.burst_max[ODP_SCHED_SYNC_PARALLEL], + "sched_basic.burst_size_max_parallel", 0, BURST_MAX, 1)) + return -1; - if (val > BURST_MAX || val < 1) { - ODP_ERR("Bad value %i\n", val); - return -1; - } - } + if (read_burst_size_conf(sched->config.burst_default[ODP_SCHED_SYNC_ORDERED], + "sched_basic.burst_size_ordered", 0, STASH_SIZE, 1)) + return -1; - ODP_PRINT("\n"); + if (read_burst_size_conf(sched->config.burst_max[ODP_SCHED_SYNC_ORDERED], + "sched_basic.burst_size_max_ordered", 0, BURST_MAX, 1)) + return -1; str = "sched_basic.group_enable.all"; if (!_odp_libconfig_lookup_int(str, &val)) { @@ -1245,7 +1291,14 @@ static inline int schedule_grp_prio(odp_queue_t *out_queue, odp_event_t out_ev[] uint32_t qi; int num_spread = sched->config.num_spread; uint32_t ring_mask = sched->ring_mask; - uint16_t burst_def = sched->config.burst_default[prio]; + const uint32_t burst_def_sync[NUM_SCHED_SYNC] = { + sched->config.burst_default[ODP_SCHED_SYNC_PARALLEL][prio], + sched->config.burst_default[ODP_SCHED_SYNC_ATOMIC][prio], + sched->config.burst_default[ODP_SCHED_SYNC_ORDERED][prio]}; + const uint32_t burst_max_sync[NUM_SCHED_SYNC] = { + sched->config.burst_max[ODP_SCHED_SYNC_PARALLEL][prio], + sched->config.burst_max[ODP_SCHED_SYNC_ATOMIC][prio], + sched->config.burst_max[ODP_SCHED_SYNC_ORDERED][prio]}; /* Select the first spread based on weights */ spr = first_spr; @@ -1256,7 +1309,7 @@ static inline int schedule_grp_prio(odp_queue_t *out_queue, odp_event_t out_ev[] odp_queue_t handle; ring_u32_t *ring; int pktin; - uint16_t max_deq = burst_def; + uint32_t max_deq; int stashed = 1; odp_event_t *ev_tbl = sched_local.stash.ev; @@ -1282,16 +1335,16 @@ static inline int schedule_grp_prio(odp_queue_t *out_queue, odp_event_t out_ev[] sync_ctx = sched_sync_type(qi); ordered = (sync_ctx == ODP_SCHED_SYNC_ORDERED); + max_deq = burst_def_sync[sync_ctx]; /* When application's array is larger than default burst * size, output all events directly there. Also, ordered * queues are not stashed locally to improve * parallelism. Ordered context can only be released * when the local cache is empty. */ - if (max_num > burst_def || ordered) { - uint16_t burst_max; + if (max_num > max_deq || ordered) { + const uint32_t burst_max = burst_max_sync[sync_ctx]; - burst_max = sched->config.burst_max[prio]; stashed = 0; ev_tbl = out_ev; max_deq = max_num; @@ -2035,12 +2088,14 @@ static int schedule_capability(odp_schedule_capability_t *capa) static void schedule_print(void) { - int spr, prio, grp; + int spr, prio, grp, pos; uint32_t num_queues, num_active; ring_u32_t *ring; odp_schedule_capability_t capa; int num_spread = sched->config.num_spread; const int col_width = 24; + const int size = 512; + char str[size]; (void)schedule_capability(&capa); @@ -2053,42 +2108,42 @@ static void schedule_print(void) ODP_PRINT(" prefer ratio: %u\n", sched->config.prefer_ratio); ODP_PRINT("\n"); - ODP_PRINT(" Number of active event queues:\n"); - ODP_PRINT(" spread\n"); - ODP_PRINT(" "); + pos = 0; + pos += _odp_snprint(&str[pos], size - pos, " Number of active event queues:\n"); + pos += _odp_snprint(&str[pos], size - pos, " spread\n"); + pos += _odp_snprint(&str[pos], size - pos, " "); for (spr = 0; spr < num_spread; spr++) - ODP_PRINT(" %7i", spr); + pos += _odp_snprint(&str[pos], size - pos, " %7i", spr); - ODP_PRINT("\n"); + ODP_PRINT("%s\n", str); for (prio = 0; prio < NUM_PRIO; prio++) { - ODP_PRINT(" prio %i", prio); - for (grp = 0; grp < NUM_SCHED_GRPS; grp++) if (sched->prio_q_mask[grp][prio]) break; - if (grp == NUM_SCHED_GRPS) { - ODP_PRINT(":-\n"); + if (grp == NUM_SCHED_GRPS) continue; - } - ODP_PRINT("\n"); + ODP_PRINT(" prio: %i\n", prio); for (grp = 0; grp < NUM_SCHED_GRPS; grp++) { if (sched->sched_grp[grp].allocated == 0) continue; - ODP_PRINT(" group %i:", grp); + pos = 0; + pos += _odp_snprint(&str[pos], size - pos, " group %i:", grp); for (spr = 0; spr < num_spread; spr++) { num_queues = sched->prio_q_count[grp][prio][spr]; ring = &sched->prio_q[grp][prio][spr].ring; num_active = ring_u32_len(ring); - ODP_PRINT(" %3u/%3u", num_active, num_queues); + pos += _odp_snprint(&str[pos], size - pos, " %3u/%3u", + num_active, num_queues); } - ODP_PRINT("\n"); + + ODP_PRINT("%s\n", str); } } @@ -2099,12 +2154,15 @@ static void schedule_print(void) if (sched->sched_grp[grp].allocated == 0) continue; - ODP_PRINT(" group %i: %-*s", grp, col_width, sched->sched_grp[grp].name); + pos = 0; + pos += _odp_snprint(&str[pos], size - pos, " group %i: %-*s", grp, col_width, + sched->sched_grp[grp].name); for (spr = 0; spr < num_spread; spr++) - ODP_PRINT(" %u", sched->sched_grp[grp].spread_thrs[spr]); + pos += _odp_snprint(&str[pos], size - pos, " %u", + sched->sched_grp[grp].spread_thrs[spr]); - ODP_PRINT("\n"); + ODP_PRINT("%s\n", str); } ODP_PRINT("\n"); diff --git a/platform/linux-generic/odp_stash.c b/platform/linux-generic/odp_stash.c index 1bbbc8d8b..8fe9c1096 100644 --- a/platform/linux-generic/odp_stash.c +++ b/platform/linux-generic/odp_stash.c @@ -1,12 +1,14 @@ -/* Copyright (c) 2020-2021, Nokia +/* Copyright (c) 2020-2022, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -#include <odp/api/ticketlock.h> #include <odp/api/shared_memory.h> #include <odp/api/stash.h> +#include <odp/api/std_types.h> +#include <odp/api/ticketlock.h> + #include <odp/api/plat/strong_types.h> #include <odp_config_internal.h> @@ -118,6 +120,8 @@ int odp_stash_capability(odp_stash_capability_t *capa, odp_stash_type_t type) capa->max_stashes = CONFIG_MAX_STASHES - CONFIG_INTERNAL_STASHES; capa->max_num_obj = MAX_RING_SIZE; capa->max_obj_size = sizeof(uint64_t); + capa->max_get_batch = MIN_RING_SIZE; + capa->max_put_batch = MIN_RING_SIZE; capa->stats.bit.count = 1; return 0; @@ -311,7 +315,7 @@ odp_stash_t odp_stash_lookup(const char *name) return ODP_STASH_INVALID; } -int32_t odp_stash_put(odp_stash_t st, const void *obj, int32_t num) +static inline int32_t stash_put(odp_stash_t st, const void *obj, int32_t num) { stash_t *stash; uint32_t obj_size; @@ -367,7 +371,19 @@ int32_t odp_stash_put(odp_stash_t st, const void *obj, int32_t num) return -1; } -int32_t odp_stash_put_u32(odp_stash_t st, const uint32_t val[], int32_t num) +int32_t odp_stash_put(odp_stash_t st, const void *obj, int32_t num) +{ + return stash_put(st, obj, num); +} + +int32_t odp_stash_put_batch(odp_stash_t st, const void *obj, int32_t num) +{ + /* Returns always 'num', or -1 on failure. */ + return stash_put(st, obj, num); +} + +static inline int32_t stash_put_u32(odp_stash_t st, const uint32_t val[], + int32_t num) { stash_t *stash = (stash_t *)(uintptr_t)st; @@ -381,7 +397,20 @@ int32_t odp_stash_put_u32(odp_stash_t st, const uint32_t val[], int32_t num) return num; } -int32_t odp_stash_put_u64(odp_stash_t st, const uint64_t val[], int32_t num) +int32_t odp_stash_put_u32(odp_stash_t st, const uint32_t val[], int32_t num) +{ + return stash_put_u32(st, val, num); +} + +int32_t odp_stash_put_u32_batch(odp_stash_t st, const uint32_t val[], + int32_t num) +{ + /* Returns always 'num', or -1 on failure. */ + return stash_put_u32(st, val, num); +} + +static inline int32_t stash_put_u64(odp_stash_t st, const uint64_t val[], + int32_t num) { stash_t *stash = (stash_t *)(uintptr_t)st; @@ -395,7 +424,20 @@ int32_t odp_stash_put_u64(odp_stash_t st, const uint64_t val[], int32_t num) return num; } -int32_t odp_stash_put_ptr(odp_stash_t st, const uintptr_t ptr[], int32_t num) +int32_t odp_stash_put_u64(odp_stash_t st, const uint64_t val[], int32_t num) +{ + return stash_put_u64(st, val, num); +} + +int32_t odp_stash_put_u64_batch(odp_stash_t st, const uint64_t val[], + int32_t num) +{ + /* Returns always 'num', or -1 on failure. */ + return stash_put_u64(st, val, num); +} + +static inline int32_t stash_put_ptr(odp_stash_t st, const uintptr_t ptr[], + int32_t num) { stash_t *stash = (stash_t *)(uintptr_t)st; @@ -416,7 +458,19 @@ int32_t odp_stash_put_ptr(odp_stash_t st, const uintptr_t ptr[], int32_t num) return num; } -int32_t odp_stash_get(odp_stash_t st, void *obj, int32_t num) +int32_t odp_stash_put_ptr(odp_stash_t st, const uintptr_t ptr[], int32_t num) +{ + return stash_put_ptr(st, ptr, num); +} + +int32_t odp_stash_put_ptr_batch(odp_stash_t st, const uintptr_t ptr[], + int32_t num) +{ + /* Returns always 'num', or -1 on failure. */ + return stash_put_ptr(st, ptr, num); +} + +static inline int32_t stash_get(odp_stash_t st, void *obj, int32_t num, odp_bool_t batch) { stash_t *stash; uint32_t obj_size; @@ -432,13 +486,19 @@ int32_t odp_stash_get(odp_stash_t st, void *obj, int32_t num) if (obj_size == sizeof(uint64_t)) { ring_u64_t *ring_u64 = &stash->ring_u64.hdr; - return ring_u64_deq_multi(ring_u64, stash->ring_mask, obj, num); + if (batch) + return ring_u64_deq_batch(ring_u64, stash->ring_mask, obj, num); + else + return ring_u64_deq_multi(ring_u64, stash->ring_mask, obj, num); } if (obj_size == sizeof(uint32_t)) { ring_u32_t *ring_u32 = &stash->ring_u32.hdr; - return ring_u32_deq_multi(ring_u32, stash->ring_mask, obj, num); + if (batch) + return ring_u32_deq_batch(ring_u32, stash->ring_mask, obj, num); + else + return ring_u32_deq_multi(ring_u32, stash->ring_mask, obj, num); } if (obj_size == sizeof(uint16_t)) { @@ -446,8 +506,10 @@ int32_t odp_stash_get(odp_stash_t st, void *obj, int32_t num) ring_u32_t *ring_u32 = &stash->ring_u32.hdr; uint32_t u32[num]; - num_deq = ring_u32_deq_multi(ring_u32, stash->ring_mask, - u32, num); + if (batch) + num_deq = ring_u32_deq_batch(ring_u32, stash->ring_mask, u32, num); + else + num_deq = ring_u32_deq_multi(ring_u32, stash->ring_mask, u32, num); for (i = 0; i < num_deq; i++) u16_ptr[i] = u32[i]; @@ -460,8 +522,10 @@ int32_t odp_stash_get(odp_stash_t st, void *obj, int32_t num) ring_u32_t *ring_u32 = &stash->ring_u32.hdr; uint32_t u32[num]; - num_deq = ring_u32_deq_multi(ring_u32, stash->ring_mask, - u32, num); + if (batch) + num_deq = ring_u32_deq_batch(ring_u32, stash->ring_mask, u32, num); + else + num_deq = ring_u32_deq_multi(ring_u32, stash->ring_mask, u32, num); for (i = 0; i < num_deq; i++) u8_ptr[i] = u32[i]; @@ -472,6 +536,16 @@ int32_t odp_stash_get(odp_stash_t st, void *obj, int32_t num) return -1; } +int32_t odp_stash_get(odp_stash_t st, void *obj, int32_t num) +{ + return stash_get(st, obj, num, 0); +} + +int32_t odp_stash_get_batch(odp_stash_t st, void *obj, int32_t num) +{ + return stash_get(st, obj, num, 1); +} + int32_t odp_stash_get_u32(odp_stash_t st, uint32_t val[], int32_t num) { stash_t *stash = (stash_t *)(uintptr_t)st; @@ -485,6 +559,18 @@ int32_t odp_stash_get_u32(odp_stash_t st, uint32_t val[], int32_t num) num); } +int32_t odp_stash_get_u32_batch(odp_stash_t st, uint32_t val[], int32_t num) +{ + stash_t *stash = (stash_t *)(uintptr_t)st; + + if (odp_unlikely(st == ODP_STASH_INVALID)) + return -1; + + ODP_ASSERT(stash->obj_size == sizeof(uint32_t)); + + return ring_u32_deq_batch(&stash->ring_u32.hdr, stash->ring_mask, val, num); +} + int32_t odp_stash_get_u64(odp_stash_t st, uint64_t val[], int32_t num) { stash_t *stash = (stash_t *)(uintptr_t)st; @@ -498,6 +584,18 @@ int32_t odp_stash_get_u64(odp_stash_t st, uint64_t val[], int32_t num) num); } +int32_t odp_stash_get_u64_batch(odp_stash_t st, uint64_t val[], int32_t num) +{ + stash_t *stash = (stash_t *)(uintptr_t)st; + + if (odp_unlikely(st == ODP_STASH_INVALID)) + return -1; + + ODP_ASSERT(stash->obj_size == sizeof(uint64_t)); + + return ring_u64_deq_batch(&stash->ring_u64.hdr, stash->ring_mask, val, num); +} + int32_t odp_stash_get_ptr(odp_stash_t st, uintptr_t ptr[], int32_t num) { stash_t *stash = (stash_t *)(uintptr_t)st; @@ -518,6 +616,24 @@ int32_t odp_stash_get_ptr(odp_stash_t st, uintptr_t ptr[], int32_t num) return -1; } +int32_t odp_stash_get_ptr_batch(odp_stash_t st, uintptr_t ptr[], int32_t num) +{ + stash_t *stash = (stash_t *)(uintptr_t)st; + + if (odp_unlikely(st == ODP_STASH_INVALID)) + return -1; + + ODP_ASSERT(stash->obj_size == sizeof(uintptr_t)); + + if (sizeof(uintptr_t) == sizeof(uint32_t)) + return ring_u32_deq_batch(&stash->ring_u32.hdr, stash->ring_mask, + (uint32_t *)(uintptr_t)ptr, num); + else if (sizeof(uintptr_t) == sizeof(uint64_t)) + return ring_u64_deq_batch(&stash->ring_u64.hdr, stash->ring_mask, + (uint64_t *)(uintptr_t)ptr, num); + return -1; +} + int odp_stash_flush_cache(odp_stash_t st) { if (odp_unlikely(st == ODP_STASH_INVALID)) diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index c122df537..c0ca12ae0 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -112,29 +112,37 @@ const _odp_timeout_inline_offset_t _odp_timeout_inline_offset ODP_ALIGNED_CACHE = { .expiration = offsetof(odp_timeout_hdr_t, expiration), .timer = offsetof(odp_timeout_hdr_t, timer), - .user_ptr = offsetof(odp_timeout_hdr_t, user_ptr) + .user_ptr = offsetof(odp_timeout_hdr_t, user_ptr), + .uarea_addr = offsetof(odp_timeout_hdr_t, uarea_addr), }; #include <odp/visibility_end.h> -typedef struct +typedef union #if USE_128BIT_ATOMICS ODP_ALIGNED(16) /* 16-byte atomic operations need properly aligned addresses */ #endif tick_buf_s { - /* Expiration tick or TMO_xxx */ - odp_atomic_u64_t exp_tck; - union { - /* ODP_EVENT_INVALID if timer not active */ - odp_event_t tmo_event; - - /* Ensures that tick_buf_t is 128 bits */ - uint64_t tmo_u64; - }; +#if USE_128BIT_ATOMICS + odp_atomic_u128_t tb_atomic_u128; + + odp_u128_t tb_u128; +#endif + + struct { + /* Expiration tick or TMO_xxx */ + odp_atomic_u64_t exp_tck; + union { + /* ODP_EVENT_INVALID if timer not active */ + odp_event_t tmo_event; + /* Ensures that tick_buf_t is 128 bits */ + uint64_t tmo_u64; + }; + }; } tick_buf_t; -#ifndef ODP_ATOMIC_U64_LOCK +#if USE_128BIT_ATOMICS ODP_STATIC_ASSERT(sizeof(tick_buf_t) == 16, "sizeof(tick_buf_t) == 16"); #endif @@ -683,11 +691,11 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event, new.tmo_u64 = 0; old.tmo_u64 = 0; - do { - /* Relaxed and non-atomic read of current values */ - old.exp_tck.v = tb->exp_tck.v; - old.tmo_event = tb->tmo_event; + /* Relaxed and non-atomic read of current values */ + old.exp_tck.v = tb->exp_tck.v; + old.tmo_event = tb->tmo_event; + do { /* Check if there actually is a timeout event * present */ if (old.tmo_event == ODP_EVENT_INVALID) { @@ -702,9 +710,8 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event, /* Atomic CAS will fail if we experienced torn reads, * retry update sequence until CAS succeeds */ - } while (!_odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb, - (_odp_u128_t *)&old, (_odp_u128_t *)&new, - _ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX)); + } while (!odp_atomic_cas_rel_u128(&tb->tb_atomic_u128, + &old.tb_u128, new.tb_u128)); #else /* Take a related lock */ while (_odp_atomic_flag_tas(IDX2LOCK(tp, idx))) @@ -830,11 +837,11 @@ static odp_event_t timer_cancel(timer_pool_t *tp, uint32_t idx) new.tmo_u64 = 0; old.tmo_u64 = 0; - do { - /* Relaxed and non-atomic read of current values */ - old.exp_tck.v = tb->exp_tck.v; - old.tmo_event = tb->tmo_event; + /* Relaxed and non-atomic read of current values */ + old.exp_tck.v = tb->exp_tck.v; + old.tmo_event = tb->tmo_event; + do { /* Check if it is not expired already */ if (old.exp_tck.v & TMO_INACTIVE) { old.tmo_event = ODP_EVENT_INVALID; @@ -847,11 +854,9 @@ static odp_event_t timer_cancel(timer_pool_t *tp, uint32_t idx) /* Atomic CAS will fail if we experienced torn reads, * retry update sequence until CAS succeeds */ - } while (!_odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb, - (_odp_u128_t *)&old, - (_odp_u128_t *)&new, - _ODP_MEMMODEL_RLS, - _ODP_MEMMODEL_RLX)); + } while (!odp_atomic_cas_rel_u128(&tb->tb_atomic_u128, &old.tb_u128, + new.tb_u128)); + old_event = old.tmo_event; #else /* Take a related lock */ @@ -905,9 +910,8 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) new.exp_tck.v = exp_tck | TMO_INACTIVE; new.tmo_event = ODP_EVENT_INVALID; - int succ = _odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb, - (_odp_u128_t *)&old, (_odp_u128_t *)&new, - _ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX); + int succ = odp_atomic_cas_rel_u128(&tb->tb_atomic_u128, + &old.tb_u128, new.tb_u128); if (succ) tmo_event = old.tmo_event; /* Else CAS failed, something changed => skip timer diff --git a/platform/linux-generic/test/inline-timer.conf b/platform/linux-generic/test/inline-timer.conf index c7379e38a..261aa0141 100644 --- a/platform/linux-generic/test/inline-timer.conf +++ b/platform/linux-generic/test/inline-timer.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.21" +config_file_version = "0.1.22" timer: { # Enable inline timer implementation diff --git a/platform/linux-generic/test/packet_align.conf b/platform/linux-generic/test/packet_align.conf index 433899017..8d2d00e63 100644 --- a/platform/linux-generic/test/packet_align.conf +++ b/platform/linux-generic/test/packet_align.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.21" +config_file_version = "0.1.22" pool: { pkt: { diff --git a/platform/linux-generic/test/process-mode.conf b/platform/linux-generic/test/process-mode.conf index a36c9fc3d..1e0e7cc95 100644 --- a/platform/linux-generic/test/process-mode.conf +++ b/platform/linux-generic/test/process-mode.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.21" +config_file_version = "0.1.22" # Shared memory options shm: { diff --git a/platform/linux-generic/test/sched-basic.conf b/platform/linux-generic/test/sched-basic.conf index 16654595b..e63ffa2f3 100644 --- a/platform/linux-generic/test/sched-basic.conf +++ b/platform/linux-generic/test/sched-basic.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.21" +config_file_version = "0.1.22" # Test scheduler with an odd spread value and without dynamic load balance sched_basic: { diff --git a/platform/linux-generic/test/validation/api/shmem/Makefile.am b/platform/linux-generic/test/validation/api/shmem/Makefile.am index 07d311d2f..309eceb92 100644 --- a/platform/linux-generic/test/validation/api/shmem/Makefile.am +++ b/platform/linux-generic/test/validation/api/shmem/Makefile.am @@ -5,8 +5,7 @@ test_PROGRAMS = shmem_linux shmem_odp1 shmem_odp2 #shmem_linux is stand alone, pure linux (no ODP): shmem_linux_SOURCES = shmem_linux.c shmem_linux.h shmem_common.h -shmem_linux_CFLAGS = $(AM_CFLAGS) -I$(top_builddir)/include -shmem_linux_LDFLAGS = $(AM_LDFLAGS) -lrt +shmem_linux_LDFLAGS = shmem_linux_LDADD = #shmem_odp1 and shmem_odp2 are the 2 ODP processes: |