diff options
Diffstat (limited to 'platform/linux-generic/odp_packet.c')
| -rw-r--r-- | platform/linux-generic/odp_packet.c | 2991 |
1 files changed, 1483 insertions, 1508 deletions
diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index f18bd4dd5..6559ba606 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -1,18 +1,39 @@ -/* Copyright (c) 2013, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2013-2018 Linaro Limited + * Copyright (c) 2019-2023 Nokia */ -#include <odp/api/plat/packet_inlines.h> +#include <odp/autoheader_external.h> + +#include <odp/api/byteorder.h> +#include <odp/api/hash.h> +#include <odp/api/hints.h> #include <odp/api/packet.h> -#include <odp_packet_internal.h> +#include <odp/api/packet_flags.h> +#include <odp/api/packet_io.h> +#include <odp/api/proto_stats.h> +#include <odp/api/timer.h> + +#include <odp_parse_internal.h> +#include <odp_chksum_internal.h> #include <odp_debug_internal.h> -#include <odp/api/hints.h> -#include <odp/api/byteorder.h> +#include <odp_event_internal.h> +#include <odp_event_validation_internal.h> +#include <odp_macros_internal.h> +#include <odp_packet_internal.h> +#include <odp_packet_io_internal.h> +#include <odp_pool_internal.h> +#include <odp_string_internal.h> + +/* Inlined API functions */ +#include <odp/api/plat/byteorder_inlines.h> +#include <odp/api/plat/event_inlines.h> +#include <odp/api/plat/packet_inlines.h> +#include <odp/api/plat/packet_io_inlines.h> #include <protocols/eth.h> #include <protocols/ip.h> +#include <protocols/sctp.h> #include <protocols/tcp.h> #include <protocols/udp.h> @@ -21,224 +42,152 @@ #include <stdio.h> #include <inttypes.h> -/* Initial packet segment data length */ -#define BASE_LEN CONFIG_PACKET_MAX_SEG_LEN - #include <odp/visibility_begin.h> /* Fill in packet header field offsets for inline functions */ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { - .data = offsetof(odp_packet_hdr_t, buf_hdr.seg[0].data), - .seg_len = offsetof(odp_packet_hdr_t, buf_hdr.seg[0].len), + .seg_data = offsetof(odp_packet_hdr_t, seg_data), + .seg_len = offsetof(odp_packet_hdr_t, seg_len), + .seg_next = offsetof(odp_packet_hdr_t, seg_next), .frame_len = offsetof(odp_packet_hdr_t, frame_len), .headroom = offsetof(odp_packet_hdr_t, headroom), .tailroom = offsetof(odp_packet_hdr_t, tailroom), - .unshared_len = offsetof(odp_packet_hdr_t, unshared_len), - .ref_hdr = offsetof(odp_packet_hdr_t, ref_hdr), - .ref_offset = offsetof(odp_packet_hdr_t, ref_offset), - .ref_len = offsetof(odp_packet_hdr_t, ref_len), - .pool = offsetof(odp_packet_hdr_t, buf_hdr.pool_hdl), + .pool = offsetof(odp_packet_hdr_t, event_hdr.pool), .input = offsetof(odp_packet_hdr_t, input), - .segcount = offsetof(odp_packet_hdr_t, buf_hdr.segcount), - .user_ptr = offsetof(odp_packet_hdr_t, buf_hdr.buf_ctx), - .user_area = offsetof(odp_packet_hdr_t, buf_hdr.uarea_addr), - .user_area_size = offsetof(odp_packet_hdr_t, buf_hdr.uarea_size), + .seg_count = offsetof(odp_packet_hdr_t, seg_count), + .user_ptr = offsetof(odp_packet_hdr_t, user_ptr), + .user_area = offsetof(odp_packet_hdr_t, uarea_addr), + .l2_offset = offsetof(odp_packet_hdr_t, p.l2_offset), + .l3_offset = offsetof(odp_packet_hdr_t, p.l3_offset), + .l4_offset = offsetof(odp_packet_hdr_t, p.l4_offset), .flow_hash = offsetof(odp_packet_hdr_t, flow_hash), .timestamp = offsetof(odp_packet_hdr_t, timestamp), - .input_flags = offsetof(odp_packet_hdr_t, p.input_flags) - + .input_flags = offsetof(odp_packet_hdr_t, p.input_flags), + .flags = offsetof(odp_packet_hdr_t, p.flags), + .cls_mark = offsetof(odp_packet_hdr_t, cls_mark), + .ipsec_ctx = offsetof(odp_packet_hdr_t, ipsec_ctx), + .crypto_op = offsetof(odp_packet_hdr_t, crypto_op_result), }; #include <odp/visibility_end.h> -static inline odp_packet_hdr_t *packet_hdr(odp_packet_t pkt) -{ - return (odp_packet_hdr_t *)(uintptr_t)pkt; -} - -static inline odp_buffer_t buffer_handle(odp_packet_hdr_t *pkt_hdr) -{ - return (odp_buffer_t)pkt_hdr; -} +/* Check that invalid values are the same. Some versions of Clang and pedantic + * build have trouble with the strong type casting, and complain that these + * invalid values are not integral constants. + * + * Invalid values are required to be equal for _odp_buffer_is_valid() to work + * properly. */ +#ifndef __clang__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +ODP_STATIC_ASSERT(ODP_PACKET_INVALID == 0, "Packet invalid not 0"); +ODP_STATIC_ASSERT(ODP_BUFFER_INVALID == 0, "Buffer invalid not 0"); +ODP_STATIC_ASSERT(ODP_EVENT_INVALID == 0, "Event invalid not 0"); +ODP_STATIC_ASSERT(ODP_PACKET_VECTOR_INVALID == 0, "Packet vector invalid not 0"); +ODP_STATIC_ASSERT(ODP_PACKET_TX_COMPL_INVALID == 0, "Packet TX completion invalid not 0"); +ODP_STATIC_ASSERT(ODP_TIMEOUT_INVALID == 0, "Timeout invalid not 0"); +#pragma GCC diagnostic pop +#endif -static inline void packet_ref_inc(odp_packet_hdr_t *pkt_hdr) +static inline odp_packet_hdr_t *packet_seg_to_hdr(odp_packet_seg_t seg) { - odp_atomic_inc_u32(&pkt_hdr->ref_count); + return (odp_packet_hdr_t *)(uintptr_t)seg; } -static inline uint32_t packet_ref_dec(odp_packet_hdr_t *pkt_hdr) +static inline odp_packet_seg_t packet_hdr_to_seg(odp_packet_hdr_t *pkt_hdr) { - return odp_atomic_fetch_dec_u32(&pkt_hdr->ref_count); + return (odp_packet_seg_t)pkt_hdr; } -static inline odp_packet_hdr_t *buf_to_packet_hdr(odp_buffer_t buf) +/* + * Return pointer to the current segment and step cur_hdr forward. + */ +static inline odp_packet_hdr_t *packet_seg_step(odp_packet_hdr_t **cur_hdr) { - return (odp_packet_hdr_t *)buf_hdl_to_hdr(buf); -} + odp_packet_hdr_t *hdr = *cur_hdr; -odp_packet_t _odp_packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr) -{ - return (odp_packet_t)buf_hdr; -} + *cur_hdr = hdr->seg_next; -static inline odp_buffer_t packet_to_buffer(odp_packet_t pkt) -{ - return (odp_buffer_t)pkt; + return hdr; } -static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, - uint32_t seg_idx) +static inline void packet_seg_find_idx(odp_packet_hdr_t **pkt_hdr, + uint32_t find_idx) { - return pkt_hdr->buf_hdr.seg[seg_idx].len; -} + odp_packet_hdr_t *hdr = *pkt_hdr; + uint32_t idx = 0; -static inline uint8_t *packet_seg_data(odp_packet_hdr_t *pkt_hdr, - uint32_t seg_idx) -{ - return pkt_hdr->buf_hdr.seg[seg_idx].data; -} + while (odp_unlikely(idx < find_idx)) { + idx++; + hdr = hdr->seg_next; + } -static inline uint32_t packet_last_seg(odp_packet_hdr_t *pkt_hdr) -{ - if (CONFIG_PACKET_MAX_SEGS == 1) - return 0; - else - return pkt_hdr->buf_hdr.segcount - 1; + *pkt_hdr = hdr; } -static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr) -{ - return packet_seg_len(pkt_hdr, 0); -} - -static inline uint32_t packet_last_seg_len(odp_packet_hdr_t *pkt_hdr) +static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, + uint32_t seg_idx) { - int last = packet_last_seg(pkt_hdr); + packet_seg_find_idx(&pkt_hdr, seg_idx); - return packet_seg_len(pkt_hdr, last); -} - -static inline void *packet_data(odp_packet_hdr_t *pkt_hdr) -{ - return pkt_hdr->buf_hdr.seg[0].data; + return pkt_hdr->seg_len; } static inline void *packet_tail(odp_packet_hdr_t *pkt_hdr) { - int last = packet_last_seg(pkt_hdr); - uint32_t seg_len = pkt_hdr->buf_hdr.seg[last].len; + odp_packet_hdr_t *last_seg = packet_last_seg(pkt_hdr); - return pkt_hdr->buf_hdr.seg[last].data + seg_len; + return last_seg->seg_data + last_seg->seg_len; } -static inline uint32_t seg_headroom(odp_packet_hdr_t *pkt_hdr, int seg) +static inline uint32_t seg_headroom(odp_packet_hdr_t *pkt_seg) { - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[seg].hdr; + _odp_event_hdr_t *hdr = &pkt_seg->event_hdr; + pool_t *pool = _odp_pool_entry(hdr->pool); uint8_t *base = hdr->base_data; - uint8_t *head = pkt_hdr->buf_hdr.seg[seg].data; + uint8_t *head = pkt_seg->seg_data; - return CONFIG_PACKET_HEADROOM + (head - base); + return pool->headroom + (head - base); } -static inline uint32_t seg_tailroom(odp_packet_hdr_t *pkt_hdr, int seg) +static inline uint32_t seg_tailroom(odp_packet_hdr_t *pkt_seg) { - uint32_t seg_len = pkt_hdr->buf_hdr.seg[seg].len; - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[seg].hdr; - uint8_t *tail = pkt_hdr->buf_hdr.seg[seg].data + seg_len; + _odp_event_hdr_t *hdr = &pkt_seg->event_hdr; + uint8_t *tail = pkt_seg->seg_data + pkt_seg->seg_len; return hdr->buf_end - tail; } -static inline void push_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) -{ - pkt_hdr->headroom -= len; - pkt_hdr->frame_len += len; - pkt_hdr->unshared_len += len; - pkt_hdr->buf_hdr.seg[0].data -= len; - pkt_hdr->buf_hdr.seg[0].len += len; -} - -static inline void pull_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) -{ - pkt_hdr->headroom += len; - pkt_hdr->frame_len -= len; - pkt_hdr->unshared_len -= len; - pkt_hdr->buf_hdr.seg[0].data += len; - pkt_hdr->buf_hdr.seg[0].len -= len; -} - static inline void push_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) { - int last = packet_last_seg(pkt_hdr); + odp_packet_hdr_t *last_seg = packet_last_seg(pkt_hdr); pkt_hdr->tailroom -= len; pkt_hdr->frame_len += len; - pkt_hdr->unshared_len += len; - pkt_hdr->buf_hdr.seg[last].len += len; -} - -/* Copy all metadata for segmentation modification. Segment data and lengths - * are not copied. */ -static inline void packet_seg_copy_md(odp_packet_hdr_t *dst, - odp_packet_hdr_t *src) -{ - dst->p = src->p; - - /* lengths are not copied: - * .frame_len - * .headroom - * .tailroom - */ - - dst->input = src->input; - dst->dst_queue = src->dst_queue; - dst->flow_hash = src->flow_hash; - dst->timestamp = src->timestamp; - dst->op_result = src->op_result; - - /* buffer header side packet metadata */ - dst->buf_hdr.buf_u64 = src->buf_hdr.buf_u64; - dst->buf_hdr.uarea_addr = src->buf_hdr.uarea_addr; - dst->buf_hdr.uarea_size = src->buf_hdr.uarea_size; - - /* reference related metadata */ - dst->ref_len = src->ref_len; - dst->unshared_len = src->unshared_len; - - /* segmentation data is not copied: - * buf_hdr.seg[] - * buf_hdr.segcount - */ + last_seg->seg_len += len; } -static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, - uint32_t offset, uint32_t *seg_len, int *seg_idx) +static inline void *packet_map(void *pkt_ptr, uint32_t offset, + uint32_t *seg_len, odp_packet_seg_t *seg) { void *addr; uint32_t len; - int seg = 0; - int seg_count = pkt_hdr->buf_hdr.segcount; - - /* Special processing for references */ - while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { - offset -= (pkt_hdr->frame_len - pkt_hdr->ref_offset); - offset += (pkt_hdr->ref_hdr->frame_len - pkt_hdr->ref_len); - pkt_hdr = pkt_hdr->ref_hdr; - seg_count = pkt_hdr->buf_hdr.segcount; - } + odp_packet_hdr_t *pkt_hdr = pkt_ptr; + int seg_count = pkt_hdr->seg_count; - if (odp_unlikely(offset > pkt_hdr->frame_len)) + if (odp_unlikely(offset >= pkt_hdr->frame_len)) return NULL; - if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || seg_count == 1)) { - addr = pkt_hdr->buf_hdr.seg[0].data + offset; - len = pkt_hdr->buf_hdr.seg[0].len - offset; + if (odp_likely(seg_count == 1)) { + addr = pkt_hdr->seg_data + offset; + len = pkt_hdr->seg_len - offset; } else { - int i; + odp_packet_hdr_t *next_hdr = pkt_hdr; uint32_t seg_start = 0, seg_end = 0; - for (i = 0; i < seg_count; i++) { - seg_end += pkt_hdr->buf_hdr.seg[i].len; + while (next_hdr != NULL) { + pkt_hdr = packet_seg_step(&next_hdr); + seg_end += pkt_hdr->seg_len; if (odp_likely(offset < seg_end)) break; @@ -246,94 +195,168 @@ static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, seg_start = seg_end; } - addr = pkt_hdr->buf_hdr.seg[i].data + (offset - seg_start); - len = pkt_hdr->buf_hdr.seg[i].len - (offset - seg_start); - seg = i; + addr = pkt_hdr->seg_data + (offset - seg_start); + len = pkt_hdr->seg_len - (offset - seg_start); } if (seg_len) *seg_len = len; - if (seg_idx) - *seg_idx = seg; + if (seg) + *seg = packet_hdr_to_seg(pkt_hdr); return addr; } -void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) +#include <odp/visibility_begin.h> + +/* This file uses the inlined version directly. Inlined API calls use this when + * offset does not point to the first segment. */ +void *_odp_packet_map(void *pkt_ptr, uint32_t offset, uint32_t *seg_len, + odp_packet_seg_t *seg) +{ + return packet_map(pkt_ptr, offset, seg_len, seg); +} + +int _odp_packet_copy_from_mem_seg(odp_packet_t pkt, uint32_t offset, + uint32_t len, const void *src) { - /* Reset parser metadata before new parse */ - pkt_hdr->p.error_flags.all = 0; - pkt_hdr->p.input_flags.all = 0; - pkt_hdr->p.output_flags.all = 0; - pkt_hdr->p.l2_offset = 0; - pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; - pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; + void *mapaddr; + uint32_t seglen = 0; /* GCC */ + uint32_t cpylen; + const uint8_t *srcaddr = (const uint8_t *)src; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + if (offset + len > pkt_hdr->frame_len) + return -1; - /* Ensure dummy pkt hdrs used in I/O recv classification are valid */ - pkt_hdr->ref_hdr = NULL; + while (len > 0) { + mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); + cpylen = len > seglen ? seglen : len; + memcpy(mapaddr, srcaddr, cpylen); + offset += cpylen; + srcaddr += cpylen; + len -= cpylen; + } + + return 0; +} + +int _odp_packet_copy_to_mem_seg(odp_packet_t pkt, uint32_t offset, + uint32_t len, void *dst) +{ + void *mapaddr; + uint32_t seglen = 0; /* GCC */ + uint32_t cpylen; + uint8_t *dstaddr = (uint8_t *)dst; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + if (offset + len > pkt_hdr->frame_len) + return -1; + + while (len > 0) { + mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); + cpylen = len > seglen ? seglen : len; + memcpy(dstaddr, mapaddr, cpylen); + offset += cpylen; + dstaddr += cpylen; + len -= cpylen; + } + + return 0; +} + +#include <odp/visibility_end.h> + +static inline void link_segments(odp_packet_hdr_t *pkt_hdr[], int num) +{ + int cur = 0; + odp_packet_hdr_t *hdr; + odp_packet_hdr_t *head = pkt_hdr[0]; + uint32_t seg_len = _odp_pool_entry(head->event_hdr.pool)->seg_len; + + while (1) { + _odp_event_hdr_t *event_hdr = &pkt_hdr[cur]->event_hdr; + + hdr = pkt_hdr[cur]; + hdr->seg_data = event_hdr->base_data; + hdr->seg_len = seg_len; + + /* init_segments() handles first seg ref_cnt init */ + if (ODP_DEBUG == 1 && cur > 0) { + uint32_t prev_ref; + odp_atomic_u32_t *ref_cnt; + + ref_cnt = &pkt_hdr[cur]->ref_cnt; + prev_ref = odp_atomic_fetch_inc_u32(ref_cnt); + + _ODP_ASSERT(prev_ref == 0); + } + + cur++; + + if (cur == num) { + /* Last segment */ + hdr->seg_next = NULL; + return; + } + + hdr->seg_next = pkt_hdr[cur]; + } } static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) { odp_packet_hdr_t *hdr; - int i; + uint32_t seg_len; /* First segment is the packet descriptor */ hdr = pkt_hdr[0]; + seg_len = _odp_pool_entry(hdr->event_hdr.pool)->seg_len; - hdr->buf_hdr.seg[0].data = hdr->buf_hdr.base_data; - hdr->buf_hdr.seg[0].len = BASE_LEN; - packet_ref_count_set(hdr, 1); + /* Defaults for single segment packet */ + hdr->seg_data = hdr->event_hdr.base_data; + hdr->seg_len = seg_len; + hdr->seg_next = NULL; - /* Link segments */ - if (CONFIG_PACKET_MAX_SEGS != 1) { - hdr->buf_hdr.segcount = num; - - if (odp_unlikely(num > 1)) { - for (i = 1; i < num; i++) { - odp_buffer_hdr_t *buf_hdr; - - packet_ref_count_set(pkt_hdr[i], 1); - buf_hdr = &pkt_hdr[i]->buf_hdr; - hdr->buf_hdr.seg[i].hdr = buf_hdr; - hdr->buf_hdr.seg[i].data = buf_hdr->base_data; - hdr->buf_hdr.seg[i].len = BASE_LEN; - } - } + hdr->seg_count = num; + + if (ODP_DEBUG == 1) { + uint32_t prev_ref = + odp_atomic_fetch_inc_u32(&hdr->ref_cnt); + + _ODP_ASSERT(prev_ref == 0); } + + /* Link segments */ + if (odp_unlikely(num > 1)) + link_segments(pkt_hdr, num); } -static inline void reset_seg(odp_packet_hdr_t *pkt_hdr, int first, int num) +static inline void reset_segments(odp_packet_hdr_t *pkt_hdr) { - odp_buffer_hdr_t *hdr; void *base; - int i; + uint32_t seg_len = _odp_pool_entry(pkt_hdr->event_hdr.pool)->seg_len; + + while (pkt_hdr != NULL) { + base = pkt_hdr->event_hdr.base_data; - for (i = first; i < first + num; i++) { - hdr = pkt_hdr->buf_hdr.seg[i].hdr; - base = hdr->base_data; - pkt_hdr->buf_hdr.seg[i].len = BASE_LEN; - pkt_hdr->buf_hdr.seg[i].data = base; + pkt_hdr->seg_len = seg_len; + pkt_hdr->seg_data = base; + + pkt_hdr = pkt_hdr->seg_next; } } /* Calculate the number of segments */ -static inline int num_segments(uint32_t len) +static inline int num_segments(uint32_t len, uint32_t seg_len) { - uint32_t max_seg_len; - int num; - - if (CONFIG_PACKET_MAX_SEGS == 1) - return 1; + int num = 1; - num = 1; - max_seg_len = CONFIG_PACKET_MAX_SEG_LEN; + if (odp_unlikely(len > seg_len)) { + num = len / seg_len; - if (odp_unlikely(len > max_seg_len)) { - num = len / max_seg_len; - - if (odp_likely((num * max_seg_len) != len)) + if (odp_likely((num * seg_len) != len)) num += 1; } @@ -342,31 +365,10 @@ static inline int num_segments(uint32_t len) static inline void add_all_segs(odp_packet_hdr_t *to, odp_packet_hdr_t *from) { - int i; - int n = to->buf_hdr.segcount; - int num = from->buf_hdr.segcount; + odp_packet_hdr_t *last = packet_last_seg(to); - for (i = 0; i < num; i++) { - to->buf_hdr.seg[n + i].hdr = from->buf_hdr.seg[i].hdr; - to->buf_hdr.seg[n + i].data = from->buf_hdr.seg[i].data; - to->buf_hdr.seg[n + i].len = from->buf_hdr.seg[i].len; - } - - to->buf_hdr.segcount = n + num; -} - -static inline void copy_num_segs(odp_packet_hdr_t *to, odp_packet_hdr_t *from, - int first, int num) -{ - int i; - - for (i = 0; i < num; i++) { - to->buf_hdr.seg[i].hdr = from->buf_hdr.seg[first + i].hdr; - to->buf_hdr.seg[i].data = from->buf_hdr.seg[first + i].data; - to->buf_hdr.seg[i].len = from->buf_hdr.seg[first + i].len; - } - - to->buf_hdr.segcount = num; + last->seg_next = from; + to->seg_count += from->seg_count; } static inline odp_packet_hdr_t *alloc_segments(pool_t *pool, int num) @@ -374,11 +376,11 @@ static inline odp_packet_hdr_t *alloc_segments(pool_t *pool, int num) odp_packet_hdr_t *pkt_hdr[num]; int ret; - ret = buffer_alloc_multi(pool, (odp_buffer_hdr_t **)pkt_hdr, num); + ret = _odp_event_alloc_multi(pool, (_odp_event_hdr_t **)pkt_hdr, num); if (odp_unlikely(ret != num)) { if (ret > 0) - buffer_free_multi((odp_buffer_hdr_t **)pkt_hdr, ret); + _odp_event_free_multi((_odp_event_hdr_t **)pkt_hdr, ret); return NULL; } @@ -400,110 +402,173 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, if (new_hdr == NULL) return NULL; - seg_len = len - ((num - 1) * pool->max_seg_len); - offset = pool->max_seg_len - seg_len; + seg_len = len - ((num - 1) * pool->seg_len); + offset = pool->seg_len - seg_len; if (head) { /* add into the head*/ add_all_segs(new_hdr, pkt_hdr); /* adjust first segment length */ - new_hdr->buf_hdr.seg[0].data += offset; - new_hdr->buf_hdr.seg[0].len = seg_len; + new_hdr->seg_data += offset; + new_hdr->seg_len = seg_len; - packet_seg_copy_md(new_hdr, pkt_hdr); - new_hdr->frame_len = pkt_hdr->frame_len + len; - new_hdr->unshared_len = pkt_hdr->unshared_len + len; - new_hdr->headroom = pool->headroom + offset; - new_hdr->tailroom = pkt_hdr->tailroom; + _odp_packet_copy_md(new_hdr, pkt_hdr, 0); + new_hdr->frame_len = pkt_hdr->frame_len + len; + new_hdr->headroom = pool->headroom + offset; + new_hdr->tailroom = pkt_hdr->tailroom; pkt_hdr = new_hdr; } else { - int last; + odp_packet_hdr_t *last_seg; /* add into the tail */ add_all_segs(pkt_hdr, new_hdr); /* adjust last segment length */ - last = packet_last_seg(pkt_hdr); - pkt_hdr->buf_hdr.seg[last].len = seg_len; + last_seg = packet_last_seg(pkt_hdr); + last_seg->seg_len = seg_len; - pkt_hdr->frame_len += len; - pkt_hdr->unshared_len += len; - pkt_hdr->tailroom = pool->tailroom + offset; + pkt_hdr->frame_len += len; + pkt_hdr->tailroom = pool->tailroom + offset; } return pkt_hdr; } -static inline void free_bufs(odp_packet_hdr_t *pkt_hdr, int first, int num) +static inline void segment_ref_inc(odp_packet_hdr_t *seg_hdr) { - int i, nfree; - odp_buffer_hdr_t *buf_hdr[num]; + uint32_t ref_cnt = odp_atomic_load_u32(&seg_hdr->ref_cnt); + + /* First count increment after alloc */ + if (odp_likely(ref_cnt == 0)) + odp_atomic_store_u32(&seg_hdr->ref_cnt, 2); + else + odp_atomic_inc_u32(&seg_hdr->ref_cnt); +} - for (i = 0, nfree = 0; i < num; i++) { - odp_packet_hdr_t *hdr = pkt_hdr->buf_hdr.seg[first + i].hdr; +static inline uint32_t segment_ref_dec(odp_packet_hdr_t *seg_hdr) +{ + return odp_atomic_fetch_dec_u32(&seg_hdr->ref_cnt); +} - if (packet_ref_count(hdr) == 1 || packet_ref_dec(hdr) == 1) { - ODP_ASSERT((packet_ref_count_set(hdr, 0), 1)); - buf_hdr[nfree++] = &hdr->buf_hdr; +static inline uint32_t segment_ref(odp_packet_hdr_t *seg_hdr) +{ + return odp_atomic_load_u32(&seg_hdr->ref_cnt); +} + +static inline int is_multi_ref(uint32_t ref_cnt) +{ + return (ref_cnt > 1); +} + +static inline void packet_free_multi(odp_packet_hdr_t *hdr[], int num) +{ + int i; + uint32_t ref_cnt; + int num_ref = 0; + + for (i = 0; i < num; i++) { + /* Zero when reference API has not been used */ + ref_cnt = segment_ref(hdr[i]); + + if (odp_unlikely(ref_cnt)) { + ref_cnt = segment_ref_dec(hdr[i]); + + if (is_multi_ref(ref_cnt)) { + num_ref++; + continue; + } } + + /* Skip references and pack to be freed headers to array head */ + if (odp_unlikely(num_ref)) + hdr[i - num_ref] = hdr[i]; } - if (nfree > 0) - buffer_free_multi(buf_hdr, nfree); + num -= num_ref; + + if (odp_likely(num)) + _odp_event_free_multi((_odp_event_hdr_t **)(uintptr_t)hdr, num); +} + +static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num) +{ + int i; + odp_packet_hdr_t *pkt_hdrs[num]; + odp_packet_hdr_t *seg_hdr = pkt_hdr; + + for (i = 0; i < num; i++) { + pkt_hdrs[i] = seg_hdr; + seg_hdr = seg_hdr->seg_next; + } + + packet_free_multi(pkt_hdrs, num); } static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, int num, uint32_t free_len, uint32_t pull_len, int head) { - int num_remain = pkt_hdr->buf_hdr.segcount - num; + odp_packet_hdr_t *seg_hdr; + int i; + int num_remain = pkt_hdr->seg_count - num; + odp_packet_hdr_t *hdr = pkt_hdr; + odp_packet_hdr_t *last_hdr = packet_last_seg(pkt_hdr); + odp_packet_hdr_t *pkt_hdrs[num]; if (head) { odp_packet_hdr_t *new_hdr; - int i, nfree; - odp_buffer_hdr_t *buf_hdr[num]; - for (i = 0, nfree = 0; i < num; i++) { - new_hdr = pkt_hdr->buf_hdr.seg[i].hdr; - - if (packet_ref_count(new_hdr) == 1 || - packet_ref_dec(new_hdr) == 1) { - ODP_ASSERT((packet_ref_count_set(new_hdr, 0), - 1)); - buf_hdr[nfree++] = &new_hdr->buf_hdr; - } + for (i = 0; i < num; i++) { + seg_hdr = packet_seg_step(&hdr); + pkt_hdrs[i] = seg_hdr; } - /* First remaining segment is the new packet descriptor */ - new_hdr = pkt_hdr->buf_hdr.seg[num].hdr; + /* The first remaining header is the new packet descriptor. + * Copy remaining segments from the last to-be-removed header + * to the new header. */ + new_hdr = hdr; - copy_num_segs(new_hdr, pkt_hdr, num, num_remain); - packet_seg_copy_md(new_hdr, pkt_hdr); + new_hdr->seg_next = hdr->seg_next; + new_hdr->seg_count = num_remain; + + _odp_packet_copy_md(new_hdr, pkt_hdr, 0); /* Tailroom not changed */ - new_hdr->tailroom = pkt_hdr->tailroom; - new_hdr->headroom = seg_headroom(new_hdr, 0); - new_hdr->frame_len = pkt_hdr->frame_len - free_len; - new_hdr->unshared_len = pkt_hdr->unshared_len - free_len; + new_hdr->tailroom = pkt_hdr->tailroom; + + new_hdr->headroom = seg_headroom(new_hdr); + + new_hdr->frame_len = pkt_hdr->frame_len - free_len; pull_head(new_hdr, pull_len); pkt_hdr = new_hdr; - if (nfree > 0) - buffer_free_multi(buf_hdr, nfree); + packet_free_multi(pkt_hdrs, num); } else { - /* Free last 'num' bufs */ - free_bufs(pkt_hdr, num_remain, num); + /* Free last 'num' bufs. + * First, find the last remaining header. */ + packet_seg_find_idx(&hdr, num_remain - 1); + last_hdr = hdr; + + packet_seg_step(&hdr); + + for (i = 0; i < num; i++) { + seg_hdr = packet_seg_step(&hdr); + pkt_hdrs[i] = seg_hdr; + } + + packet_free_multi(pkt_hdrs, num); /* Head segment remains, no need to copy or update majority * of the metadata. */ - pkt_hdr->buf_hdr.segcount = num_remain; + last_hdr->seg_next = NULL; + + pkt_hdr->seg_count = num_remain; pkt_hdr->frame_len -= free_len; - pkt_hdr->unshared_len -= free_len; - pkt_hdr->tailroom = seg_tailroom(pkt_hdr, num_remain - 1); + pkt_hdr->tailroom = seg_tailroom(pkt_hdr); pull_tail(pkt_hdr, pull_len); } @@ -518,9 +583,11 @@ static inline int packet_alloc(pool_t *pool, uint32_t len, int max_pkt, int num = max_pkt; int max_buf = max_pkt * num_seg; odp_packet_hdr_t *pkt_hdr[max_buf]; + odp_packet_hdr_t *hdr_next; + odp_packet_hdr_t *hdr; - num_buf = buffer_alloc_multi(pool, (odp_buffer_hdr_t **)pkt_hdr, - max_buf); + num_buf = _odp_event_alloc_multi(pool, (_odp_event_hdr_t **)pkt_hdr, + max_buf); /* Failed to allocate all segments */ if (odp_unlikely(num_buf != max_buf)) { @@ -530,37 +597,49 @@ static inline int packet_alloc(pool_t *pool, uint32_t len, int max_pkt, num_free = num_buf - (num * num_seg); if (num_free > 0) { - odp_buffer_hdr_t **p; + _odp_event_hdr_t **p; - p = (odp_buffer_hdr_t **)&pkt_hdr[num_buf - num_free]; - buffer_free_multi(p, num_free); + p = (_odp_event_hdr_t **)&pkt_hdr[num_buf - num_free]; + _odp_event_free_multi(p, num_free); } if (num == 0) return 0; } - for (i = 0; i < num; i++) { - odp_packet_hdr_t *hdr; + hdr_next = pkt_hdr[0]; + odp_prefetch(hdr_next); + odp_prefetch((uint8_t *)hdr_next + ODP_CACHE_LINE_SIZE); + + for (i = 0; i < num - 1; i++) { + hdr = hdr_next; + hdr_next = pkt_hdr[(i + 1) * num_seg]; + + odp_prefetch(hdr_next); + odp_prefetch((uint8_t *)hdr_next + ODP_CACHE_LINE_SIZE); /* First buffer is the packet descriptor */ - hdr = pkt_hdr[i * num_seg]; pkt[i] = packet_handle(hdr); init_segments(&pkt_hdr[i * num_seg], num_seg); packet_init(hdr, len); } + /* Last packet */ + pkt[i] = packet_handle(hdr_next); + init_segments(&pkt_hdr[i * num_seg], num_seg); + packet_init(hdr_next, len); + return num; } -int packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, - odp_packet_t pkt[], int max_num) +int _odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, + odp_packet_t pkt[], int max_num) { - pool_t *pool = pool_entry_from_hdl(pool_hdl); + pool_t *pool = _odp_pool_entry(pool_hdl); int num, num_seg; - num_seg = num_segments(len); + num_seg = num_segments(len, pool->seg_len); num = packet_alloc(pool, len, max_num, num_seg, pkt); return num; @@ -568,19 +647,16 @@ int packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, uint32_t len) { - pool_t *pool = pool_entry_from_hdl(pool_hdl); + pool_t *pool = _odp_pool_entry(pool_hdl); odp_packet_t pkt; int num, num_seg; - if (odp_unlikely(pool->params.type != ODP_POOL_PACKET)) { - __odp_errno = EINVAL; - return ODP_PACKET_INVALID; - } + _ODP_ASSERT(pool->type == ODP_POOL_PACKET); - if (odp_unlikely(len > pool->max_len)) + if (odp_unlikely(len > pool->max_len || len == 0)) return ODP_PACKET_INVALID; - num_seg = num_segments(len); + num_seg = num_segments(len, pool->seg_len); num = packet_alloc(pool, len, 1, num_seg, &pkt); if (odp_unlikely(num == 0)) @@ -592,132 +668,130 @@ odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, uint32_t len) int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, odp_packet_t pkt[], int max_num) { - pool_t *pool = pool_entry_from_hdl(pool_hdl); + pool_t *pool = _odp_pool_entry(pool_hdl); int num, num_seg; - if (odp_unlikely(pool->params.type != ODP_POOL_PACKET)) { - __odp_errno = EINVAL; - return -1; - } + _ODP_ASSERT(pool->type == ODP_POOL_PACKET); - if (odp_unlikely(len > pool->max_len)) + if (odp_unlikely(len > pool->max_len || len == 0)) return -1; - num_seg = num_segments(len); + num_seg = num_segments(len, pool->seg_len); num = packet_alloc(pool, len, max_num, num_seg, pkt); return num; } -static inline void packet_free(odp_packet_hdr_t *pkt_hdr) +void odp_packet_free(odp_packet_t pkt) { - odp_packet_hdr_t *ref_hdr; - odp_buffer_hdr_t *buf_hdr; - uint32_t ref_count; - int num_seg; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + int num_seg = pkt_hdr->seg_count; - do { - buf_hdr = &pkt_hdr->buf_hdr; - ref_count = packet_ref_count(pkt_hdr); - num_seg = pkt_hdr->buf_hdr.segcount; - ref_hdr = pkt_hdr->ref_hdr; - ODP_ASSERT(ref_count >= 1); + _odp_packet_validate(pkt, _ODP_EV_PACKET_FREE); - if (odp_likely((CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1) && - ref_count == 1)) { - ODP_ASSERT((packet_ref_count_set(pkt_hdr, 0), 1)); - buffer_free_multi(&buf_hdr, 1); - } else { - free_bufs(pkt_hdr, 0, num_seg); - } + _ODP_ASSERT(segment_ref(pkt_hdr) > 0); - pkt_hdr = ref_hdr; - } while (pkt_hdr); + if (odp_likely(num_seg == 1)) + packet_free_multi(&pkt_hdr, 1); + else + free_all_segments(pkt_hdr, num_seg); } -void odp_packet_free(odp_packet_t pkt) +static inline void packet_free_multi_ev(const odp_packet_t pkt[], int num, _odp_ev_id_t id) { - packet_free(packet_hdr(pkt)); -} + if (odp_unlikely(!num)) + return; -void odp_packet_free_multi(const odp_packet_t pkt[], int num) -{ - odp_packet_hdr_t *pkt_hdr, *ref_hdr, *hdr; - int nbufs = num * CONFIG_PACKET_MAX_SEGS * 2; - odp_buffer_hdr_t *buf_hdr[nbufs]; - int num_seg; - int i, j; - uint32_t ref_count; - int nfree = 0; + odp_packet_hdr_t *pkt_hdrs[num]; + int i; + int num_freed = 0; - for (i = 0; i < num; i++) { - pkt_hdr = packet_hdr(pkt[i]); + _odp_packet_validate_multi(pkt, num, id); - do { - num_seg = pkt_hdr->buf_hdr.segcount; - ref_hdr = pkt_hdr->ref_hdr; + for (i = 0; i < num; i++) { + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt[i]); + int num_seg = pkt_hdr->seg_count; - /* Make sure we have enough space for this pkt's segs */ - if (nfree + num_seg > nbufs) { - buffer_free_multi(buf_hdr, nfree); - nfree = 0; - } + _ODP_ASSERT(segment_ref(pkt_hdr) > 0); - for (j = 0; j < num_seg; j++) { - hdr = pkt_hdr->buf_hdr.seg[j].hdr; - ref_count = packet_ref_count(hdr); - ODP_ASSERT(ref_count >= 1); - - if (ref_count == 1 || - packet_ref_dec(hdr) == 1) { - ODP_ASSERT - ((packet_ref_count_set(hdr, 0), - 1)); - buf_hdr[nfree++] = &hdr->buf_hdr; - } - } + if (odp_unlikely(num_seg > 1)) { + free_all_segments(pkt_hdr, num_seg); + num_freed++; + continue; + } - pkt_hdr = ref_hdr; - } while (pkt_hdr); + pkt_hdrs[i - num_freed] = pkt_hdr; } - if (nfree > 0) - buffer_free_multi(buf_hdr, nfree); + if (odp_likely(num - num_freed)) + packet_free_multi(pkt_hdrs, num - num_freed); +} + +void odp_packet_free_multi(const odp_packet_t pkt[], int num) +{ + packet_free_multi_ev(pkt, num, _ODP_EV_PACKET_FREE_MULTI); +} + +void odp_packet_free_sp(const odp_packet_t pkt[], int num) +{ + packet_free_multi_ev(pkt, num, _ODP_EV_PACKET_FREE_SP); +} + +uint32_t odp_packet_reset_max_len(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + pool_t *pool = _odp_pool_entry(pkt_hdr->event_hdr.pool); + + return pool->seg_len * pkt_hdr->seg_count; } int odp_packet_reset(odp_packet_t pkt, uint32_t len) { odp_packet_hdr_t *const pkt_hdr = packet_hdr(pkt); - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; - int num = pkt_hdr->buf_hdr.segcount; + pool_t *pool = _odp_pool_entry(pkt_hdr->event_hdr.pool); + int num = pkt_hdr->seg_count; + int num_req; - if (odp_unlikely(len > (pool->max_seg_len * num))) + if (odp_unlikely(len > (pool->seg_len * num)) || len == 0) return -1; - if (pkt_hdr->ref_hdr) - packet_free(pkt_hdr->ref_hdr); - - reset_seg(pkt_hdr, 0, num); + /* Free possible extra segments */ + num_req = num_segments(len, pool->seg_len); + if (odp_unlikely(num_req < num)) + free_segments(pkt_hdr, num - num_req, 0, 0, 0); + reset_segments(pkt_hdr); packet_init(pkt_hdr, len); return 0; } -odp_packet_t odp_packet_from_event(odp_event_t ev) +void odp_packet_reset_meta(odp_packet_t pkt) { - if (odp_unlikely(ev == ODP_EVENT_INVALID)) - return ODP_PACKET_INVALID; + _ODP_ASSERT(pkt != ODP_PACKET_INVALID); - return (odp_packet_t)buf_to_packet_hdr((odp_buffer_t)ev); + _odp_packet_reset_md(packet_hdr(pkt)); } -odp_event_t odp_packet_to_event(odp_packet_t pkt) +int odp_event_filter_packet(const odp_event_t event[], + odp_packet_t packet[], + odp_event_t remain[], int num) { - if (odp_unlikely(pkt == ODP_PACKET_INVALID)) - return ODP_EVENT_INVALID; + int i; + int num_pkt = 0; + int num_rem = 0; - return (odp_event_t)buffer_handle(packet_hdr(pkt)); + for (i = 0; i < num; i++) { + if (odp_event_type(event[i]) == ODP_EVENT_PACKET) { + packet[num_pkt] = odp_packet_from_event(event[i]); + num_pkt++; + } else { + remain[num_rem] = event[i]; + num_rem++; + } + } + + return num_pkt; } /* @@ -730,47 +804,14 @@ odp_event_t odp_packet_to_event(odp_packet_t pkt) uint32_t odp_packet_buf_len(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t buf_len = 0; - - do { - buf_len += pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount; - pkt_hdr = pkt_hdr->ref_hdr; - } while (pkt_hdr); + pool_t *pool = _odp_pool_entry(pkt_hdr->event_hdr.pool); - return buf_len; -} - -uint32_t odp_packet_unshared_len(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t pkt_len = 0, offset = 0; - - if (packet_ref_count(pkt_hdr) == 1) - pkt_hdr->unshared_len = pkt_hdr->frame_len; - - do { - if (packet_ref_count(pkt_hdr) > 1) { - if (offset == 0) - pkt_len += pkt_hdr->unshared_len; - break; - } - - pkt_len += pkt_hdr->frame_len - offset; - offset = pkt_hdr->ref_offset; - - if (pkt_hdr->ref_hdr) - offset += (pkt_hdr->ref_hdr->frame_len - - pkt_hdr->ref_len); - - pkt_hdr = pkt_hdr->ref_hdr; - } while (pkt_hdr); - - return pkt_len; + return pool->max_seg_len * pkt_hdr->seg_count; } void *odp_packet_tail(odp_packet_t pkt) { - odp_packet_hdr_t *pkt_hdr = packet_last_hdr(pkt, NULL); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); return packet_tail(pkt_hdr); } @@ -786,166 +827,6 @@ void *odp_packet_push_head(odp_packet_t pkt, uint32_t len) return packet_data(pkt_hdr); } -static inline uint32_t pack_seg_head(odp_packet_hdr_t *pkt_hdr, int seg) -{ - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[seg].hdr; - uint32_t len = pkt_hdr->buf_hdr.seg[seg].len; - uint8_t *src = pkt_hdr->buf_hdr.seg[seg].data; - uint8_t *dst = hdr->base_data; - - if (dst != src) { - memmove(dst, src, len); - pkt_hdr->buf_hdr.seg[seg].data = dst; - } - - return len; -} - -static inline uint32_t pack_seg_tail(odp_packet_hdr_t *pkt_hdr, int seg) -{ - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[seg].hdr; - uint32_t len = pkt_hdr->buf_hdr.seg[seg].len; - uint8_t *src = pkt_hdr->buf_hdr.seg[seg].data; - uint8_t *dst = hdr->base_data + BASE_LEN - len; - - if (dst != src) { - memmove(dst, src, len); - pkt_hdr->buf_hdr.seg[seg].data = dst; - } - - return len; -} - -static inline uint32_t fill_seg_head(odp_packet_hdr_t *pkt_hdr, int dst_seg, - int src_seg, uint32_t max_len) -{ - uint32_t len = pkt_hdr->buf_hdr.seg[src_seg].len; - uint8_t *src = pkt_hdr->buf_hdr.seg[src_seg].data; - uint32_t offset = pkt_hdr->buf_hdr.seg[dst_seg].len; - uint8_t *dst = pkt_hdr->buf_hdr.seg[dst_seg].data + offset; - - if (len > max_len) - len = max_len; - - memmove(dst, src, len); - - pkt_hdr->buf_hdr.seg[dst_seg].len += len; - pkt_hdr->buf_hdr.seg[src_seg].len -= len; - pkt_hdr->buf_hdr.seg[src_seg].data += len; - - if (pkt_hdr->buf_hdr.seg[src_seg].len == 0) { - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[src_seg].hdr; - - pkt_hdr->buf_hdr.seg[src_seg].data = hdr->base_data; - } - - return len; -} - -static inline uint32_t fill_seg_tail(odp_packet_hdr_t *pkt_hdr, int dst_seg, - int src_seg, uint32_t max_len) -{ - uint32_t src_len = pkt_hdr->buf_hdr.seg[src_seg].len; - uint8_t *src = pkt_hdr->buf_hdr.seg[src_seg].data; - uint8_t *dst = pkt_hdr->buf_hdr.seg[dst_seg].data; - uint32_t len = src_len; - - if (len > max_len) - len = max_len; - - src += src_len - len; - dst -= len; - - memmove(dst, src, len); - - pkt_hdr->buf_hdr.seg[dst_seg].data -= len; - pkt_hdr->buf_hdr.seg[dst_seg].len += len; - pkt_hdr->buf_hdr.seg[src_seg].len -= len; - - if (pkt_hdr->buf_hdr.seg[src_seg].len == 0) { - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[src_seg].hdr; - - pkt_hdr->buf_hdr.seg[src_seg].data = hdr->base_data; - } - - return len; -} - -static inline int move_data_to_head(odp_packet_hdr_t *pkt_hdr, int segs) -{ - int dst_seg, src_seg; - uint32_t len, free_len; - uint32_t moved = 0; - - for (dst_seg = 0; dst_seg < segs; dst_seg++) { - len = pack_seg_head(pkt_hdr, dst_seg); - moved += len; - - if (len == BASE_LEN) - continue; - - free_len = BASE_LEN - len; - - for (src_seg = dst_seg + 1; CONFIG_PACKET_MAX_SEGS > 1 && - src_seg < segs; src_seg++) { - len = fill_seg_head(pkt_hdr, dst_seg, src_seg, - free_len); - moved += len; - - if (len == free_len) { - /* dst seg is full */ - break; - } - - /* src seg is empty */ - free_len -= len; - } - - if (moved == pkt_hdr->frame_len) - break; - } - - /* last segment which have data */ - return dst_seg; -} - -static inline int move_data_to_tail(odp_packet_hdr_t *pkt_hdr, int segs) -{ - int dst_seg, src_seg; - uint32_t len, free_len; - uint32_t moved = 0; - - for (dst_seg = segs - 1; dst_seg >= 0; dst_seg--) { - len = pack_seg_tail(pkt_hdr, dst_seg); - moved += len; - - if (len == BASE_LEN) - continue; - - free_len = BASE_LEN - len; - - for (src_seg = dst_seg - 1; src_seg >= 0; src_seg--) { - len = fill_seg_tail(pkt_hdr, dst_seg, src_seg, - free_len); - moved += len; - - if (len == free_len) { - /* dst seg is full */ - break; - } - - /* src seg is empty */ - free_len -= len; - } - - if (moved == pkt_hdr->frame_len) - break; - } - - /* first segment which have data */ - return dst_seg; -} - int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len) { @@ -955,111 +836,37 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, int ret = 0; if (len > headroom) { - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + pool_t *pool = _odp_pool_entry(pkt_hdr->event_hdr.pool); int num; - int segs; + void *ptr; if (odp_unlikely((frame_len + len) > pool->max_len)) return -1; - num = num_segments(len - headroom); - segs = pkt_hdr->buf_hdr.segcount; - - if (odp_unlikely((segs + num) > CONFIG_PACKET_MAX_SEGS)) { - /* Corner case: fail request if packet has - * references since we cannot shuffle segments - * since another thread may be accessing them - * concurrently */ - if (packet_ref_count(pkt_hdr) > 1) - return -1; - - /* Cannot directly add new segments */ - odp_packet_hdr_t *new_hdr; - int new_segs = 0; - int free_segs = 0; - uint32_t offset; - - num = num_segments(frame_len + len); - - if (num > segs) { - /* Allocate additional segments */ - new_segs = num - segs; - new_hdr = alloc_segments(pool, new_segs); - - if (new_hdr == NULL) - return -1; - - } else if (num < segs) { - free_segs = segs - num; - } - - /* Pack all data to packet tail */ - move_data_to_tail(pkt_hdr, segs); - reset_seg(pkt_hdr, 0, segs); - - if (new_segs) { - add_all_segs(new_hdr, pkt_hdr); - packet_seg_copy_md(new_hdr, pkt_hdr); - segs += new_segs; - - pkt_hdr = new_hdr; - *pkt = packet_handle(pkt_hdr); - } else if (CONFIG_PACKET_MAX_SEGS > 1 && free_segs) { - new_hdr = pkt_hdr->buf_hdr.seg[free_segs].hdr; - packet_seg_copy_md(new_hdr, pkt_hdr); - - /* Free extra segs */ - free_bufs(pkt_hdr, 0, free_segs); - - segs -= free_segs; - pkt_hdr = new_hdr; - *pkt = packet_handle(pkt_hdr); - } - - frame_len += len; - offset = (segs * BASE_LEN) - frame_len; - - pkt_hdr->buf_hdr.seg[0].data += offset; - pkt_hdr->buf_hdr.seg[0].len -= offset; - - pkt_hdr->buf_hdr.segcount = segs; - pkt_hdr->frame_len = frame_len; - pkt_hdr->unshared_len = frame_len; - pkt_hdr->headroom = offset + pool->headroom; - pkt_hdr->tailroom = pool->tailroom; - - /* Data was moved */ - ret = 1; - } else { - void *ptr; - - push_head(pkt_hdr, headroom); - ptr = add_segments(pkt_hdr, pool, len - headroom, - num, 1); + num = num_segments(len - headroom, pool->seg_len); + if (odp_unlikely(pkt_hdr->seg_count + num > PKT_MAX_SEGS)) + return -1; - if (ptr == NULL) { - /* segment alloc failed, rollback changes */ - pull_head(pkt_hdr, headroom); - return -1; - } + push_head(pkt_hdr, headroom); + ptr = add_segments(pkt_hdr, pool, len - headroom, num, 1); - *pkt = packet_handle(ptr); - pkt_hdr = ptr; + if (ptr == NULL) { + /* segment alloc failed, rollback changes */ + pull_head(pkt_hdr, headroom); + return -1; } + + *pkt = packet_handle(ptr); + pkt_hdr = ptr; } else { push_head(pkt_hdr, len); } - if (data_ptr || seg_len) { - uint32_t seg_ln = 0; - void *data = packet_map(pkt_hdr, 0, &seg_ln, NULL); - - if (data_ptr) - *data_ptr = data; + if (data_ptr) + *data_ptr = packet_data(pkt_hdr); - if (seg_len) - *seg_len = seg_ln; - } + if (seg_len) + *seg_len = packet_first_seg_len(pkt_hdr); return ret; } @@ -1068,11 +875,9 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (len > pkt_hdr->frame_len) + if (len >= pkt_hdr->seg_len) return NULL; - ODP_ASSERT(len <= pkt_hdr->unshared_len); - pull_head(pkt_hdr, len); return packet_data(pkt_hdr); } @@ -1080,33 +885,13 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len_out) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt), *nxt_hdr; + odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); uint32_t seg_len = packet_first_seg_len(pkt_hdr); - int ret = 0; - if (len > packet_len(pkt_hdr)) + if (len >= pkt_hdr->frame_len) return -1; - ODP_ASSERT(len <= odp_packet_unshared_len(*pkt)); - - /* Special processing for references */ - while (len >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); - nxt_hdr = pkt_hdr->ref_hdr; - len -= pkt_hdr->frame_len; - len += pkt_hdr->ref_offset + - (nxt_hdr->frame_len - pkt_hdr->ref_len); - pkt_hdr->ref_hdr = NULL; - packet_free(pkt_hdr); - pkt_hdr = nxt_hdr; - seg_len = packet_first_seg_len(pkt_hdr); - *pkt = packet_handle(pkt_hdr); - ret = 1; - } - - if (CONFIG_PACKET_MAX_SEGS == 1 || - len < seg_len || - pkt_hdr->buf_hdr.segcount == 1) { + if (len < seg_len) { pull_head(pkt_hdr, len); } else { int num = 0; @@ -1123,28 +908,24 @@ int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, *pkt = packet_handle(pkt_hdr); } - if (data_ptr || seg_len_out) { - void *data_head = packet_map(pkt_hdr, 0, &seg_len, NULL); - - if (data_ptr) - *data_ptr = data_head; + if (data_ptr) + *data_ptr = packet_data(pkt_hdr); - if (seg_len_out) - *seg_len_out = seg_len; - } + if (seg_len_out) + *seg_len_out = packet_first_seg_len(pkt_hdr); - return ret; + return 0; } void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) { - odp_packet_hdr_t *pkt_hdr = packet_last_hdr(pkt, NULL); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); void *old_tail; if (len > pkt_hdr->tailroom) return NULL; - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); + _ODP_ASSERT(odp_packet_has_ref(pkt) == 0); old_tail = packet_tail(pkt_hdr); push_tail(pkt_hdr, len); @@ -1155,86 +936,33 @@ void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len_out) { - odp_packet_hdr_t *pkt_hdr = packet_last_hdr(*pkt, NULL); + odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); uint32_t frame_len = pkt_hdr->frame_len; uint32_t tailroom = pkt_hdr->tailroom; uint32_t tail_off = frame_len; int ret = 0; - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); + _ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); if (len > tailroom) { - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + pool_t *pool = _odp_pool_entry(pkt_hdr->event_hdr.pool); int num; - int segs; + void *ptr; if (odp_unlikely((frame_len + len) > pool->max_len)) return -1; - num = num_segments(len - tailroom); - segs = pkt_hdr->buf_hdr.segcount; - - if (odp_unlikely((segs + num) > CONFIG_PACKET_MAX_SEGS)) { - /* Cannot directly add new segments */ - odp_packet_hdr_t *new_hdr; - int new_segs = 0; - int free_segs = 0; - uint32_t offset; - - num = num_segments(frame_len + len); - - if (num > segs) { - /* Allocate additional segments */ - new_segs = num - segs; - new_hdr = alloc_segments(pool, new_segs); - - if (new_hdr == NULL) - return -1; - - } else if (num < segs) { - free_segs = segs - num; - } - - /* Pack all data to packet head */ - move_data_to_head(pkt_hdr, segs); - reset_seg(pkt_hdr, 0, segs); - - if (new_segs) { - /* Add new segs */ - add_all_segs(pkt_hdr, new_hdr); - segs += new_segs; - } else if (free_segs) { - /* Free extra segs */ - free_bufs(pkt_hdr, segs - free_segs, free_segs); - - segs -= free_segs; - } - - frame_len += len; - offset = (segs * BASE_LEN) - frame_len; - - pkt_hdr->buf_hdr.seg[segs - 1].len -= offset; - - pkt_hdr->buf_hdr.segcount = segs; - pkt_hdr->frame_len = frame_len; - pkt_hdr->headroom = pool->headroom; - pkt_hdr->tailroom = offset + pool->tailroom; - - /* Data was moved */ - ret = 1; - } else { - void *ptr; - - push_tail(pkt_hdr, tailroom); + num = num_segments(len - tailroom, pool->seg_len); + if (odp_unlikely(pkt_hdr->seg_count + num > PKT_MAX_SEGS)) + return -1; - ptr = add_segments(pkt_hdr, pool, len - tailroom, - num, 0); + push_tail(pkt_hdr, tailroom); + ptr = add_segments(pkt_hdr, pool, len - tailroom, num, 0); - if (ptr == NULL) { - /* segment alloc failed, rollback changes */ - pull_tail(pkt_hdr, tailroom); - return -1; - } + if (ptr == NULL) { + /* segment alloc failed, rollback changes */ + pull_tail(pkt_hdr, tailroom); + return -1; } } else { push_tail(pkt_hdr, len); @@ -1249,11 +977,13 @@ int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + odp_packet_hdr_t *last_seg = packet_last_seg(pkt_hdr); + + _ODP_ASSERT(odp_packet_has_ref(pkt) == 0); - if (len > packet_last_seg_len(pkt_hdr)) + if (len >= last_seg->seg_len) return NULL; - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); pull_tail(pkt_hdr, len); return packet_tail(pkt_hdr); @@ -1264,37 +994,25 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, { int last; uint32_t seg_len; - uint32_t offset; - odp_packet_hdr_t *first_hdr = packet_hdr(*pkt); - odp_packet_hdr_t *pkt_hdr, *prev_hdr; + odp_packet_hdr_t *last_seg; + odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); - if (len > packet_len(first_hdr)) + if (len >= pkt_hdr->frame_len) return -1; - pkt_hdr = packet_last_hdr(*pkt, &offset); - - /* Special processing for references */ - while (len >= pkt_hdr->frame_len - offset && first_hdr->ref_hdr) { - len -= (pkt_hdr->frame_len - offset); - prev_hdr = packet_prev_hdr(first_hdr, pkt_hdr, &offset); - ODP_ASSERT(packet_ref_count(prev_hdr) == 1); - prev_hdr->ref_hdr = NULL; - packet_free(pkt_hdr); - pkt_hdr = prev_hdr; - } + _ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); - last = packet_last_seg(pkt_hdr); - seg_len = packet_seg_len(pkt_hdr, last); + last = pkt_hdr->seg_count - 1; + last_seg = packet_last_seg(pkt_hdr); + seg_len = last_seg->seg_len; - if (CONFIG_PACKET_MAX_SEGS == 1 || - len < seg_len || - pkt_hdr->buf_hdr.segcount == 1) { + if (len < seg_len) { pull_tail(pkt_hdr, len); } else { int num = 0; uint32_t pull_len = 0; + /* Reverse order */ while (seg_len <= len) { pull_len = len - seg_len; num++; @@ -1315,12 +1033,9 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t *len, odp_packet_seg_t *seg) { - int seg_idx; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - void *addr = packet_map(pkt_hdr, offset, len, &seg_idx); - if (addr != NULL && seg != NULL) - *seg = _odp_packet_seg_from_ndx(seg_idx); + void *addr = packet_map(pkt_hdr, offset, len, seg); return addr; } @@ -1332,93 +1047,11 @@ void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t *len, * */ -int odp_packet_input_index(odp_packet_t pkt) -{ - return odp_pktio_index(packet_hdr(pkt)->input); -} - -void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ctx) -{ - packet_hdr(pkt)->buf_hdr.buf_cctx = ctx; -} - -void *odp_packet_l2_ptr(odp_packet_t pkt, uint32_t *len) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (!packet_hdr_has_l2(pkt_hdr)) - return NULL; - return packet_map(pkt_hdr, pkt_hdr->p.l2_offset, len, NULL); -} - -uint32_t odp_packet_l2_offset(odp_packet_t pkt) +uint16_t odp_packet_ones_comp(odp_packet_t pkt, odp_packet_data_range_t *range) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (!packet_hdr_has_l2(pkt_hdr)) - return ODP_PACKET_OFFSET_INVALID; - return pkt_hdr->p.l2_offset; -} - -int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (offset >= pkt_hdr->frame_len) - return -1; - - packet_hdr_has_l2_set(pkt_hdr, 1); - pkt_hdr->p.l2_offset = offset; - return 0; -} - -void *odp_packet_l3_ptr(odp_packet_t pkt, uint32_t *len) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - return packet_map(pkt_hdr, pkt_hdr->p.l3_offset, len, NULL); -} - -uint32_t odp_packet_l3_offset(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - return pkt_hdr->p.l3_offset; -} - -int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (offset >= pkt_hdr->frame_len) - return -1; - - pkt_hdr->p.l3_offset = offset; - return 0; -} - -void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - return packet_map(pkt_hdr, pkt_hdr->p.l4_offset, len, NULL); -} - -uint32_t odp_packet_l4_offset(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - return pkt_hdr->p.l4_offset; -} - -int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (offset >= pkt_hdr->frame_len) - return -1; - - pkt_hdr->p.l4_offset = offset; + (void)pkt; + range->length = 0; + range->offset = 0; return 0; } @@ -1426,56 +1059,7 @@ void odp_packet_flow_hash_set(odp_packet_t pkt, uint32_t flow_hash) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - pkt_hdr->flow_hash = flow_hash; - pkt_hdr->p.input_flags.flow_hash = 1; -} - -void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - pkt_hdr->timestamp = timestamp; - pkt_hdr->p.input_flags.timestamp = 1; -} - -int odp_packet_num_segs(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t segcount = 0, i; - uint32_t seg_offset = 0, offset; - - do { - segcount += pkt_hdr->buf_hdr.segcount - seg_offset; - offset = pkt_hdr->ref_offset; - pkt_hdr = pkt_hdr->ref_hdr; - if (pkt_hdr) { - for (i = 0, seg_offset = 0; - i < pkt_hdr->buf_hdr.segcount; - i++, seg_offset++) { - if (offset < pkt_hdr->buf_hdr.seg[i].len) - break; - offset -= pkt_hdr->buf_hdr.seg[i].len; - } - } - } while (pkt_hdr); - - return segcount; -} - -odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) -{ - return _odp_packet_seg_from_ndx(odp_packet_num_segs(pkt) - 1); -} - -odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >= - packet_last_seg(pkt_hdr))) - return ODP_PACKET_SEG_INVALID; - - return seg + 1; + packet_set_flow_hash(pkt_hdr, flow_hash); } /* @@ -1485,56 +1069,9 @@ odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg) * */ -void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t seg_offset = 0, offset = 0, i; - uint32_t seg_ndx = _odp_packet_seg_to_ndx(seg); - - while (seg_ndx >= pkt_hdr->buf_hdr.segcount - seg_offset && - pkt_hdr->ref_hdr) { - seg_ndx -= (pkt_hdr->buf_hdr.segcount - seg_offset); - offset = pkt_hdr->ref_offset; - pkt_hdr = pkt_hdr->ref_hdr; - for (i = 0, seg_offset = 0; - i < pkt_hdr->buf_hdr.segcount; - i++, seg_offset++) { - if (offset < pkt_hdr->buf_hdr.seg[i].len) - break; - offset -= pkt_hdr->buf_hdr.seg[i].len; - } - } - - if (odp_unlikely(seg_ndx + seg_offset >= pkt_hdr->buf_hdr.segcount)) - return NULL; - - return packet_seg_data(pkt_hdr, seg_ndx + seg_offset) + offset; -} - -uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg) +odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t seg_offset = 0, offset = 0, i; - uint32_t seg_ndx = _odp_packet_seg_to_ndx(seg); - - while (seg_ndx >= pkt_hdr->buf_hdr.segcount - seg_offset && - pkt_hdr->ref_hdr) { - seg_ndx -= (pkt_hdr->buf_hdr.segcount - seg_offset); - offset = pkt_hdr->ref_offset; - pkt_hdr = pkt_hdr->ref_hdr; - for (i = 0, seg_offset = 0; - i < pkt_hdr->buf_hdr.segcount; - i++, seg_offset++) { - if (offset < pkt_hdr->buf_hdr.seg[i].len) - break; - offset -= pkt_hdr->buf_hdr.seg[i].len; - } - } - - if (odp_unlikely(seg_ndx + seg_offset >= pkt_hdr->buf_hdr.segcount)) - return 0; - - return packet_seg_len(pkt_hdr, seg_ndx + seg_offset) - offset; + return (odp_packet_seg_t)packet_last_seg(packet_hdr(pkt)); } /* @@ -1549,15 +1086,13 @@ int odp_packet_add_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) odp_packet_t pkt = *pkt_ptr; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); uint32_t pktlen = pkt_hdr->frame_len; - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + odp_pool_t pool = pkt_hdr->event_hdr.pool; odp_packet_t newpkt; if (offset > pktlen) return -1; - ODP_ASSERT(odp_packet_unshared_len(*pkt_ptr) >= offset); - - newpkt = odp_packet_alloc(pool->pool_hdl, pktlen + len); + newpkt = odp_packet_alloc(pool, pktlen + len); if (newpkt == ODP_PACKET_INVALID) return -1; @@ -1569,7 +1104,7 @@ int odp_packet_add_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) return -1; } - _odp_packet_copy_md_to_packet(pkt, newpkt); + _odp_packet_copy_md(packet_hdr(newpkt), pkt_hdr, 0); odp_packet_free(pkt); *pkt_ptr = newpkt; @@ -1580,16 +1115,14 @@ int odp_packet_rem_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) { odp_packet_t pkt = *pkt_ptr; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t pktlen = packet_len(pkt_hdr); - pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + uint32_t pktlen = pkt_hdr->frame_len; + odp_pool_t pool = pkt_hdr->event_hdr.pool; odp_packet_t newpkt; - if (offset + len > pktlen) + if (offset + len >= pktlen) return -1; - ODP_ASSERT(odp_packet_unshared_len(*pkt_ptr) >= offset + len); - - newpkt = odp_packet_alloc(pool->pool_hdl, pktlen - len); + newpkt = odp_packet_alloc(pool, pktlen - len); if (newpkt == ODP_PACKET_INVALID) return -1; @@ -1601,7 +1134,7 @@ int odp_packet_rem_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) return -1; } - _odp_packet_copy_md_to_packet(pkt, newpkt); + _odp_packet_copy_md(packet_hdr(newpkt), pkt_hdr, 0); odp_packet_free(pkt); *pkt_ptr = newpkt; @@ -1615,6 +1148,7 @@ int odp_packet_align(odp_packet_t *pkt, uint32_t offset, uint32_t len, uint32_t shift; uint32_t seglen = 0; /* GCC */ odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); + pool_t *pool = _odp_pool_entry(pkt_hdr->event_hdr.pool); void *addr = packet_map(pkt_hdr, offset, &seglen, NULL); uint64_t uaddr = (uint64_t)(uintptr_t)addr; uint64_t misalign; @@ -1622,21 +1156,21 @@ int odp_packet_align(odp_packet_t *pkt, uint32_t offset, uint32_t len, if (align > ODP_CACHE_LINE_SIZE) return -1; - ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); + _ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); if (seglen >= len) { misalign = align <= 1 ? 0 : - ROUNDUP_ALIGN(uaddr, align) - uaddr; + _ODP_ROUNDUP_ALIGN(uaddr, align) - uaddr; if (misalign == 0) return 0; shift = align - misalign; } else { - if (len > pkt_hdr->buf_hdr.size) + if (len > pool->max_seg_len) return -1; shift = len - seglen; uaddr -= shift; misalign = align <= 1 ? 0 : - ROUNDUP_ALIGN(uaddr, align) - uaddr; + _ODP_ROUNDUP_ALIGN(uaddr, align) - uaddr; if (misalign) shift += align - misalign; } @@ -1646,7 +1180,7 @@ int odp_packet_align(odp_packet_t *pkt, uint32_t offset, uint32_t len, return rc; (void)odp_packet_move_data(*pkt, 0, shift, - _odp_packet_len(*pkt) - shift); + odp_packet_len(*pkt) - shift); (void)odp_packet_trunc_tail(pkt, shift, NULL, NULL); return 1; @@ -1656,20 +1190,20 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) { odp_packet_hdr_t *dst_hdr = packet_hdr(*dst); odp_packet_hdr_t *src_hdr = packet_hdr(src); - int dst_segs = dst_hdr->buf_hdr.segcount; - int src_segs = src_hdr->buf_hdr.segcount; - pool_t *dst_pool = dst_hdr->buf_hdr.pool_ptr; - pool_t *src_pool = src_hdr->buf_hdr.pool_ptr; + pool_t *dst_pool = _odp_pool_entry(dst_hdr->event_hdr.pool); + pool_t *src_pool = _odp_pool_entry(src_hdr->event_hdr.pool); uint32_t dst_len = dst_hdr->frame_len; uint32_t src_len = src_hdr->frame_len; - ODP_ASSERT(packet_ref_count(dst_hdr) == 1); + _ODP_ASSERT(odp_packet_has_ref(*dst) == 0); + + if (odp_unlikely(dst_len + src_len > dst_pool->max_len)) { + _ODP_ERR("concat would result oversized packet\n"); + return -1; + } - /* Do a copy if resulting packet would be out of segments or packets - * are from different pools or src is a reference. */ - if (odp_unlikely((dst_segs + src_segs) > CONFIG_PACKET_MAX_SEGS) || - odp_unlikely(dst_pool != src_pool) || - odp_unlikely(packet_ref_count(src_hdr) > 1)) { + /* Do a copy if packets are from different pools. */ + if (odp_unlikely(dst_pool != src_pool)) { if (odp_packet_extend_tail(dst, src_len, NULL, NULL) >= 0) { (void)odp_packet_copy_from_pkt(*dst, dst_len, src, 0, src_len); @@ -1682,11 +1216,14 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) return -1; } + if (odp_unlikely(dst_hdr->seg_count + src_hdr->seg_count > + PKT_MAX_SEGS)) + return -1; + add_all_segs(dst_hdr, src_hdr); - dst_hdr->frame_len = dst_len + src_len; - dst_hdr->unshared_len = dst_len + src_len; - dst_hdr->tailroom = src_hdr->tailroom; + dst_hdr->frame_len = dst_len + src_len; + dst_hdr->tailroom = src_hdr->tailroom; /* Data was not moved in memory */ return 0; @@ -1694,12 +1231,13 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) { - uint32_t pktlen = _odp_packet_len(*pkt); + uint32_t pktlen = odp_packet_len(*pkt); if (len >= pktlen || tail == NULL) return -1; - ODP_ASSERT(odp_packet_unshared_len(*pkt) >= len); + _ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); + *tail = odp_packet_copy_part(*pkt, len, pktlen - len, odp_packet_pool(*pkt)); @@ -1710,109 +1248,6 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) } /* - * References - */ - -static inline void packet_ref(odp_packet_hdr_t *pkt_hdr) -{ - uint32_t i; - odp_packet_hdr_t *hdr; - - do { - for (i = 0; i < pkt_hdr->buf_hdr.segcount; i++) { - hdr = pkt_hdr->buf_hdr.seg[i].hdr; - packet_ref_inc(hdr); - } - - pkt_hdr = pkt_hdr->ref_hdr; - } while (pkt_hdr); -} - -static inline odp_packet_t packet_splice(odp_packet_hdr_t *pkt_hdr, - uint32_t offset, - odp_packet_hdr_t *ref_hdr) -{ - /* Catch attempted references to stale handles in debug builds */ - ODP_ASSERT(packet_ref_count(pkt_hdr) > 0); - - /* Splicing is from the last section of src pkt */ - while (ref_hdr->ref_hdr) - ref_hdr = ref_hdr->ref_hdr; - - /* Find section where splice begins */ - while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { - offset -= (pkt_hdr->frame_len - pkt_hdr->ref_offset); - offset += (pkt_hdr->ref_hdr->frame_len - pkt_hdr->ref_len); - pkt_hdr = pkt_hdr->ref_hdr; - } - - ref_hdr->ref_hdr = pkt_hdr; - ref_hdr->ref_offset = offset; - ref_hdr->ref_len = pkt_hdr->frame_len; - - if (packet_ref_count(pkt_hdr) == 1 || offset < pkt_hdr->unshared_len) - pkt_hdr->unshared_len = offset; - - packet_ref(pkt_hdr); - return packet_handle(ref_hdr); -} - -odp_packet_t odp_packet_ref_static(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - packet_ref(pkt_hdr); - pkt_hdr->unshared_len = 0; - return pkt; -} - -odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) -{ - odp_packet_t hdr; - odp_packet_hdr_t *pkt_hdr; - - if (pkt == ODP_PACKET_INVALID) - return ODP_PACKET_INVALID; - - pkt_hdr = packet_hdr(pkt); - if (offset >= packet_len(pkt_hdr)) - return ODP_PACKET_INVALID; - - hdr = odp_packet_alloc(odp_packet_pool(pkt), 0); - - if (hdr == ODP_PACKET_INVALID) - return ODP_PACKET_INVALID; - - return packet_splice(pkt_hdr, offset, packet_hdr(hdr)); -} - -odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, - odp_packet_t hdr) -{ - odp_packet_hdr_t *pkt_hdr; - - if (pkt == ODP_PACKET_INVALID || - hdr == ODP_PACKET_INVALID || - pkt == hdr) - return ODP_PACKET_INVALID; - - ODP_ASSERT(odp_packet_has_ref(hdr) == 0); - - pkt_hdr = packet_hdr(pkt); - if (offset >= packet_len(pkt_hdr)) - return ODP_PACKET_INVALID; - - return packet_splice(pkt_hdr, offset, packet_hdr(hdr)); -} - -int odp_packet_has_ref(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - return pkt_hdr->ref_hdr != NULL || packet_ref_count(pkt_hdr) > 1; -} - -/* * * Copy * ******************************************************** @@ -1821,24 +1256,35 @@ int odp_packet_has_ref(odp_packet_t pkt) odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_pool_t pool) { - uint32_t pktlen = odp_packet_len(pkt); - odp_packet_t newpkt = odp_packet_alloc(pool, pktlen); + odp_packet_hdr_t *srchdr = packet_hdr(pkt); + uint32_t pktlen = srchdr->frame_len; + odp_packet_t newpkt; + int md_copy; - if (newpkt != ODP_PACKET_INVALID) { - if (_odp_packet_copy_md_to_packet(pkt, newpkt) || - odp_packet_copy_from_pkt(newpkt, 0, pkt, 0, pktlen)) { - odp_packet_free(newpkt); - newpkt = ODP_PACKET_INVALID; - } + md_copy = _odp_packet_copy_md_possible(pool, odp_packet_pool(pkt)); + if (odp_unlikely(md_copy < 0)) { + _ODP_ERR("Unable to copy packet metadata\n"); + return ODP_PACKET_INVALID; + } + + newpkt = odp_packet_alloc(pool, pktlen); + if (odp_unlikely(newpkt == ODP_PACKET_INVALID)) + return ODP_PACKET_INVALID; + + if (odp_unlikely(odp_packet_copy_from_pkt(newpkt, 0, pkt, 0, pktlen))) { + odp_packet_free(newpkt); + return ODP_PACKET_INVALID; } + _odp_packet_copy_md(packet_hdr(newpkt), srchdr, 1); + return newpkt; } odp_packet_t odp_packet_copy_part(odp_packet_t pkt, uint32_t offset, uint32_t len, odp_pool_t pool) { - uint32_t pktlen = _odp_packet_len(pkt); + uint32_t pktlen = odp_packet_len(pkt); odp_packet_t newpkt; if (offset >= pktlen || offset + len > pktlen) @@ -1851,56 +1297,6 @@ odp_packet_t odp_packet_copy_part(odp_packet_t pkt, uint32_t offset, return newpkt; } -int odp_packet_copy_to_mem(odp_packet_t pkt, uint32_t offset, - uint32_t len, void *dst) -{ - void *mapaddr; - uint32_t seglen = 0; /* GCC */ - uint32_t cpylen; - uint8_t *dstaddr = (uint8_t *)dst; - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (offset + len > packet_len(pkt_hdr)) - return -1; - - while (len > 0) { - mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); - cpylen = len > seglen ? seglen : len; - memcpy(dstaddr, mapaddr, cpylen); - offset += cpylen; - dstaddr += cpylen; - len -= cpylen; - } - - return 0; -} - -int odp_packet_copy_from_mem(odp_packet_t pkt, uint32_t offset, - uint32_t len, const void *src) -{ - void *mapaddr; - uint32_t seglen = 0; /* GCC */ - uint32_t cpylen; - const uint8_t *srcaddr = (const uint8_t *)src; - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (offset + len > packet_len(pkt_hdr)) - return -1; - - ODP_ASSERT(odp_packet_unshared_len(pkt) >= offset + len); - - while (len > 0) { - mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); - cpylen = len > seglen ? seglen : len; - memcpy(mapaddr, srcaddr, cpylen); - offset += cpylen; - srcaddr += cpylen; - len -= cpylen; - } - - return 0; -} - int odp_packet_copy_from_pkt(odp_packet_t dst, uint32_t dst_offset, odp_packet_t src, uint32_t src_offset, uint32_t len) @@ -1914,12 +1310,10 @@ int odp_packet_copy_from_pkt(odp_packet_t dst, uint32_t dst_offset, uint32_t src_seglen = 0; /* GCC */ int overlap; - if (dst_offset + len > packet_len(dst_hdr) || - src_offset + len > packet_len(src_hdr)) + if (dst_offset + len > dst_hdr->frame_len || + src_offset + len > src_hdr->frame_len) return -1; - ODP_ASSERT(odp_packet_unshared_len(dst) >= dst_offset + len); - overlap = (dst_hdr == src_hdr && ((dst_offset <= src_offset && dst_offset + len >= src_offset) || @@ -2003,7 +1397,7 @@ int _odp_packet_cmp_data(odp_packet_t pkt, uint32_t offset, int ret; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - ODP_ASSERT(offset + len <= pkt_hdr->frame_len); + _ODP_ASSERT(offset + len <= pkt_hdr->frame_len); while (len > 0) { mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); @@ -2025,69 +1419,210 @@ int _odp_packet_cmp_data(odp_packet_t pkt, uint32_t offset, * ******************************************************** * */ +static int packet_print_input_flags(odp_packet_hdr_t *hdr, char *str, int max) +{ + int len = 0; + + if (hdr->p.input_flags.l2) + len += _odp_snprint(&str[len], max - len, "l2 "); + if (hdr->p.input_flags.l3) + len += _odp_snprint(&str[len], max - len, "l3 "); + if (hdr->p.input_flags.l4) + len += _odp_snprint(&str[len], max - len, "l4 "); + if (hdr->p.input_flags.eth) + len += _odp_snprint(&str[len], max - len, "eth "); + if (hdr->p.input_flags.vlan) + len += _odp_snprint(&str[len], max - len, "vlan "); + if (hdr->p.input_flags.arp) + len += _odp_snprint(&str[len], max - len, "arp "); + if (hdr->p.input_flags.ipv4) + len += _odp_snprint(&str[len], max - len, "ipv4 "); + if (hdr->p.input_flags.ipv6) + len += _odp_snprint(&str[len], max - len, "ipv6 "); + if (hdr->p.input_flags.ipsec) + len += _odp_snprint(&str[len], max - len, "ipsec "); + if (hdr->p.input_flags.udp) + len += _odp_snprint(&str[len], max - len, "udp "); + if (hdr->p.input_flags.tcp) + len += _odp_snprint(&str[len], max - len, "tcp "); + if (hdr->p.input_flags.sctp) + len += _odp_snprint(&str[len], max - len, "sctp "); + if (hdr->p.input_flags.icmp) + len += _odp_snprint(&str[len], max - len, "icmp "); + + return len; +} void odp_packet_print(odp_packet_t pkt) { odp_packet_seg_t seg; - int max_len = 512; + int max_len = 4096; char str[max_len]; int len = 0; int n = max_len - 1; odp_packet_hdr_t *hdr = packet_hdr(pkt); - odp_buffer_t buf = packet_to_buffer(pkt); - - len += snprintf(&str[len], n - len, "Packet "); - len += odp_buffer_snprint(&str[len], n - len, buf); - len += snprintf(&str[len], n - len, " input_flags 0x%" PRIx64 "\n", - hdr->p.input_flags.all); - len += snprintf(&str[len], n - len, " error_flags 0x%" PRIx32 "\n", - hdr->p.error_flags.all); - len += snprintf(&str[len], n - len, - " output_flags 0x%" PRIx32 "\n", - hdr->p.output_flags.all); - len += snprintf(&str[len], n - len, - " l2_offset %" PRIu32 "\n", hdr->p.l2_offset); - len += snprintf(&str[len], n - len, - " l3_offset %" PRIu32 "\n", hdr->p.l3_offset); - len += snprintf(&str[len], n - len, - " l4_offset %" PRIu32 "\n", hdr->p.l4_offset); - len += snprintf(&str[len], n - len, - " frame_len %" PRIu32 "\n", packet_len(hdr)); - len += snprintf(&str[len], n - len, - " input %" PRIu64 "\n", - odp_pktio_to_u64(hdr->input)); - len += snprintf(&str[len], n - len, - " headroom %" PRIu32 "\n", - odp_packet_headroom(pkt)); - len += snprintf(&str[len], n - len, - " tailroom %" PRIu32 "\n", - odp_packet_tailroom(pkt)); - len += snprintf(&str[len], n - len, - " num_segs %i\n", odp_packet_num_segs(pkt)); + + len += _odp_snprint(&str[len], n - len, "Packet info\n"); + len += _odp_snprint(&str[len], n - len, "-----------\n"); + len += _odp_snprint(&str[len], n - len, " handle 0x%" PRIx64 "\n", + odp_packet_to_u64(pkt)); + len += _odp_snprint(&str[len], n - len, " pool index %u\n", hdr->event_hdr.index.pool); + len += _odp_snprint(&str[len], n - len, " buf index %u\n", + hdr->event_hdr.index.event); + len += _odp_snprint(&str[len], n - len, " ev subtype %i\n", hdr->event_hdr.subtype); + len += _odp_snprint(&str[len], n - len, " input_flags 0x%" PRIx64 "\n", + hdr->p.input_flags.all); + if (hdr->p.input_flags.all) { + len += _odp_snprint(&str[len], n - len, " "); + len += packet_print_input_flags(hdr, &str[len], n - len); + len += _odp_snprint(&str[len], n - len, "\n"); + } + len += _odp_snprint(&str[len], n - len, + " flags 0x%" PRIx32 "\n", hdr->p.flags.all_flags); + len += _odp_snprint(&str[len], n - len, + " cls_mark %" PRIu64 "\n", odp_packet_cls_mark(pkt)); + len += _odp_snprint(&str[len], n - len, + " user ptr %p\n", hdr->user_ptr); + len += _odp_snprint(&str[len], n - len, + " user area %p\n", hdr->uarea_addr); + len += _odp_snprint(&str[len], n - len, + " l2_offset %" PRIu32 "\n", hdr->p.l2_offset); + len += _odp_snprint(&str[len], n - len, + " l3_offset %" PRIu32 "\n", hdr->p.l3_offset); + len += _odp_snprint(&str[len], n - len, + " l4_offset %" PRIu32 "\n", hdr->p.l4_offset); + len += _odp_snprint(&str[len], n - len, + " frame_len %" PRIu32 "\n", hdr->frame_len); + len += _odp_snprint(&str[len], n - len, + " input %" PRIu64 "\n", odp_pktio_to_u64(hdr->input)); + len += _odp_snprint(&str[len], n - len, + " headroom %" PRIu32 "\n", odp_packet_headroom(pkt)); + len += _odp_snprint(&str[len], n - len, + " tailroom %" PRIu32 "\n", odp_packet_tailroom(pkt)); + len += _odp_snprint(&str[len], n - len, + " num_segs %i\n", odp_packet_num_segs(pkt)); seg = odp_packet_first_seg(pkt); - while (seg != ODP_PACKET_SEG_INVALID) { - len += snprintf(&str[len], n - len, - " seg_len %" PRIu32 "\n", - odp_packet_seg_data_len(pkt, seg)); + for (int seg_idx = 0; seg != ODP_PACKET_SEG_INVALID; seg_idx++) { + odp_packet_hdr_t *seg_hdr = packet_seg_to_hdr(seg); + char seg_str[max_len]; + int str_len; + + str_len = _odp_snprint(&seg_str[0], max_len, + " [%d] seg_len %-4" PRIu32 " seg_data %p ref_cnt %u\n", + seg_idx, odp_packet_seg_data_len(pkt, seg), + odp_packet_seg_data(pkt, seg), segment_ref(seg_hdr)); + + /* Prevent print buffer overflow */ + if (n - len - str_len < 10) { + len += _odp_snprint(&str[len], n - len, " ...\n"); + break; + } + len += _odp_snprint(&str[len], n - len, "%s", seg_str); seg = odp_packet_next_seg(pkt, seg); } - str[len] = '\0'; + _ODP_PRINT("%s\n", str); +} + +void odp_packet_print_data(odp_packet_t pkt, uint32_t offset, + uint32_t byte_len) +{ + odp_packet_hdr_t *hdr = packet_hdr(pkt); + uint32_t bytes_per_row = 16; + int num_rows = (byte_len + bytes_per_row - 1) / bytes_per_row; + int max_len = 256 + (3 * byte_len) + (3 * num_rows); + char str[max_len]; + int len = 0; + int n = max_len - 1; + uint32_t data_len = odp_packet_len(pkt); + pool_t *pool = _odp_pool_entry(hdr->event_hdr.pool); + + len += _odp_snprint(&str[len], n - len, "Packet data\n"); + len += _odp_snprint(&str[len], n - len, "-----------\n"); + len += _odp_snprint(&str[len], n - len, + " handle 0x%" PRIx64 "\n", odp_packet_to_u64(pkt)); + len += _odp_snprint(&str[len], n - len, + " pool index %" PRIu32 "\n", pool->pool_idx); + len += _odp_snprint(&str[len], n - len, + " buf index %" PRIu32 "\n", hdr->event_hdr.index.event); + len += _odp_snprint(&str[len], n - len, + " seg_count %" PRIu16 "\n", hdr->seg_count); + len += _odp_snprint(&str[len], n - len, + " data len %" PRIu32 "\n", data_len); + len += _odp_snprint(&str[len], n - len, + " data ptr %p\n", odp_packet_data(pkt)); + len += _odp_snprint(&str[len], n - len, + " print offset %" PRIu32 "\n", offset); + len += _odp_snprint(&str[len], n - len, + " print length %" PRIu32 "\n", byte_len); + + if (offset + byte_len > data_len) { + len += _odp_snprint(&str[len], n - len, " BAD OFFSET OR LEN\n"); + _ODP_PRINT("%s\n", str); + return; + } + + while (byte_len) { + uint32_t copy_len; + uint8_t data[bytes_per_row]; + uint32_t i; + + if (byte_len > bytes_per_row) + copy_len = bytes_per_row; + else + copy_len = byte_len; + + odp_packet_copy_to_mem(pkt, offset, copy_len, data); + + len += _odp_snprint(&str[len], n - len, " "); + + for (i = 0; i < copy_len; i++) + len += _odp_snprint(&str[len], n - len, " %02x", data[i]); + + len += _odp_snprint(&str[len], n - len, "\n"); - ODP_PRINT("\n%s\n", str); + byte_len -= copy_len; + offset += copy_len; + } + + _ODP_PRINT("%s\n", str); } int odp_packet_is_valid(odp_packet_t pkt) { - if (odp_buffer_is_valid(packet_to_buffer(pkt)) == 0) + odp_event_t ev; + + if (pkt == ODP_PACKET_INVALID) + return 0; + + ev = odp_packet_to_event(pkt); + + if (_odp_event_is_valid(ev) == 0) return 0; - if (odp_event_type(odp_packet_to_event(pkt)) != ODP_EVENT_PACKET) + if (odp_event_type(ev) != ODP_EVENT_PACKET) return 0; + if (odp_unlikely(_odp_packet_validate(pkt, _ODP_EV_PACKET_IS_VALID))) + return 0; + + switch (odp_event_subtype(ev)) { + case ODP_EVENT_PACKET_BASIC: + /* Fall through */ + case ODP_EVENT_PACKET_COMP: + /* Fall through */ + case ODP_EVENT_PACKET_CRYPTO: + /* Fall through */ + case ODP_EVENT_PACKET_IPSEC: + /* Fall through */ + break; + default: + return 0; + } + return 1; } @@ -2098,365 +1633,805 @@ int odp_packet_is_valid(odp_packet_t pkt) * */ -int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt) +static uint64_t packet_sum_partial(odp_packet_hdr_t *pkt_hdr, + uint32_t l3_offset, + uint32_t offset, + uint32_t len) { - odp_packet_hdr_t *srchdr = packet_hdr(srcpkt); - odp_packet_hdr_t *dsthdr = packet_hdr(dstpkt); + uint64_t sum = 0; - dsthdr->input = srchdr->input; - dsthdr->dst_queue = srchdr->dst_queue; - dsthdr->buf_hdr.buf_u64 = srchdr->buf_hdr.buf_u64; - if (dsthdr->buf_hdr.uarea_addr != NULL && - srchdr->buf_hdr.uarea_addr != NULL) - memcpy(dsthdr->buf_hdr.uarea_addr, - srchdr->buf_hdr.uarea_addr, - dsthdr->buf_hdr.uarea_size <= - srchdr->buf_hdr.uarea_size ? - dsthdr->buf_hdr.uarea_size : - srchdr->buf_hdr.uarea_size); + if (offset + len > pkt_hdr->frame_len) + return 0; - copy_packet_parser_metadata(srchdr, dsthdr); + while (len > 0) { + uint32_t seglen = 0; /* GCC */ + void *mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); - /* Metadata copied, but return indication of whether the packet - * user area was truncated in the process. Note this can only - * happen when copying between different pools. - */ - return dsthdr->buf_hdr.uarea_size < srchdr->buf_hdr.uarea_size; + if (seglen > len) + seglen = len; + + sum += chksum_partial(mapaddr, seglen, offset - l3_offset); + len -= seglen; + offset += seglen; + } + + return sum; } -/** - * Parser helper function for IPv4 - */ -static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, - uint32_t *offset, uint32_t frame_len) -{ - const _odp_ipv4hdr_t *ipv4 = (const _odp_ipv4hdr_t *)*parseptr; - uint8_t ver = _ODP_IPV4HDR_VER(ipv4->ver_ihl); - uint8_t ihl = _ODP_IPV4HDR_IHL(ipv4->ver_ihl); - uint16_t frag_offset; - uint32_t dstaddr = odp_be_to_cpu_32(ipv4->dst_addr); - uint32_t l3_len = odp_be_to_cpu_16(ipv4->tot_len); - - if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN) || - odp_unlikely(ver != 4) || - (l3_len > frame_len - *offset)) { - prs->error_flags.ip_err = 1; - return 0; +static inline uint16_t packet_sum(odp_packet_hdr_t *pkt_hdr, + uint32_t l3_offset, + uint32_t offset, + uint32_t len, + uint64_t sum) +{ + sum += packet_sum_partial(pkt_hdr, l3_offset, offset, len); + return chksum_finalize(sum); +} + +static uint32_t packet_sum_crc32c(odp_packet_hdr_t *pkt_hdr, + uint32_t offset, + uint32_t len, + uint32_t init_val) +{ + uint32_t sum = init_val; + + if (offset + len > pkt_hdr->frame_len) + return sum; + + while (len > 0) { + uint32_t seglen = 0; /* GCC */ + void *mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); + + if (seglen > len) + seglen = len; + + sum = odp_hash_crc32c(mapaddr, seglen, sum); + len -= seglen; + offset += seglen; } - *offset += ihl * 4; - *parseptr += ihl * 4; + return sum; +} - if (odp_unlikely(ihl > _ODP_IPV4HDR_IHL_MIN)) - prs->input_flags.ipopt = 1; +static inline int packet_ipv4_chksum(odp_packet_t pkt, + uint32_t offset, + _odp_ipv4hdr_t *ip, + odp_u16sum_t *chksum) +{ + unsigned int nleft = _ODP_IPV4HDR_IHL(ip->ver_ihl) * 4; + uint16_t buf[nleft / 2]; + int res; - /* A packet is a fragment if: - * "more fragments" flag is set (all fragments except the last) - * OR - * "fragment offset" field is nonzero (all fragments except the first) - */ - frag_offset = odp_be_to_cpu_16(ipv4->frag_offset); - if (odp_unlikely(_ODP_IPV4HDR_IS_FRAGMENT(frag_offset))) - prs->input_flags.ipfrag = 1; + if (odp_unlikely(nleft < sizeof(*ip))) + return -1; + ip->chksum = 0; + memcpy(buf, ip, sizeof(*ip)); + res = odp_packet_copy_to_mem(pkt, offset + sizeof(*ip), + nleft - sizeof(*ip), + buf + sizeof(*ip) / 2); + if (odp_unlikely(res < 0)) + return res; - /* Handle IPv4 broadcast / multicast */ - prs->input_flags.ip_bcast = (dstaddr == 0xffffffff); - prs->input_flags.ip_mcast = (dstaddr >> 28) == 0xd; + *chksum = ~chksum_finalize(chksum_partial(buf, nleft, 0)); - return ipv4->proto; + return 0; } +#define _ODP_IPV4HDR_CSUM_OFFSET ODP_OFFSETOF(_odp_ipv4hdr_t, chksum) +#define _ODP_IPV4ADDR_OFFSSET ODP_OFFSETOF(_odp_ipv4hdr_t, src_addr) +#define _ODP_IPV6ADDR_OFFSSET ODP_OFFSETOF(_odp_ipv6hdr_t, src_addr) +#define _ODP_IPV4HDR_CSUM_OFFSET ODP_OFFSETOF(_odp_ipv4hdr_t, chksum) +#define _ODP_UDP_LEN_OFFSET ODP_OFFSETOF(_odp_udphdr_t, length) +#define _ODP_UDP_CSUM_OFFSET ODP_OFFSETOF(_odp_udphdr_t, chksum) + /** - * Parser helper function for IPv6 + * Calculate and fill in IPv4 checksum + * + * @param pkt ODP packet + * + * @retval 0 on success + * @retval <0 on failure */ -static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, - uint32_t *offset, uint32_t frame_len, - uint32_t seg_len) -{ - const _odp_ipv6hdr_t *ipv6 = (const _odp_ipv6hdr_t *)*parseptr; - const _odp_ipv6hdr_ext_t *ipv6ext; - uint32_t dstaddr0 = odp_be_to_cpu_32(ipv6->dst_addr.u8[0]); - uint32_t l3_len = odp_be_to_cpu_16(ipv6->payload_len) + - _ODP_IPV6HDR_LEN; - - /* Basic sanity checks on IPv6 header */ - if ((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || - l3_len > frame_len - *offset) { - prs->error_flags.ip_err = 1; - return 0; - } +int _odp_packet_ipv4_chksum_insert(odp_packet_t pkt) +{ + uint32_t offset; + _odp_ipv4hdr_t ip; + odp_u16sum_t chksum; + int res; - /* IPv6 broadcast / multicast flags */ - prs->input_flags.ip_mcast = (dstaddr0 & 0xff000000) == 0xff000000; - prs->input_flags.ip_bcast = 0; + offset = odp_packet_l3_offset(pkt); + if (offset == ODP_PACKET_OFFSET_INVALID) + return -1; - /* Skip past IPv6 header */ - *offset += sizeof(_odp_ipv6hdr_t); - *parseptr += sizeof(_odp_ipv6hdr_t); + res = odp_packet_copy_to_mem(pkt, offset, sizeof(ip), &ip); + if (odp_unlikely(res < 0)) + return res; - /* Skip past any IPv6 extension headers */ - if (ipv6->next_hdr == _ODP_IPPROTO_HOPOPTS || - ipv6->next_hdr == _ODP_IPPROTO_ROUTE) { - prs->input_flags.ipopt = 1; + res = packet_ipv4_chksum(pkt, offset, &ip, &chksum); + if (odp_unlikely(res < 0)) + return res; - do { - ipv6ext = (const _odp_ipv6hdr_ext_t *)*parseptr; - uint16_t extlen = 8 + ipv6ext->ext_len * 8; + return odp_packet_copy_from_mem(pkt, + offset + _ODP_IPV4HDR_CSUM_OFFSET, + 2, &chksum); +} - *offset += extlen; - *parseptr += extlen; - } while ((ipv6ext->next_hdr == _ODP_IPPROTO_HOPOPTS || - ipv6ext->next_hdr == _ODP_IPPROTO_ROUTE) && - *offset < seg_len); +static int _odp_packet_tcp_udp_chksum_insert(odp_packet_t pkt, uint16_t proto) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t zero = 0; + uint64_t sum; + uint16_t l3_ver; + uint16_t chksum; + uint32_t chksum_offset; - if (*offset >= prs->l3_offset + - odp_be_to_cpu_16(ipv6->payload_len)) { - prs->error_flags.ip_err = 1; - return 0; - } + if (pkt_hdr->p.l3_offset == ODP_PACKET_OFFSET_INVALID) + return -1; + if (pkt_hdr->p.l4_offset == ODP_PACKET_OFFSET_INVALID) + return -1; - if (ipv6ext->next_hdr == _ODP_IPPROTO_FRAG) - prs->input_flags.ipfrag = 1; + odp_packet_copy_to_mem(pkt, pkt_hdr->p.l3_offset, 2, &l3_ver); - return ipv6ext->next_hdr; - } + if (_ODP_IPV4HDR_VER(l3_ver) == _ODP_IPV4) + sum = packet_sum_partial(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l3_offset + + _ODP_IPV4ADDR_OFFSSET, + 2 * _ODP_IPV4ADDR_LEN); + else + sum = packet_sum_partial(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l3_offset + + _ODP_IPV6ADDR_OFFSSET, + 2 * _ODP_IPV6ADDR_LEN); +#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN + sum += proto; +#else + sum += proto << 8; +#endif - if (odp_unlikely(ipv6->next_hdr == _ODP_IPPROTO_FRAG)) { - prs->input_flags.ipopt = 1; - prs->input_flags.ipfrag = 1; + if (proto == _ODP_IPPROTO_TCP) { + sum += odp_cpu_to_be_16(pkt_hdr->frame_len - + pkt_hdr->p.l4_offset); + chksum_offset = pkt_hdr->p.l4_offset + _ODP_UDP_CSUM_OFFSET; + } else { + sum += packet_sum_partial(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l4_offset + + _ODP_UDP_LEN_OFFSET, + 2); + chksum_offset = pkt_hdr->p.l4_offset + _ODP_UDP_CSUM_OFFSET; } + odp_packet_copy_from_mem(pkt, chksum_offset, 2, &zero); + + sum += packet_sum_partial(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l4_offset, + pkt_hdr->frame_len - + pkt_hdr->p.l4_offset); - return ipv6->next_hdr; + chksum = ~chksum_finalize(sum); + + if (proto == _ODP_IPPROTO_UDP && chksum == 0) + chksum = 0xffff; + + return odp_packet_copy_from_mem(pkt, + chksum_offset, + 2, &chksum); } /** - * Parser helper function for TCP + * Calculate and fill in TCP checksum + * + * @param pkt ODP packet + * + * @retval 0 on success + * @retval <0 on failure */ -static inline void parse_tcp(packet_parser_t *prs, - const uint8_t **parseptr, uint32_t *offset) +int _odp_packet_tcp_chksum_insert(odp_packet_t pkt) { - const _odp_tcphdr_t *tcp = (const _odp_tcphdr_t *)*parseptr; - - if (tcp->hl < sizeof(_odp_tcphdr_t) / sizeof(uint32_t)) - prs->error_flags.tcp_err = 1; - else if ((uint32_t)tcp->hl * 4 > sizeof(_odp_tcphdr_t)) - prs->input_flags.tcpopt = 1; - - if (offset) - *offset += (uint32_t)tcp->hl * 4; - *parseptr += (uint32_t)tcp->hl * 4; + return _odp_packet_tcp_udp_chksum_insert(pkt, _ODP_IPPROTO_TCP); } /** - * Parser helper function for UDP + * Calculate and fill in UDP checksum + * + * @param pkt ODP packet + * + * @retval 0 on success + * @retval <0 on failure */ -static inline void parse_udp(packet_parser_t *prs, - const uint8_t **parseptr, uint32_t *offset) +int _odp_packet_udp_chksum_insert(odp_packet_t pkt) { - const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr; - uint32_t udplen = odp_be_to_cpu_16(udp->length); - - if (odp_unlikely(udplen < sizeof(_odp_udphdr_t))) - prs->error_flags.udp_err = 1; - - if (offset) - *offset += sizeof(_odp_udphdr_t); - *parseptr += sizeof(_odp_udphdr_t); + return _odp_packet_tcp_udp_chksum_insert(pkt, _ODP_IPPROTO_UDP); } /** - * Parse common packet headers up to given layer + * Calculate and fill in SCTP checksum + * + * @param pkt ODP packet * - * The function expects at least PACKET_PARSE_SEG_LEN bytes of data to be - * available from the ptr. + * @retval 0 on success + * @retval <0 on failure */ -int packet_parse_common(packet_parser_t *prs, const uint8_t *ptr, - uint32_t frame_len, uint32_t seg_len, - odp_pktio_parser_layer_t layer) +int _odp_packet_sctp_chksum_insert(odp_packet_t pkt) { - uint32_t offset; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t sum; + + if (pkt_hdr->p.l4_offset == ODP_PACKET_OFFSET_INVALID) + return -1; + + sum = 0; + odp_packet_copy_from_mem(pkt, pkt_hdr->p.l4_offset + 8, 4, &sum); + sum = ~packet_sum_crc32c(pkt_hdr, pkt_hdr->p.l4_offset, + pkt_hdr->frame_len - pkt_hdr->p.l4_offset, + ~0); + return odp_packet_copy_from_mem(pkt, pkt_hdr->p.l4_offset + 8, 4, &sum); +} + +int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, + odp_pktin_config_opt_t opt, uint64_t l4_part_sum) +{ + /* UDP chksum == 0 case is covered in parse_udp() */ + if (opt.bit.udp_chksum && + pkt_hdr->p.input_flags.udp && + !pkt_hdr->p.input_flags.ipfrag && + !pkt_hdr->p.input_flags.udp_chksum_zero) { + uint16_t sum = ~packet_sum(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l4_offset, + pkt_hdr->frame_len - + pkt_hdr->p.l4_offset, + l4_part_sum); + + pkt_hdr->p.input_flags.l4_chksum_done = 1; + if (sum != 0) { + pkt_hdr->p.flags.l4_chksum_err = 1; + pkt_hdr->p.flags.udp_err = 1; + _ODP_DBG("UDP chksum fail (%x)!\n", sum); + if (opt.bit.drop_udp_err) + return -1; + } + } + + if (opt.bit.tcp_chksum && + pkt_hdr->p.input_flags.tcp && + !pkt_hdr->p.input_flags.ipfrag) { + uint16_t sum = ~packet_sum(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l4_offset, + pkt_hdr->frame_len - + pkt_hdr->p.l4_offset, + l4_part_sum); + + pkt_hdr->p.input_flags.l4_chksum_done = 1; + if (sum != 0) { + pkt_hdr->p.flags.l4_chksum_err = 1; + pkt_hdr->p.flags.tcp_err = 1; + _ODP_DBG("TCP chksum fail (%x)!\n", sum); + if (opt.bit.drop_tcp_err) + return -1; + } + } + + if (opt.bit.sctp_chksum && + pkt_hdr->p.input_flags.sctp && + !pkt_hdr->p.input_flags.ipfrag) { + uint32_t seg_len = 0; + _odp_sctphdr_t hdr_copy; + uint32_t sum = ~packet_sum_crc32c(pkt_hdr, + pkt_hdr->p.l4_offset + + _ODP_SCTPHDR_LEN, + pkt_hdr->frame_len - + pkt_hdr->p.l4_offset - + _ODP_SCTPHDR_LEN, + l4_part_sum); + _odp_sctphdr_t *sctp = packet_map(pkt_hdr, + pkt_hdr->p.l4_offset, + &seg_len, NULL); + if (odp_unlikely(seg_len < sizeof(*sctp))) { + odp_packet_t pkt = packet_handle(pkt_hdr); + + sctp = &hdr_copy; + odp_packet_copy_to_mem(pkt, pkt_hdr->p.l4_offset, + sizeof(*sctp), sctp); + } + pkt_hdr->p.input_flags.l4_chksum_done = 1; + if (sum != sctp->chksum) { + pkt_hdr->p.flags.l4_chksum_err = 1; + pkt_hdr->p.flags.sctp_err = 1; + _ODP_DBG("SCTP chksum fail (%x/%x)!\n", sum, sctp->chksum); + if (opt.bit.drop_sctp_err) + return -1; + } + } + + return pkt_hdr->p.flags.all.error != 0; +} + +int odp_packet_parse(odp_packet_t pkt, uint32_t offset, + const odp_packet_parse_param_t *param) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + const uint8_t *data; + uint32_t seg_len; + uint32_t seg_end; + uint32_t packet_len = pkt_hdr->frame_len; + odp_proto_t proto = param->proto; + odp_proto_layer_t layer = param->last_layer; + int ret; uint16_t ethtype; - const uint8_t *parseptr; - uint8_t ip_proto; - const _odp_ethhdr_t *eth; - uint16_t macaddr0, macaddr2, macaddr4; - const _odp_vlanhdr_t *vlan; + uint64_t l4_part_sum = 0; + const uint32_t min_seglen = PARSE_ETH_BYTES + PARSE_L3_L4_BYTES; + uint8_t buf[min_seglen]; + odp_pktin_config_opt_t opt; - if (layer == ODP_PKTIO_PARSER_LAYER_NONE) - return 0; + if (proto == ODP_PROTO_NONE || layer == ODP_PROTO_LAYER_NONE) + return -1; + + data = packet_map(pkt_hdr, offset, &seg_len, NULL); + + if (data == NULL) + return -1; + + /* + * We must not have a packet segment boundary within the parsed + * packet data range. Copy enough data to a temporary buffer for + * parsing if necessary. + */ + if (odp_unlikely(pkt_hdr->seg_count > 1) && + odp_unlikely(seg_len < min_seglen)) { + seg_len = min_seglen; + if (seg_len > packet_len - offset) + seg_len = packet_len - offset; + odp_packet_copy_to_mem(pkt, offset, seg_len, buf); + data = buf; + } + + seg_end = offset + seg_len; /* one past the maximum offset */ - /* We only support Ethernet for now */ - prs->input_flags.eth = 1; - /* Assume valid L2 header, no CRC/FCS check in SW */ - prs->input_flags.l2 = 1; - /* Detect jumbo frames */ - if (frame_len > _ODP_ETH_LEN_MAX) - prs->input_flags.jumbo = 1; - - offset = sizeof(_odp_ethhdr_t); - eth = (const _odp_ethhdr_t *)ptr; - - /* Handle Ethernet broadcast/multicast addresses */ - macaddr0 = odp_be_to_cpu_16(*((const uint16_t *)(const void *)eth)); - prs->input_flags.eth_mcast = (macaddr0 & 0x0100) == 0x0100; - - if (macaddr0 == 0xffff) { - macaddr2 = - odp_be_to_cpu_16(*((const uint16_t *) - (const void *)eth + 1)); - macaddr4 = - odp_be_to_cpu_16(*((const uint16_t *) - (const void *)eth + 2)); - prs->input_flags.eth_bcast = - (macaddr2 == 0xffff) && (macaddr4 == 0xffff); + /* Reset parser flags, keep other flags */ + packet_parse_reset(pkt_hdr, 0); + + if (proto == ODP_PROTO_ETH) { + /* Assume valid L2 header, no CRC/FCS check in SW */ + pkt_hdr->p.l2_offset = offset; + + ethtype = _odp_parse_eth(&pkt_hdr->p, &data, &offset, packet_len); + } else if (proto == ODP_PROTO_IPV4) { + ethtype = _ODP_ETHTYPE_IPV4; + } else if (proto == ODP_PROTO_IPV6) { + ethtype = _ODP_ETHTYPE_IPV6; } else { - prs->input_flags.eth_bcast = 0; + ethtype = 0; /* Invalid */ } - /* Get Ethertype */ - ethtype = odp_be_to_cpu_16(eth->type); - parseptr = (const uint8_t *)(eth + 1); + opt.all_bits = 0; + opt.bit.ipv4_chksum = param->chksums.chksum.ipv4; + opt.bit.udp_chksum = param->chksums.chksum.udp; + opt.bit.tcp_chksum = param->chksums.chksum.tcp; + opt.bit.sctp_chksum = param->chksums.chksum.sctp; - /* Check for SNAP vs. DIX */ - if (ethtype < _ODP_ETH_LEN_MAX) { - prs->input_flags.snap = 1; - if (ethtype > frame_len - offset) { - prs->error_flags.snap_len = 1; - goto parse_exit; - } - ethtype = odp_be_to_cpu_16(*((const uint16_t *)(uintptr_t) - (parseptr + 6))); - offset += 8; - parseptr += 8; + ret = _odp_packet_parse_common_l3_l4(&pkt_hdr->p, data, offset, + packet_len, seg_end, layer, + ethtype, &l4_part_sum, opt); + + if (ret) + return -1; + + if (layer >= ODP_PROTO_LAYER_L4) { + ret = _odp_packet_l4_chksum(pkt_hdr, opt, l4_part_sum); + if (ret) + return -1; } - /* Parse the VLAN header(s), if present */ - if (ethtype == _ODP_ETHTYPE_VLAN_OUTER) { - prs->input_flags.vlan_qinq = 1; - prs->input_flags.vlan = 1; + return 0; +} + +int odp_packet_parse_multi(const odp_packet_t pkt[], const uint32_t offset[], + int num, const odp_packet_parse_param_t *param) +{ + int i; - vlan = (const _odp_vlanhdr_t *)parseptr; - ethtype = odp_be_to_cpu_16(vlan->type); - offset += sizeof(_odp_vlanhdr_t); - parseptr += sizeof(_odp_vlanhdr_t); + for (i = 0; i < num; i++) + if (odp_packet_parse(pkt[i], offset[i], param)) + return i; + + return num; +} + +void odp_packet_parse_result(odp_packet_t pkt, + odp_packet_parse_result_t *result) +{ + /* TODO: optimize to single word copy when packet header stores bits + * directly into odp_packet_parse_result_flag_t */ + result->flag.all = 0; + result->flag.has_error = odp_packet_has_error(pkt); + result->flag.has_l2_error = odp_packet_has_l2_error(pkt); + result->flag.has_l3_error = odp_packet_has_l3_error(pkt); + result->flag.has_l4_error = odp_packet_has_l4_error(pkt); + result->flag.has_l2 = odp_packet_has_l2(pkt); + result->flag.has_l3 = odp_packet_has_l3(pkt); + result->flag.has_l4 = odp_packet_has_l4(pkt); + result->flag.has_eth = odp_packet_has_eth(pkt); + result->flag.has_eth_bcast = odp_packet_has_eth_bcast(pkt); + result->flag.has_eth_mcast = odp_packet_has_eth_mcast(pkt); + result->flag.has_jumbo = odp_packet_has_jumbo(pkt); + result->flag.has_vlan = odp_packet_has_vlan(pkt); + result->flag.has_vlan_qinq = odp_packet_has_vlan_qinq(pkt); + result->flag.has_arp = odp_packet_has_arp(pkt); + result->flag.has_ipv4 = odp_packet_has_ipv4(pkt); + result->flag.has_ipv6 = odp_packet_has_ipv6(pkt); + result->flag.has_ip_bcast = odp_packet_has_ip_bcast(pkt); + result->flag.has_ip_mcast = odp_packet_has_ip_mcast(pkt); + result->flag.has_ipfrag = odp_packet_has_ipfrag(pkt); + result->flag.has_ipopt = odp_packet_has_ipopt(pkt); + result->flag.has_ipsec = odp_packet_has_ipsec(pkt); + result->flag.has_udp = odp_packet_has_udp(pkt); + result->flag.has_tcp = odp_packet_has_tcp(pkt); + result->flag.has_sctp = odp_packet_has_sctp(pkt); + result->flag.has_icmp = odp_packet_has_icmp(pkt); + + result->packet_len = odp_packet_len(pkt); + result->l2_offset = odp_packet_l2_offset(pkt); + result->l3_offset = odp_packet_l3_offset(pkt); + result->l4_offset = odp_packet_l4_offset(pkt); + result->l3_chksum_status = odp_packet_l3_chksum_status(pkt); + result->l4_chksum_status = odp_packet_l4_chksum_status(pkt); + result->l2_type = odp_packet_l2_type(pkt); + result->l3_type = odp_packet_l3_type(pkt); + result->l4_type = odp_packet_l4_type(pkt); +} + +void odp_packet_parse_result_multi(const odp_packet_t pkt[], + odp_packet_parse_result_t *result[], + int num) +{ + int i; + + for (i = 0; i < num; i++) + odp_packet_parse_result(pkt[i], result[i]); +} + +uint64_t odp_packet_to_u64(odp_packet_t hdl) +{ + return _odp_pri(hdl); +} + +uint64_t odp_packet_seg_to_u64(odp_packet_seg_t hdl) +{ + return _odp_pri(hdl); +} + +odp_packet_t odp_packet_ref_static(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + while (pkt_hdr != NULL) { + segment_ref_inc(pkt_hdr); + pkt_hdr = pkt_hdr->seg_next; } - if (ethtype == _ODP_ETHTYPE_VLAN) { - prs->input_flags.vlan = 1; - vlan = (const _odp_vlanhdr_t *)parseptr; - ethtype = odp_be_to_cpu_16(vlan->type); - offset += sizeof(_odp_vlanhdr_t); - parseptr += sizeof(_odp_vlanhdr_t); + return pkt; +} + +odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) +{ + odp_packet_t new; + int ret; + + new = odp_packet_copy(pkt, odp_packet_pool(pkt)); + + if (new == ODP_PACKET_INVALID) { + _ODP_ERR("copy failed\n"); + return ODP_PACKET_INVALID; } - if (layer == ODP_PKTIO_PARSER_LAYER_L2) - return prs->error_flags.all != 0; + ret = odp_packet_trunc_head(&new, offset, NULL, NULL); - /* Set l3_offset+flag only for known ethtypes */ - prs->l3_offset = offset; - prs->input_flags.l3 = 1; + if (ret < 0) { + _ODP_ERR("trunk_head failed\n"); + odp_packet_free(new); + return ODP_PACKET_INVALID; + } - /* Parse Layer 3 headers */ - switch (ethtype) { - case _ODP_ETHTYPE_IPV4: - prs->input_flags.ipv4 = 1; - ip_proto = parse_ipv4(prs, &parseptr, &offset, frame_len); - break; + return new; +} - case _ODP_ETHTYPE_IPV6: - prs->input_flags.ipv6 = 1; - ip_proto = parse_ipv6(prs, &parseptr, &offset, frame_len, - seg_len); - break; +odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, + odp_packet_t hdr) +{ + odp_packet_t ref; + int ret; - case _ODP_ETHTYPE_ARP: - prs->input_flags.arp = 1; - ip_proto = 255; /* Reserved invalid by IANA */ - break; + ref = odp_packet_ref(pkt, offset); - default: - prs->input_flags.l3 = 0; - prs->l3_offset = ODP_PACKET_OFFSET_INVALID; - ip_proto = 255; /* Reserved invalid by IANA */ + if (ref == ODP_PACKET_INVALID) { + _ODP_DBG("reference create failed\n"); + return ODP_PACKET_INVALID; + } + + ret = odp_packet_concat(&hdr, ref); + + if (ret < 0) { + _ODP_DBG("concat failed\n"); + odp_packet_free(ref); + return ODP_PACKET_INVALID; } - if (layer == ODP_PKTIO_PARSER_LAYER_L3) - return prs->error_flags.all != 0; + return hdr; +} - /* Set l4_offset+flag only for known ip_proto */ - prs->l4_offset = offset; - prs->input_flags.l4 = 1; +int odp_packet_has_ref(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t ref_cnt; - /* Parse Layer 4 headers */ - switch (ip_proto) { - case _ODP_IPPROTO_ICMPv4: - /* Fall through */ + while (pkt_hdr != NULL) { + ref_cnt = segment_ref(pkt_hdr); - case _ODP_IPPROTO_ICMPv6: - prs->input_flags.icmp = 1; - break; + if (is_multi_ref(ref_cnt)) + return 1; - case _ODP_IPPROTO_TCP: - if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len)) - return -1; - prs->input_flags.tcp = 1; - parse_tcp(prs, &parseptr, NULL); - break; + pkt_hdr = pkt_hdr->seg_next; + } - case _ODP_IPPROTO_UDP: - if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len)) - return -1; - prs->input_flags.udp = 1; - parse_udp(prs, &parseptr, NULL); - break; + return 0; +} - case _ODP_IPPROTO_AH: - prs->input_flags.ipsec = 1; - prs->input_flags.ipsec_ah = 1; - break; +void odp_packet_lso_request_clr(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - case _ODP_IPPROTO_ESP: - prs->input_flags.ipsec = 1; - prs->input_flags.ipsec_esp = 1; - break; + pkt_hdr->p.flags.lso = 0; +} - case _ODP_IPPROTO_SCTP: - prs->input_flags.sctp = 1; - break; +int odp_packet_has_lso_request(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - default: - prs->input_flags.l4 = 0; - prs->l4_offset = ODP_PACKET_OFFSET_INVALID; + return pkt_hdr->p.flags.lso; +} + +uint32_t odp_packet_payload_offset(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + if (pkt_hdr->p.flags.payload_off) + return pkt_hdr->payload_offset; + + return ODP_PACKET_OFFSET_INVALID; +} + +int odp_packet_payload_offset_set(odp_packet_t pkt, uint32_t offset) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + pkt_hdr->p.flags.payload_off = 1; + pkt_hdr->payload_offset = offset; + + return 0; +} + +void odp_packet_aging_tmo_set(odp_packet_t pkt, uint64_t tmo_ns) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + pkt_hdr->p.flags.tx_aging = tmo_ns ? 1 : 0; + pkt_hdr->tx_aging_ns = tmo_ns; +} + +uint64_t odp_packet_aging_tmo(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + return pkt_hdr->p.flags.tx_aging ? pkt_hdr->tx_aging_ns : 0; +} + +int odp_packet_tx_compl_request(odp_packet_t pkt, const odp_packet_tx_compl_opt_t *opt) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + switch (opt->mode) { + case ODP_PACKET_TX_COMPL_DISABLED: + pkt_hdr->p.flags.tx_compl_ev = 0; + pkt_hdr->p.flags.tx_compl_poll = 0; break; + case ODP_PACKET_TX_COMPL_EVENT: + _ODP_ASSERT(opt->queue != ODP_QUEUE_INVALID); + pkt_hdr->p.flags.tx_compl_ev = 1; + pkt_hdr->p.flags.tx_compl_poll = 0; + pkt_hdr->dst_queue = opt->queue; + break; + case ODP_PACKET_TX_COMPL_POLL: + pkt_hdr->p.flags.tx_compl_ev = 0; + pkt_hdr->p.flags.tx_compl_poll = 1; + pkt_hdr->tx_compl_id = opt->compl_id; + break; + default: + _ODP_ERR("Bad TX completion mode: %i\n", opt->mode); + return -1; } -parse_exit: - return prs->error_flags.all != 0; + + return 0; } -/** - * Simple packet parser - */ -int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, - odp_pktio_parser_layer_t layer) +int odp_packet_has_tx_compl_request(odp_packet_t pkt) { - uint32_t seg_len = packet_first_seg_len(pkt_hdr); - void *base = packet_data(pkt_hdr); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - return packet_parse_common(&pkt_hdr->p, base, pkt_hdr->frame_len, - seg_len, layer); + return pkt_hdr->p.flags.tx_compl_ev || pkt_hdr->p.flags.tx_compl_poll; } -uint64_t odp_packet_to_u64(odp_packet_t hdl) +void odp_packet_tx_compl_free(odp_packet_tx_compl_t tx_compl) { - return _odp_pri(hdl); + if (odp_unlikely(tx_compl == ODP_PACKET_TX_COMPL_INVALID)) { + _ODP_ERR("Bad TX completion event handle\n"); + return; + } + + odp_buffer_free((odp_buffer_t)tx_compl); } -uint64_t odp_packet_seg_to_u64(odp_packet_seg_t hdl) +void *odp_packet_tx_compl_user_ptr(odp_packet_tx_compl_t tx_compl) { - return _odp_pri(hdl); + if (odp_unlikely(tx_compl == ODP_PACKET_TX_COMPL_INVALID)) { + _ODP_ERR("Bad TX completion event handle\n"); + return NULL; + } + + _odp_pktio_tx_compl_t *data = odp_buffer_addr((odp_buffer_t)tx_compl); + + return (void *)(uintptr_t)data->user_ptr; } -/* Include non-inlined versions of API functions */ -#if ODP_ABI_COMPAT == 1 -#include <odp/api/plat/packet_inlines_api.h> -#endif +int odp_packet_tx_compl_done(odp_pktio_t pktio, uint32_t compl_id) +{ + return odp_atomic_load_acq_u32(&get_pktio_entry(pktio)->tx_compl_status[compl_id]); +} + +void odp_packet_free_ctrl_set(odp_packet_t pkt, odp_packet_free_ctrl_t ctrl) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + if (ctrl == ODP_PACKET_FREE_CTRL_DONT_FREE) + pkt_hdr->p.flags.free_ctrl = 1; + else + pkt_hdr->p.flags.free_ctrl = 0; +} + +odp_packet_free_ctrl_t odp_packet_free_ctrl(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + if (pkt_hdr->p.flags.free_ctrl) + return ODP_PACKET_FREE_CTRL_DONT_FREE; + + return ODP_PACKET_FREE_CTRL_DISABLED; +} + +odp_packet_reass_status_t +odp_packet_reass_status(odp_packet_t pkt) +{ + (void)pkt; + return ODP_PACKET_REASS_NONE; +} + +int odp_packet_reass_info(odp_packet_t pkt, odp_packet_reass_info_t *info) +{ + (void)pkt; + (void)info; + return -1; +} + +int +odp_packet_reass_partial_state(odp_packet_t pkt, odp_packet_t frags[], + odp_packet_reass_partial_state_t *res) +{ + (void)pkt; + (void)frags; + (void)res; + return -ENOTSUP; +} + +uint32_t odp_packet_disassemble(odp_packet_t pkt, odp_packet_buf_t pkt_buf[], uint32_t num) +{ + uint32_t i; + odp_packet_seg_t seg; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + pool_t *pool = _odp_pool_entry(pkt_hdr->event_hdr.pool); + uint32_t num_segs = odp_packet_num_segs(pkt); + + if (odp_unlikely(pool->type != ODP_POOL_PACKET)) { + _ODP_ERR("Not a packet pool\n"); + return 0; + } + + if (odp_unlikely(pool->pool_ext == 0)) { + _ODP_ERR("Not an external memory pool\n"); + return 0; + } + + if (odp_unlikely(num < num_segs)) { + _ODP_ERR("Not enough buffer handles %u. Packet has %u segments.\n", num, num_segs); + return 0; + } + + seg = odp_packet_first_seg(pkt); + + for (i = 0; i < num_segs; i++) { + pkt_buf[i] = (odp_packet_buf_t)(uintptr_t)packet_seg_to_hdr(seg); + seg = odp_packet_next_seg(pkt, seg); + } + + return num_segs; +} + +odp_packet_t odp_packet_reassemble(odp_pool_t pool_hdl, odp_packet_buf_t pkt_buf[], uint32_t num) +{ + uint32_t i, data_len, tailroom; + odp_packet_hdr_t *cur_seg, *next_seg; + odp_packet_hdr_t *pkt_hdr = (odp_packet_hdr_t *)(uintptr_t)pkt_buf[0]; + uint32_t headroom = odp_packet_buf_data_offset(pkt_buf[0]); + + pool_t *pool = _odp_pool_entry(pool_hdl); + + if (odp_unlikely(pool->type != ODP_POOL_PACKET)) { + _ODP_ERR("Not a packet pool\n"); + return ODP_PACKET_INVALID; + } + + if (odp_unlikely(pool->pool_ext == 0)) { + _ODP_ERR("Not an external memory pool\n"); + return ODP_PACKET_INVALID; + } + + if (odp_unlikely(num == 0)) { + _ODP_ERR("Bad number of buffers: %u\n", num); + return ODP_PACKET_INVALID; + } + + cur_seg = pkt_hdr; + data_len = 0; + + for (i = 0; i < num; i++) { + next_seg = NULL; + if (i < num - 1) + next_seg = (odp_packet_hdr_t *)(uintptr_t)pkt_buf[i + 1]; + + data_len += cur_seg->seg_len; + cur_seg->seg_next = next_seg; + cur_seg = next_seg; + } + + tailroom = pool->ext_param.pkt.buf_size - sizeof(odp_packet_hdr_t); + tailroom -= pool->ext_param.pkt.app_header_size; + tailroom -= odp_packet_buf_data_len(pkt_buf[num - 1]); + tailroom -= pool->trailer_size; + + pkt_hdr->seg_count = num; + pkt_hdr->frame_len = data_len; + pkt_hdr->headroom = headroom; + pkt_hdr->tailroom = tailroom; + + /* Reset metadata */ + pkt_hdr->event_hdr.subtype = ODP_EVENT_PACKET_BASIC; + pkt_hdr->input = ODP_PKTIO_INVALID; + packet_parse_reset(pkt_hdr, 1); + + return packet_handle(pkt_hdr); +} + +void odp_packet_proto_stats_request(odp_packet_t pkt, odp_packet_proto_stats_opt_t *opt) +{ + (void)pkt; + (void)opt; +} + +odp_proto_stats_t odp_packet_proto_stats(odp_packet_t pkt) +{ + (void)pkt; + + return ODP_PROTO_STATS_INVALID; +} |