diff options
author | Maxim Uvarov <maxim.uvarov@linaro.org> | 2017-09-14 17:56:51 +0300 |
---|---|---|
committer | Maxim Uvarov <maxim.uvarov@linaro.org> | 2017-09-14 22:15:34 +0300 |
commit | 3c9248f5c98b194d91712fafed4bf3b21327f2a8 (patch) | |
tree | 9689c46621e352b0a34e097784af62dc71a2ae03 | |
parent | 6b6253c30f88c80bf632436ff06c1b000860a2f1 (diff) | |
parent | 91ef9f7e224056af351cbaf99ccfa98ee815460d (diff) |
Merge branch 'master' into api-next
Signed-off-by: Maxim Uvarov <maxim.uvarov@linaro.org>
-rw-r--r-- | .travis.yml | 134 | ||||
-rw-r--r-- | include/odp/arch/default/api/abi/packet.h | 2 | ||||
-rw-r--r-- | platform/linux-generic/include/odp/api/plat/packet_inlines.h | 44 | ||||
-rw-r--r-- | platform/linux-generic/include/odp/api/plat/packet_inlines_api.h | 16 | ||||
-rw-r--r-- | platform/linux-generic/include/odp/api/plat/packet_types.h | 38 | ||||
-rw-r--r-- | platform/linux-generic/include/odp_buffer_internal.h | 71 | ||||
-rw-r--r-- | platform/linux-generic/include/odp_packet_internal.h | 133 | ||||
-rw-r--r-- | platform/linux-generic/odp_packet.c | 1519 | ||||
-rw-r--r-- | platform/linux-generic/odp_pool.c | 8 | ||||
-rw-r--r-- | test/common_plat/validation/api/packet/packet.c | 109 |
10 files changed, 978 insertions, 1096 deletions
diff --git a/.travis.yml b/.travis.yml index 0b5fb87d0..8cf6ed22b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,6 +69,10 @@ env: - CONF="--enable-schedule-iquery" - CONF="--enable-schedule-scalable" - CONF="--enable-dpdk-zero-copy" + - CROSS_ARCH="arm64" + - CROSS_ARCH="armhf" CFLAGS="-march=armv7-a" + - CROSS_ARCH="powerpc" + - CROSS_ARCH="i386" before_install: @@ -79,13 +83,36 @@ before_install: BUILD_GNU_TYPE=`dpkg-architecture -a"$CROSS_ARCH" -qDEB_BUILD_GNU_TYPE` ; CROSS_GNU_TYPE=`dpkg-architecture -a"$CROSS_ARCH" -qDEB_HOST_GNU_TYPE` ; CROSS_MULTIARCH=`dpkg-architecture -a"$CROSS_ARCH" -qDEB_HOST_MULTIARCH` ; - CROSS="--host="$CROSS_GNU_TYPE" --build="$BUILD_GNU_TYPE" --prefix=/usr --includedir=/usr/include/"$CROSS_MULTIARCH" --libdir=/usr/lib/"$CROSS_MULTIARCH" --libexecdir=/usr/lib/"$CROSS_MULTIARCH"" ; + CROSS="--host="$CROSS_GNU_TYPE" --build="$BUILD_GNU_TYPE"" ; sudo dpkg --add-architecture "$CROSS_ARCH" ; sudo -E apt-add-repository -y "deb http://ports.ubuntu.com trusty main" ; sudo -E apt-add-repository -y "deb http://ports.ubuntu.com trusty-updates main" ; sudo -E apt-get -y update || true ; - sudo -E apt-get -y --no-install-suggests --no-install-recommends --force-yes install build-essential gcc-"$CROSS_GNU_TYPE" pkg-config-"$CROSS_GNU_TYPE" ; - sudo -E apt-get -y --no-install-suggests --no-install-recommends --force-yes install libc6-dev:"$CROSS_ARCH" libconfig-dev:"$CROSS_ARCH" libssl-dev:"$CROSS_ARCH" zlib1g-dev:"$CROSS_ARCH" libconfig-dev:"$CROSS_ARCH" ; + sudo -E apt-get -y --no-install-suggests --no-install-recommends --force-yes install build-essential ; + if [ "$CROSS_ARCH" = "i386" ] ; + then + sudo -E apt-get -y --no-install-suggests --no-install-recommends --force-yes install g++-multilib ; + else + sudo -E apt-get -y --no-install-suggests --no-install-recommends --force-yes install g++-"$CROSS_GNU_TYPE" ; + fi ; + sudo -E apt-get -y --no-install-suggests --no-install-recommends --force-yes install libc6-dev:"$CROSS_ARCH" libssl-dev:"$CROSS_ARCH" zlib1g-dev:"$CROSS_ARCH" libconfig-dev:"$CROSS_ARCH" libstdc++-4.8-dev:"$CROSS_ARCH"; + export PKG_CONFIG_PATH=/usr/lib/${CROSS_MULTIARCH}/pkgconfig:/usr/${CROSS_MULTIARCH}/lib/pkgconfig ; + fi + - if [ "${CC#clang}" != "${CC}" ] ; + then + if [ -n "$CROSS_ARCH" ] ; + then + export CC="${CC} --target=$CROSS_GNU_TYPE" ; + fi ; + export CXX="${CC/clang/clang++}"; + elif [ "$CROSS_ARCH" = "i386" ] ; + then + export CC="gcc -m32" ; + export CXX="g++ -m32" ; + elif [ -n "$CROSS_ARCH" ] ; + then + export CC="$CROSS_GNU_TYPE"-gcc ; + export CXX="$CROSS_GNU_TYPE"-g++ ; fi - if test ! -L /usr/lib/ccache/${CC%% *} ; then sudo ln -s -t /usr/lib/ccache/ `which ${CC%% *}` ; fi - ccache -s @@ -116,15 +143,17 @@ install: - sudo mkdir -p /mnt/huge - sudo mount -t hugetlbfs nodev /mnt/huge - - sudo apt-get -qq update - - sudo apt-get install linux-headers-`uname -r` - - sudo pip install coverage + - if [ -z "$CROSS_ARCH" ] ; + then + sudo apt-get -qq update ; + sudo apt-get install linux-headers-`uname -r` ; + fi - gem install asciidoctor # DPDK pktio. Note that cache must be purged if dpdk version changes. - TARGET=${TARGET:-"x86_64-native-linuxapp-gcc"} - | - if [ ! -f "dpdk/${TARGET}/lib/libdpdk.a" ]; then + if [ -z "$CROSS_ARCH" -a ! -f "dpdk/${TARGET}/lib/libdpdk.a" ]; then git -c advice.detachedHead=false clone -q --depth=1 --single-branch --branch=v17.02 http://dpdk.org/git/dpdk dpdk pushd dpdk git log --oneline --decorate @@ -139,7 +168,7 @@ install: # Netmap pktio - | - if [ ! -f "netmap/LINUX/netmap.ko" ]; then + if [ -z "$CROSS_ARCH" -a ! -f "netmap/LINUX/netmap.ko" ]; then git -c advice.detachedHead=false clone -q --depth=1 --single-branch --branch=v11.2 https://github.com/luigirizzo/netmap.git pushd netmap/LINUX ./configure @@ -149,101 +178,42 @@ install: fi script: + - if [ -z "$CROSS_ARCH" ] ; then + EXTRA_CONF="--with-dpdk-path=`pwd`/dpdk/${TARGET} --with-netmap-path=`pwd`/netmap" ; + fi - ./bootstrap - ./configure --prefix=$HOME/odp-install --enable-user-guides - --with-dpdk-path=`pwd`/dpdk/${TARGET} - --with-netmap-path=`pwd`/netmap - $CONF --enable-debug=full --enable-helper-linux + $CROSS $EXTRA_CONF $CONF - make -j $(nproc) - mkdir /dev/shm/odp - - sudo LD_LIBRARY_PATH="$HOME/cunit-install/$CROSS_ARCH/lib:$LD_LIBRARY_PATH" ODP_SHM_DIR=/dev/shm/odp make check + - if [ -z "$CROSS_ARCH" ] ; then + sudo LD_LIBRARY_PATH="$HOME/cunit-install/$CROSS_ARCH/lib:$LD_LIBRARY_PATH" ODP_SHM_DIR=/dev/shm/odp make check ; + fi - make install - echo "Checking linking and run from install..." - pushd $HOME - echo "Dynamic link.." - - ${CC} ${OLDPWD}/example/hello/odp_hello.c -o odp_hello_inst `PKG_CONFIG_PATH=${HOME}/odp-install/lib/pkgconfig:${PKG_CONFIG_PATH} pkg-config --cflags --libs libodp-linux` - - LD_LIBRARY_PATH="${HOME}/odp-install/lib:$LD_LIBRARY_PATH" ./odp_hello_inst + - ${CC} ${CFLAGS} ${OLDPWD}/example/hello/odp_hello.c -o odp_hello_inst_dynamic `PKG_CONFIG_PATH=${HOME}/odp-install/lib/pkgconfig:${PKG_CONFIG_PATH} pkg-config --cflags --libs libodp-linux` - echo "Static link.." - - ${CC} ${OLDPWD}/example/hello/odp_hello.c -o odp_hello_inst `PKG_CONFIG_PATH=${HOME}/odp-install/lib/pkgconfig:${PKG_CONFIG_PATH} pkg-config --cflags --libs libodp-linux --static` -static - - ./odp_hello_inst + - ${CC} ${CFLAGS} ${OLDPWD}/example/hello/odp_hello.c -o odp_hello_inst_static `PKG_CONFIG_PATH=${HOME}/odp-install/lib/pkgconfig:${PKG_CONFIG_PATH} pkg-config --cflags --libs libodp-linux --static` -static + - if [ -z "$CROSS_ARCH" ] ; then + LD_LIBRARY_PATH="${HOME}/odp-install/lib:$LD_LIBRARY_PATH" ./odp_hello_inst ; + ./odp_hello_inst_static ; + fi + - popd - ccache -s jobs: include: - stage: test - compiler: aarch64-linux-gnu-gcc - env: TEST="aarch64-linux-gnu" CROSS_ARCH="arm64" - install: true - script: - - ./bootstrap - - ./configure --prefix=$HOME/odp-install $CROSS - --enable-debug=full - --disable-test-cpp - --enable-helper-linux - - make -j $(nproc) - - stage: test - compiler: "\"clang-3.8 --target=aarch64-linux-gnu\"" - env: TEST="clang-3.8 aarch64-linux-gnu" CROSS_ARCH="arm64" - install: true - script: - - ./bootstrap - - ./configure --prefix=$HOME/odp/odp-install $CROSS - --enable-debug=full - --disable-test-cpp - --enable-helper-linux - - make -j $(nproc) - - stage: test - compiler: arm-linux-gnueabihf-gcc - env: TEST="arm-linux-gnueabihf" CROSS_ARCH="armhf" - install: true - script: - - ./bootstrap - - ./configure --prefix=$HOME/odp-install $CROSS - --enable-debug=full - --disable-test-cpp - --enable-helper-linux - - make -j $(nproc) - - stage: test - compiler: "\"clang-3.8 --target=arm-linux-gnueabihf\"" - env: TEST="clang-3.8 arm-linux-gnueabihf" CROSS_ARCH="armhf" CFLAGS="-march=armv7-a" - install: true - script: - - ./bootstrap - - ./configure --prefix=$HOME/odp-install $CROSS - --enable-debug=full - --disable-test-cpp - --enable-helper-linux - - make -j $(nproc) - - stage: test - compiler: powerpc-linux-gnu-gcc - env: TEST="powerpc-linux-gnueabihf" CROSS_ARCH="powerpc" - install: true - script: - - ./bootstrap - - ./configure --prefix=$HOME/odp-install $CROSS - --enable-debug=full - --disable-test-cpp - --enable-helper-linux - - make -j $(nproc) - - stage: test - compiler: "\"clang-3.8 --target=powerpc-linux-gnu\"" - env: TEST="clang-3.8 powerpc-linux-gnu" CROSS_ARCH="powerpc" - install: true - script: - - ./bootstrap - - ./configure --prefix=$HOME/odp-install $CROSS - --enable-debug=full - --disable-test-cpp - --enable-helper-linux - - make -j $(nproc) - - stage: test env: TEST=coverage compiler: gcc script: + - sudo pip install coverage - ./bootstrap - ./configure --prefix=$HOME/odp-install --enable-user-guides diff --git a/include/odp/arch/default/api/abi/packet.h b/include/odp/arch/default/api/abi/packet.h index 4aac75b9e..15cf081bf 100644 --- a/include/odp/arch/default/api/abi/packet.h +++ b/include/odp/arch/default/api/abi/packet.h @@ -28,7 +28,7 @@ typedef _odp_abi_packet_seg_t *odp_packet_seg_t; #define ODP_PACKET_INVALID ((odp_packet_t)0xffffffff) #define ODP_PACKET_SEG_INVALID ((odp_packet_seg_t)0xffffffff) -#define ODP_PACKET_OFFSET_INVALID (0x0fffffff) +#define ODP_PACKET_OFFSET_INVALID 0xffff typedef enum { ODP_PACKET_GREEN = 0, diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 06b049fcb..6874d1496 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -50,31 +50,19 @@ static inline uint32_t _odp_packet_seg_len(odp_packet_t pkt) /** @internal Inline function @param pkt @return */ static inline uint32_t _odp_packet_len(odp_packet_t pkt) { - uint32_t pkt_len = _odp_pkt_get(pkt, uint32_t, frame_len); - void *ref_nxt = _odp_pkt_get(pkt, void *, ref_hdr); - void *ref_pkt = (void *)pkt; - - while (ref_nxt) { - pkt_len += _odp_pkt_get(ref_pkt, uint32_t, ref_len) - - _odp_pkt_get(ref_pkt, uint32_t, ref_offset); - - ref_pkt = ref_nxt; - ref_nxt = _odp_pkt_get(ref_nxt, void *, ref_hdr); - } - - return pkt_len; + return _odp_pkt_get(pkt, uint32_t, frame_len); } /** @internal Inline function @param pkt @return */ static inline uint32_t _odp_packet_headroom(odp_packet_t pkt) { - return _odp_pkt_get(pkt, uint32_t, headroom); + return _odp_pkt_get(pkt, uint16_t, headroom); } /** @internal Inline function @param pkt @return */ static inline uint32_t _odp_packet_tailroom(odp_packet_t pkt) { - return _odp_pkt_get(pkt, uint32_t, tailroom); + return _odp_pkt_get(pkt, uint16_t, tailroom); } /** @internal Inline function @param pkt @return */ @@ -90,6 +78,12 @@ static inline odp_pktio_t _odp_packet_input(odp_packet_t pkt) } /** @internal Inline function @param pkt @return */ +static inline int _odp_packet_num_segs(odp_packet_t pkt) +{ + return _odp_pkt_get(pkt, uint8_t, segcount); +} + +/** @internal Inline function @param pkt @return */ static inline void *_odp_packet_user_ptr(odp_packet_t pkt) { return _odp_pkt_get(pkt, void *, user_ptr); @@ -128,8 +122,7 @@ static inline void *_odp_packet_head(odp_packet_t pkt) /** @internal Inline function @param pkt @return */ static inline int _odp_packet_is_segmented(odp_packet_t pkt) { - return _odp_pkt_get(pkt, uint8_t, segcount) > 1 || - _odp_pkt_get(pkt, void *, ref_hdr) != NULL; + return _odp_pkt_get(pkt, uint8_t, segcount) > 1; } /** @internal Inline function @param pkt @return */ @@ -140,6 +133,23 @@ static inline odp_packet_seg_t _odp_packet_first_seg(odp_packet_t pkt) return _odp_packet_seg_from_ndx(0); } +/** @internal Inline function @param pkt @return */ +static inline odp_packet_seg_t _odp_packet_last_seg(odp_packet_t pkt) +{ + return _odp_packet_seg_from_ndx(_odp_packet_num_segs(pkt) - 1); +} + +/** @internal Inline function @param pkt @param seg @return */ +static inline odp_packet_seg_t _odp_packet_next_seg(odp_packet_t pkt, + odp_packet_seg_t seg) +{ + if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >= + _odp_packet_seg_to_ndx(_odp_packet_last_seg(pkt)))) + return ODP_PACKET_SEG_INVALID; + + return seg + 1; +} + /** @internal Inline function @param pkt @param offset @param len */ static inline void _odp_packet_prefetch(odp_packet_t pkt, uint32_t offset, uint32_t len) diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h b/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h index f818f820e..233bc8761 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h @@ -48,6 +48,11 @@ _ODP_INLINE odp_pktio_t odp_packet_input(odp_packet_t pkt) return _odp_packet_input(pkt); } +_ODP_INLINE int odp_packet_num_segs(odp_packet_t pkt) +{ + return _odp_packet_num_segs(pkt); +} + _ODP_INLINE void *odp_packet_user_ptr(odp_packet_t pkt) { return _odp_packet_user_ptr(pkt); @@ -88,6 +93,17 @@ _ODP_INLINE odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt) return _odp_packet_first_seg(pkt); } +_ODP_INLINE odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) +{ + return _odp_packet_last_seg(pkt); +} + +_ODP_INLINE odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, + odp_packet_seg_t seg) +{ + return _odp_packet_next_seg(pkt, seg); +} + _ODP_INLINE void odp_packet_prefetch(odp_packet_t pkt, uint32_t offset, uint32_t len) { diff --git a/platform/linux-generic/include/odp/api/plat/packet_types.h b/platform/linux-generic/include/odp/api/plat/packet_types.h index 1554c3af3..68c66312b 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_types.h @@ -36,7 +36,7 @@ typedef ODP_HANDLE_T(odp_packet_t); #define ODP_PACKET_INVALID _odp_cast_scalar(odp_packet_t, 0) -#define ODP_PACKET_OFFSET_INVALID (0x0fffffff) +#define ODP_PACKET_OFFSET_INVALID 0xffff typedef uint8_t odp_packet_seg_t; @@ -74,41 +74,33 @@ typedef enum { /** @internal Packet header field offsets for inline functions */ typedef struct _odp_packet_inline_offset_t { /** @internal field offset */ - size_t data; + uint16_t data; /** @internal field offset */ - size_t seg_len; + uint16_t seg_len; /** @internal field offset */ - size_t frame_len; + uint16_t frame_len; /** @internal field offset */ - size_t headroom; + uint16_t headroom; /** @internal field offset */ - size_t tailroom; + uint16_t tailroom; /** @internal field offset */ - size_t unshared_len; + uint16_t pool; /** @internal field offset */ - size_t ref_hdr; + uint16_t input; /** @internal field offset */ - size_t ref_offset; - /** *internal field offset */ - size_t ref_len; + uint16_t segcount; /** @internal field offset */ - size_t pool; + uint16_t user_ptr; /** @internal field offset */ - size_t input; + uint16_t user_area; /** @internal field offset */ - size_t segcount; + uint16_t user_area_size; /** @internal field offset */ - size_t user_ptr; + uint16_t flow_hash; /** @internal field offset */ - size_t user_area; + uint16_t timestamp; /** @internal field offset */ - size_t user_area_size; - /** @internal field offset */ - size_t flow_hash; - /** @internal field offset */ - size_t timestamp; - /** @internal field offset */ - size_t input_flags; + uint16_t input_flags; } _odp_packet_inline_offset_t; diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index fe0c26b97..358c0e441 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -35,38 +35,56 @@ extern "C" { #define BUFFER_BURST_SIZE CONFIG_BURST_SIZE +typedef struct seg_entry_t { + void *hdr; + uint8_t *data; + uint32_t len; +} seg_entry_t; + /* Common buffer header */ struct odp_buffer_hdr_t { - /* Buffer index in the pool */ - uint32_t index; - /* Initial buffer data pointer and length */ - uint8_t *base_data; - uint8_t *buf_end; + /* Buffer index in the pool */ + uint32_t index; - /* Max data size */ - uint32_t size; + /* Total segment count */ + uint16_t segcount; /* Pool type */ int8_t type; - /* Burst counts */ - uint8_t burst_num; - uint8_t burst_first; + /* Number of seg[] entries used */ + uint8_t num_seg; + + /* Next header which continues the segment list */ + void *next_seg; - /* Segment count */ - uint8_t segcount; + /* Last header of the segment list */ + void *last_seg; + + /* Initial buffer data pointer */ + uint8_t *base_data; + + /* Pool pointer */ + void *pool_ptr; + + /* --- 40 bytes --- */ /* Segments */ - struct { - void *hdr; - uint8_t *data; - uint32_t len; - } seg[CONFIG_PACKET_MAX_SEGS]; + seg_entry_t seg[CONFIG_PACKET_MAX_SEGS]; + + /* Burst counts */ + uint8_t burst_num; + uint8_t burst_first; /* Next buf in a list */ struct odp_buffer_hdr_t *next; + /* Burst table */ + struct odp_buffer_hdr_t *burst[BUFFER_BURST_SIZE]; + + /* --- Mostly read only data --- */ + /* User context pointer or u64 */ union { uint64_t buf_u64; @@ -74,21 +92,27 @@ struct odp_buffer_hdr_t { const void *buf_cctx; /* const alias for ctx */ }; + /* Reference count */ + odp_atomic_u32_t ref_cnt; + + /* Event type. Maybe different than pool type (crypto compl event) */ + int8_t event_type; + + /* Initial buffer tail pointer */ + uint8_t *buf_end; + /* User area pointer */ void *uarea_addr; /* User area size */ uint32_t uarea_size; - /* Event type. Maybe different than pool type (crypto compl event) */ - int8_t event_type; + /* Max data size */ + uint32_t size; /* Event subtype. Should be ODP_EVENT_NO_SUBTYPE except packets. */ int8_t event_subtype; - /* Burst table */ - struct odp_buffer_hdr_t *burst[BUFFER_BURST_SIZE]; - /* ipc mapped process can not walk over pointers, * offset has to be used */ uint64_t ipc_data_offset; @@ -97,9 +121,6 @@ struct odp_buffer_hdr_t { * inlining */ odp_pool_t pool_hdl; - /* Pool pointer */ - void *pool_ptr; - /* Data or next header */ uint8_t data[0]; } ODP_ALIGNED_CACHE; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 91fba1eac..097264236 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -19,7 +19,6 @@ extern "C" { #include <odp/api/align.h> #include <odp/api/debug.h> -#include <odp_debug_internal.h> #include <odp_buffer_internal.h> #include <odp_pool_internal.h> #include <odp_buffer_inlines.h> @@ -89,9 +88,14 @@ typedef struct { error_flags_t error_flags; output_flags_t output_flags; - uint32_t l2_offset; /**< offset to L2 hdr, e.g. Eth */ - uint32_t l3_offset; /**< offset to L3 hdr, e.g. IPv4, IPv6 */ - uint32_t l4_offset; /**< offset to L4 hdr (TCP, UDP, SCTP, also ICMP) */ + /* offset to L2 hdr, e.g. Eth */ + uint16_t l2_offset; + + /* offset to L3 hdr, e.g. IPv4, IPv6 */ + uint16_t l3_offset; + + /* offset to L4 hdr (TCP, UDP, SCTP, also ICMP) */ + uint16_t l4_offset; } packet_parser_t; /* Packet extra data length */ @@ -107,7 +111,7 @@ typedef struct { * packet_init(). Because of this any new fields added must be reviewed for * initialization requirements. */ -typedef struct odp_packet_hdr_t { +typedef struct { /* common buffer header */ odp_buffer_hdr_t buf_hdr; @@ -120,21 +124,10 @@ typedef struct odp_packet_hdr_t { odp_pktio_t input; uint32_t frame_len; - uint32_t headroom; - uint32_t tailroom; - - /* Fields used to support packet references */ - uint32_t unshared_len; - /* Next pkt_hdr in reference chain */ - struct odp_packet_hdr_t *ref_hdr; - /* Offset into next pkt_hdr that ref was created at */ - uint32_t ref_offset; - /* frame_len in next pkt_hdr at time ref was created. This - * allows original offset to be maintained when base pkt len - * is changed */ - uint32_t ref_len; - /* Incremented on refs, decremented on frees. */ - odp_atomic_u32_t ref_count; + uint32_t shared_len; + + uint16_t headroom; + uint16_t tailroom; /* * Members below are not initialized by packet_init() @@ -171,50 +164,6 @@ static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt) return (odp_packet_hdr_t *)(uintptr_t)pkt; } -static inline odp_packet_hdr_t *packet_last_hdr(odp_packet_t pkt, - uint32_t *offset) -{ - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - odp_packet_hdr_t *prev_hdr = pkt_hdr; - uint32_t ref_offset = 0; - - while (pkt_hdr->ref_hdr) { - ref_offset = pkt_hdr->ref_offset; - prev_hdr = pkt_hdr; - pkt_hdr = pkt_hdr->ref_hdr; - } - - if (offset) { - if (prev_hdr != pkt_hdr) - ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len; - *offset = ref_offset; - } - - return pkt_hdr; -} - -static inline odp_packet_hdr_t *packet_prev_hdr(odp_packet_hdr_t *pkt_hdr, - odp_packet_hdr_t *cur_hdr, - uint32_t *offset) -{ - uint32_t ref_offset = 0; - odp_packet_hdr_t *prev_hdr = pkt_hdr; - - while (pkt_hdr->ref_hdr != cur_hdr) { - ref_offset = pkt_hdr->ref_offset; - prev_hdr = pkt_hdr; - pkt_hdr = pkt_hdr->ref_hdr; - } - - if (offset) { - if (prev_hdr != pkt_hdr) - ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len; - *offset = ref_offset; - } - - return pkt_hdr; -} - static inline odp_packet_t packet_handle(odp_packet_hdr_t *pkt_hdr) { return (odp_packet_t)pkt_hdr; @@ -230,9 +179,14 @@ static inline odp_packet_t packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr) return (odp_packet_t)(odp_packet_hdr_t *)buf_hdr; } -static inline odp_buffer_t packet_to_buffer(odp_packet_t pkt) +static inline seg_entry_t *seg_entry_last(odp_packet_hdr_t *hdr) { - return (odp_buffer_t)pkt; + odp_packet_hdr_t *last; + uint8_t last_seg; + + last = hdr->buf_hdr.last_seg; + last_seg = last->buf_hdr.num_seg - 1; + return &last->buf_hdr.seg[last_seg]; } /** @@ -247,10 +201,13 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len) seg_len = len; pkt_hdr->buf_hdr.seg[0].len = len; } else { + seg_entry_t *last; + seg_len = len - ((num - 1) * CONFIG_PACKET_MAX_SEG_LEN); /* Last segment data length */ - pkt_hdr->buf_hdr.seg[num - 1].len = seg_len; + last = seg_entry_last(pkt_hdr); + last->len = seg_len; } pkt_hdr->p.input_flags.all = 0; @@ -267,6 +224,7 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len) * segment occupied by the allocated length. */ pkt_hdr->frame_len = len; + pkt_hdr->shared_len = 0; pkt_hdr->headroom = CONFIG_PACKET_HEADROOM; pkt_hdr->tailroom = CONFIG_PACKET_MAX_SEG_LEN - seg_len + CONFIG_PACKET_TAILROOM; @@ -274,9 +232,6 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len) pkt_hdr->input = ODP_PKTIO_INVALID; pkt_hdr->buf_hdr.event_subtype = ODP_EVENT_PACKET_BASIC; - /* By default packet has no references */ - pkt_hdr->unshared_len = len; - pkt_hdr->ref_hdr = NULL; } static inline void copy_packet_parser_metadata(odp_packet_hdr_t *src_hdr, @@ -296,53 +251,21 @@ static inline void copy_packet_cls_metadata(odp_packet_hdr_t *src_hdr, static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) { - int last = pkt_hdr->buf_hdr.segcount - 1; + seg_entry_t *last = seg_entry_last(pkt_hdr); pkt_hdr->tailroom += len; pkt_hdr->frame_len -= len; - pkt_hdr->unshared_len -= len; - pkt_hdr->buf_hdr.seg[last].len -= len; + last->len -= len; } static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr) { - uint32_t pkt_len = pkt_hdr->frame_len; - odp_packet_hdr_t *ref_hdr = pkt_hdr->ref_hdr; - - while (ref_hdr) { - pkt_len += (pkt_hdr->ref_len - pkt_hdr->ref_offset); - pkt_hdr = ref_hdr; - ref_hdr = ref_hdr->ref_hdr; - } - - return pkt_len; -} - -static inline uint32_t packet_ref_count(odp_packet_hdr_t *pkt_hdr) -{ - /* Breach the atomic type to do a peek at the ref count. This - * is used to bypass atomic operations if ref_count == 1 for - * performance reasons. - */ - return pkt_hdr->ref_count.v; -} - -static inline void packet_ref_count_set(odp_packet_hdr_t *pkt_hdr, uint32_t n) -{ - /* Only used during init when there are no other possible - * references to this pkt, so avoid the "atomic" overhead by - * a controlled breach of the atomic type here. This saves - * over 10% of the pathlength in routines like packet_alloc(). - */ - pkt_hdr->ref_count.v = n; + return pkt_hdr->frame_len; } static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len) { - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); - pkt_hdr->frame_len = len; - pkt_hdr->unshared_len = len; } /* Forward declarations */ diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index bb0f2b895..5dbe65d34 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -35,10 +35,6 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .frame_len = offsetof(odp_packet_hdr_t, frame_len), .headroom = offsetof(odp_packet_hdr_t, headroom), .tailroom = offsetof(odp_packet_hdr_t, tailroom), - .unshared_len = offsetof(odp_packet_hdr_t, unshared_len), - .ref_hdr = offsetof(odp_packet_hdr_t, ref_hdr), - .ref_offset = offsetof(odp_packet_hdr_t, ref_offset), - .ref_len = offsetof(odp_packet_hdr_t, ref_len), .pool = offsetof(odp_packet_hdr_t, buf_hdr.pool_hdl), .input = offsetof(odp_packet_hdr_t, input), .segcount = offsetof(odp_packet_hdr_t, buf_hdr.segcount), @@ -63,39 +59,130 @@ static inline odp_buffer_t buffer_handle(odp_packet_hdr_t *pkt_hdr) return (odp_buffer_t)pkt_hdr; } -static inline void packet_ref_inc(odp_packet_hdr_t *pkt_hdr) +static inline odp_packet_hdr_t *buf_to_packet_hdr(odp_buffer_t buf) { - odp_atomic_inc_u32(&pkt_hdr->ref_count); + return (odp_packet_hdr_t *)buf_hdl_to_hdr(buf); } -static inline uint32_t packet_ref_dec(odp_packet_hdr_t *pkt_hdr) +odp_packet_t _odp_packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr) { - return odp_atomic_fetch_dec_u32(&pkt_hdr->ref_count); + return (odp_packet_t)buf_hdr; } -static inline odp_packet_hdr_t *buf_to_packet_hdr(odp_buffer_t buf) +static inline odp_buffer_t packet_to_buffer(odp_packet_t pkt) { - return (odp_packet_hdr_t *)buf_hdl_to_hdr(buf); + return (odp_buffer_t)pkt; } -odp_packet_t _odp_packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr) +static inline seg_entry_t *seg_entry(odp_packet_hdr_t *hdr, + uint32_t seg_idx) { - return (odp_packet_t)buf_hdr; + uint32_t idx = 0; + uint8_t num_seg = hdr->buf_hdr.num_seg; + + while (odp_unlikely(idx + num_seg - 1 < seg_idx)) { + idx += num_seg; + hdr = hdr->buf_hdr.next_seg; + num_seg = hdr->buf_hdr.num_seg; + } + + idx = seg_idx - idx; + + return &hdr->buf_hdr.seg[idx]; +} + +static inline void seg_entry_find_idx(odp_packet_hdr_t **p_hdr, + uint8_t *p_idx, + uint32_t find_idx) +{ + odp_packet_hdr_t *hdr = *p_hdr; + uint32_t idx = 0; + uint8_t num_seg = hdr->buf_hdr.num_seg; + + while (odp_unlikely(idx + num_seg - 1 < find_idx)) { + idx += num_seg; + hdr = hdr->buf_hdr.next_seg; + num_seg = hdr->buf_hdr.num_seg; + } + + idx = find_idx - idx; + *p_hdr = hdr; + *p_idx = idx; +} + +/* Return pointer to the current segment entry and step cur_hdr / cur_idx + * forward. + */ +static inline seg_entry_t *seg_entry_next(odp_packet_hdr_t **cur_hdr, + uint8_t *cur_idx) +{ + odp_packet_hdr_t *hdr = *cur_hdr; + uint8_t idx = *cur_idx; + uint8_t num_seg = hdr->buf_hdr.num_seg; + + if (idx == num_seg - 1) { + *cur_hdr = hdr->buf_hdr.next_seg; + *cur_idx = 0; + } else { + *cur_idx = idx + 1; + } + + return &hdr->buf_hdr.seg[idx]; +} + +static inline void seg_entry_find_offset(odp_packet_hdr_t **p_hdr, + uint8_t *p_idx, + uint32_t *seg_offset, + uint32_t *seg_idx, + uint32_t offset) +{ + int i; + odp_packet_hdr_t *hdr, *cur_hdr; + uint8_t idx, cur_idx; + seg_entry_t *seg = NULL; + uint32_t seg_start = 0, seg_end = 0; + int seg_count; + + hdr = *p_hdr; + cur_hdr = hdr; + idx = 0; + cur_idx = 0; + seg_count = hdr->buf_hdr.segcount; + + for (i = 0; i < seg_count; i++) { + cur_hdr = hdr; + cur_idx = idx; + seg = seg_entry_next(&hdr, &idx); + seg_end += seg->len; + + if (odp_likely(offset < seg_end)) + break; + + seg_start = seg_end; + } + + *p_hdr = cur_hdr; + *p_idx = cur_idx; + *seg_offset = offset - seg_start; + *seg_idx = i; } static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) { - return pkt_hdr->buf_hdr.seg[seg_idx].len; + seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); + + return seg->len; } -static inline uint8_t *packet_seg_data(odp_packet_hdr_t *pkt_hdr, - uint32_t seg_idx) +static inline void *packet_seg_data(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) { - return pkt_hdr->buf_hdr.seg[seg_idx].data; + seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); + + return seg->data; } -static inline uint32_t packet_last_seg(odp_packet_hdr_t *pkt_hdr) +static inline uint16_t packet_last_seg(odp_packet_hdr_t *pkt_hdr) { if (CONFIG_PACKET_MAX_SEGS == 1) return 0; @@ -105,14 +192,7 @@ static inline uint32_t packet_last_seg(odp_packet_hdr_t *pkt_hdr) static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr) { - return packet_seg_len(pkt_hdr, 0); -} - -static inline uint32_t packet_last_seg_len(odp_packet_hdr_t *pkt_hdr) -{ - int last = packet_last_seg(pkt_hdr); - - return packet_seg_len(pkt_hdr, last); + return pkt_hdr->buf_hdr.seg[0].len; } static inline void *packet_data(odp_packet_hdr_t *pkt_hdr) @@ -122,26 +202,27 @@ static inline void *packet_data(odp_packet_hdr_t *pkt_hdr) static inline void *packet_tail(odp_packet_hdr_t *pkt_hdr) { - int last = packet_last_seg(pkt_hdr); - uint32_t seg_len = pkt_hdr->buf_hdr.seg[last].len; + seg_entry_t *last_seg = seg_entry_last(pkt_hdr); - return pkt_hdr->buf_hdr.seg[last].data + seg_len; + return last_seg->data + last_seg->len; } -static inline uint32_t seg_headroom(odp_packet_hdr_t *pkt_hdr, int seg) +static inline uint32_t seg_headroom(odp_packet_hdr_t *pkt_hdr, int seg_idx) { - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[seg].hdr; + seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); + odp_buffer_hdr_t *hdr = seg->hdr; uint8_t *base = hdr->base_data; - uint8_t *head = pkt_hdr->buf_hdr.seg[seg].data; + uint8_t *head = seg->data; return CONFIG_PACKET_HEADROOM + (head - base); } -static inline uint32_t seg_tailroom(odp_packet_hdr_t *pkt_hdr, int seg) +static inline uint32_t seg_tailroom(odp_packet_hdr_t *pkt_hdr, int seg_idx) { - uint32_t seg_len = pkt_hdr->buf_hdr.seg[seg].len; - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[seg].hdr; - uint8_t *tail = pkt_hdr->buf_hdr.seg[seg].data + seg_len; + seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); + + odp_buffer_hdr_t *hdr = seg->hdr; + uint8_t *tail = seg->data + seg->len; return hdr->buf_end - tail; } @@ -150,7 +231,6 @@ static inline void push_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) { pkt_hdr->headroom -= len; pkt_hdr->frame_len += len; - pkt_hdr->unshared_len += len; pkt_hdr->buf_hdr.seg[0].data -= len; pkt_hdr->buf_hdr.seg[0].len += len; } @@ -159,19 +239,17 @@ static inline void pull_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) { pkt_hdr->headroom += len; pkt_hdr->frame_len -= len; - pkt_hdr->unshared_len -= len; pkt_hdr->buf_hdr.seg[0].data += len; pkt_hdr->buf_hdr.seg[0].len -= len; } static inline void push_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) { - int last = packet_last_seg(pkt_hdr); + seg_entry_t *last_seg = seg_entry_last(pkt_hdr); pkt_hdr->tailroom -= len; pkt_hdr->frame_len += len; - pkt_hdr->unshared_len += len; - pkt_hdr->buf_hdr.seg[last].len += len; + last_seg->len += len; } /* Copy all metadata for segmentation modification. Segment data and lengths @@ -197,13 +275,12 @@ static inline void packet_seg_copy_md(odp_packet_hdr_t *dst, dst->buf_hdr.uarea_addr = src->buf_hdr.uarea_addr; dst->buf_hdr.uarea_size = src->buf_hdr.uarea_size; - /* reference related metadata */ - dst->ref_len = src->ref_len; - dst->unshared_len = src->unshared_len; - /* segmentation data is not copied: * buf_hdr.seg[] * buf_hdr.segcount + * buf_hdr.num_seg + * buf_hdr.next_seg + * buf_hdr.last_seg */ } @@ -212,18 +289,10 @@ static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, { void *addr; uint32_t len; - int seg = 0; + int seg_id = 0; int seg_count = pkt_hdr->buf_hdr.segcount; - /* Special processing for references */ - while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { - offset -= (pkt_hdr->frame_len - pkt_hdr->ref_offset); - offset += (pkt_hdr->ref_hdr->frame_len - pkt_hdr->ref_len); - pkt_hdr = pkt_hdr->ref_hdr; - seg_count = pkt_hdr->buf_hdr.segcount; - } - - if (odp_unlikely(offset > pkt_hdr->frame_len)) + if (odp_unlikely(offset >= pkt_hdr->frame_len)) return NULL; if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || seg_count == 1)) { @@ -231,10 +300,14 @@ static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, len = pkt_hdr->buf_hdr.seg[0].len - offset; } else { int i; + seg_entry_t *seg = NULL; uint32_t seg_start = 0, seg_end = 0; + odp_packet_hdr_t *hdr = pkt_hdr; + uint8_t idx = 0; for (i = 0; i < seg_count; i++) { - seg_end += pkt_hdr->buf_hdr.seg[i].len; + seg = seg_entry_next(&hdr, &idx); + seg_end += seg->len; if (odp_likely(offset < seg_end)) break; @@ -242,16 +315,16 @@ static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, seg_start = seg_end; } - addr = pkt_hdr->buf_hdr.seg[i].data + (offset - seg_start); - len = pkt_hdr->buf_hdr.seg[i].len - (offset - seg_start); - seg = i; + addr = seg->data + (offset - seg_start); + len = seg->len - (offset - seg_start); + seg_id = i; } if (seg_len) *seg_len = len; if (seg_idx) - *seg_idx = seg; + *seg_idx = seg_id; return addr; } @@ -265,52 +338,83 @@ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) pkt_hdr->p.l2_offset = 0; pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; +} - /* Ensure dummy pkt hdrs used in I/O recv classification are valid */ - pkt_hdr->ref_hdr = NULL; +static inline void link_segments(odp_packet_hdr_t *pkt_hdr[], int num) +{ + int cur, i; + odp_packet_hdr_t *hdr; + odp_packet_hdr_t *head = pkt_hdr[0]; + + cur = 0; + + while (1) { + hdr = pkt_hdr[cur]; + + for (i = 0; i < CONFIG_PACKET_MAX_SEGS; i++) { + odp_buffer_hdr_t *buf_hdr; + + buf_hdr = &pkt_hdr[cur]->buf_hdr; + hdr->buf_hdr.seg[i].hdr = buf_hdr; + hdr->buf_hdr.seg[i].data = buf_hdr->base_data; + hdr->buf_hdr.seg[i].len = BASE_LEN; + cur++; + + if (cur == num) { + /* Last segment */ + hdr->buf_hdr.num_seg = i + 1; + hdr->buf_hdr.next_seg = NULL; + head->buf_hdr.last_seg = &hdr->buf_hdr; + return; + } + } + + hdr->buf_hdr.num_seg = CONFIG_PACKET_MAX_SEGS; + hdr->buf_hdr.next_seg = pkt_hdr[cur]; + } } static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) { odp_packet_hdr_t *hdr; - int i; /* First segment is the packet descriptor */ hdr = pkt_hdr[0]; hdr->buf_hdr.seg[0].data = hdr->buf_hdr.base_data; hdr->buf_hdr.seg[0].len = BASE_LEN; - packet_ref_count_set(hdr, 1); /* Link segments */ if (CONFIG_PACKET_MAX_SEGS != 1) { hdr->buf_hdr.segcount = num; + /* Defaults for single segment packet */ + hdr->buf_hdr.num_seg = 1; + hdr->buf_hdr.next_seg = NULL; + hdr->buf_hdr.last_seg = &hdr->buf_hdr; + if (odp_unlikely(num > 1)) { - for (i = 1; i < num; i++) { - odp_buffer_hdr_t *buf_hdr; - - packet_ref_count_set(pkt_hdr[i], 1); - buf_hdr = &pkt_hdr[i]->buf_hdr; - hdr->buf_hdr.seg[i].hdr = buf_hdr; - hdr->buf_hdr.seg[i].data = buf_hdr->base_data; - hdr->buf_hdr.seg[i].len = BASE_LEN; - } + link_segments(pkt_hdr, num); + } } } static inline void reset_seg(odp_packet_hdr_t *pkt_hdr, int first, int num) { - odp_buffer_hdr_t *hdr; + odp_packet_hdr_t *hdr = pkt_hdr; void *base; int i; + seg_entry_t *seg; + uint8_t idx; + + seg_entry_find_idx(&hdr, &idx, first); - for (i = first; i < first + num; i++) { - hdr = pkt_hdr->buf_hdr.seg[i].hdr; - base = hdr->base_data; - pkt_hdr->buf_hdr.seg[i].len = BASE_LEN; - pkt_hdr->buf_hdr.seg[i].data = base; + for (i = 0; i < num; i++) { + base = hdr->buf_hdr.base_data; + seg = seg_entry_next(&hdr, &idx); + seg->len = BASE_LEN; + seg->data = base; } } @@ -338,31 +442,11 @@ static inline int num_segments(uint32_t len) static inline void add_all_segs(odp_packet_hdr_t *to, odp_packet_hdr_t *from) { - int i; - int n = to->buf_hdr.segcount; - int num = from->buf_hdr.segcount; - - for (i = 0; i < num; i++) { - to->buf_hdr.seg[n + i].hdr = from->buf_hdr.seg[i].hdr; - to->buf_hdr.seg[n + i].data = from->buf_hdr.seg[i].data; - to->buf_hdr.seg[n + i].len = from->buf_hdr.seg[i].len; - } - - to->buf_hdr.segcount = n + num; -} - -static inline void copy_num_segs(odp_packet_hdr_t *to, odp_packet_hdr_t *from, - int first, int num) -{ - int i; - - for (i = 0; i < num; i++) { - to->buf_hdr.seg[i].hdr = from->buf_hdr.seg[first + i].hdr; - to->buf_hdr.seg[i].data = from->buf_hdr.seg[first + i].data; - to->buf_hdr.seg[i].len = from->buf_hdr.seg[first + i].len; - } + odp_packet_hdr_t *last = to->buf_hdr.last_seg; - to->buf_hdr.segcount = num; + last->buf_hdr.next_seg = from; + to->buf_hdr.last_seg = from->buf_hdr.last_seg; + to->buf_hdr.segcount += from->buf_hdr.segcount; } static inline odp_packet_hdr_t *alloc_segments(pool_t *pool, int num) @@ -408,97 +492,257 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, new_hdr->buf_hdr.seg[0].len = seg_len; packet_seg_copy_md(new_hdr, pkt_hdr); - new_hdr->frame_len = pkt_hdr->frame_len + len; - new_hdr->unshared_len = pkt_hdr->unshared_len + len; - new_hdr->headroom = pool->headroom + offset; - new_hdr->tailroom = pkt_hdr->tailroom; + new_hdr->frame_len = pkt_hdr->frame_len + len; + new_hdr->headroom = pool->headroom + offset; + new_hdr->tailroom = pkt_hdr->tailroom; + new_hdr->shared_len = pkt_hdr->shared_len; pkt_hdr = new_hdr; } else { - int last; + seg_entry_t *last_seg; /* add into the tail */ add_all_segs(pkt_hdr, new_hdr); /* adjust last segment length */ - last = packet_last_seg(pkt_hdr); - pkt_hdr->buf_hdr.seg[last].len = seg_len; + last_seg = seg_entry_last(pkt_hdr); + last_seg->len = seg_len; - pkt_hdr->frame_len += len; - pkt_hdr->unshared_len += len; - pkt_hdr->tailroom = pool->tailroom + offset; + pkt_hdr->frame_len += len; + pkt_hdr->tailroom = pool->tailroom + offset; } return pkt_hdr; } -static inline void free_bufs(odp_packet_hdr_t *pkt_hdr, int first, int num) +static inline int seg_is_link(void *hdr) { - int i, nfree; - odp_buffer_hdr_t *buf_hdr[num]; + odp_packet_hdr_t *pkt_hdr = hdr; + + return pkt_hdr != pkt_hdr->buf_hdr.seg[0].hdr; +} - for (i = 0, nfree = 0; i < num; i++) { - odp_packet_hdr_t *hdr = pkt_hdr->buf_hdr.seg[first + i].hdr; +static inline void buffer_ref_inc(odp_buffer_hdr_t *buf_hdr) +{ + uint32_t ref_cnt = odp_atomic_load_u32(&buf_hdr->ref_cnt); + + /* First count increment after alloc */ + if (odp_likely(ref_cnt) == 0) + odp_atomic_store_u32(&buf_hdr->ref_cnt, 2); + else + odp_atomic_inc_u32(&buf_hdr->ref_cnt); +} + +static inline uint32_t buffer_ref_dec(odp_buffer_hdr_t *buf_hdr) +{ + return odp_atomic_fetch_dec_u32(&buf_hdr->ref_cnt); +} + +static inline uint32_t buffer_ref(odp_buffer_hdr_t *buf_hdr) +{ + return odp_atomic_load_u32(&buf_hdr->ref_cnt); +} + +static inline int is_multi_ref(uint32_t ref_cnt) +{ + return (ref_cnt > 1); +} + +static inline void packet_ref_inc(odp_packet_hdr_t *pkt_hdr) +{ + seg_entry_t *seg; + int i; + int seg_count = pkt_hdr->buf_hdr.segcount; + odp_packet_hdr_t *hdr = pkt_hdr; + uint8_t idx = 0; + + for (i = 0; i < seg_count; i++) { + seg = seg_entry_next(&hdr, &idx); + buffer_ref_inc(seg->hdr); + } +} + +static inline void packet_free_multi(odp_buffer_hdr_t *hdr[], int num) +{ + int i; + uint32_t ref_cnt; + int num_ref = 0; + + for (i = 0; i < num; i++) { + /* Zero when reference API has not been used */ + ref_cnt = buffer_ref(hdr[i]); + + if (odp_unlikely(ref_cnt)) { + ref_cnt = buffer_ref_dec(hdr[i]); + + if (is_multi_ref(ref_cnt)) { + num_ref++; + continue; + } + } + + /* Reset link header back to normal header */ + if (odp_unlikely(seg_is_link(hdr[i]))) + hdr[i]->seg[0].hdr = hdr[i]; + + /* Skip references and pack to be freed headers to array head */ + if (odp_unlikely(num_ref)) + hdr[i - num_ref] = hdr[i]; + + } + + num -= num_ref; + + if (odp_likely(num)) + buffer_free_multi(hdr, num); +} + +static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num) +{ + int i; + odp_buffer_hdr_t *buf_hdr[num + 1]; + + if (odp_likely(pkt_hdr->buf_hdr.num_seg == num)) { + for (i = 0; i < num; i++) + buf_hdr[i] = pkt_hdr->buf_hdr.seg[i].hdr; + + if (odp_unlikely(seg_is_link(pkt_hdr))) { + buf_hdr[num] = &pkt_hdr->buf_hdr; + num++; + } + } else { + seg_entry_t *seg; + odp_buffer_hdr_t *link_hdr[num]; + uint8_t idx = 0; + int links = 0; + + for (i = 0; i < num; i++) { + /* Free also link headers */ + if (odp_unlikely(idx == 0 && seg_is_link(pkt_hdr))) { + link_hdr[links] = &pkt_hdr->buf_hdr; + links++; + } - if (packet_ref_count(hdr) == 1 || packet_ref_dec(hdr) == 1) { - ODP_ASSERT((packet_ref_count_set(hdr, 0), 1)); - buf_hdr[nfree++] = &hdr->buf_hdr; + seg = seg_entry_next(&pkt_hdr, &idx); + buf_hdr[i] = seg->hdr; } + + if (odp_unlikely(links)) + packet_free_multi(link_hdr, links); } - if (nfree > 0) - buffer_free_multi(buf_hdr, nfree); + packet_free_multi(buf_hdr, num); } static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, int num, uint32_t free_len, uint32_t pull_len, int head) { + seg_entry_t *seg; + int i; int num_remain = pkt_hdr->buf_hdr.segcount - num; + odp_packet_hdr_t *hdr = pkt_hdr; + odp_packet_hdr_t *last_hdr = pkt_hdr->buf_hdr.last_seg; + uint8_t idx; + uint8_t num_seg; + odp_buffer_hdr_t *buf_hdr[num]; + odp_buffer_hdr_t *link_hdr[num]; + odp_packet_hdr_t *tmp_hdr; + int links = 0; if (head) { odp_packet_hdr_t *new_hdr; - int i, nfree; - odp_buffer_hdr_t *buf_hdr[num]; - for (i = 0, nfree = 0; i < num; i++) { - new_hdr = pkt_hdr->buf_hdr.seg[i].hdr; + idx = 0; + for (i = 0; i < num; i++) { + tmp_hdr = hdr; + seg = seg_entry_next(&hdr, &idx); + buf_hdr[i] = seg->hdr; - if (packet_ref_count(new_hdr) == 1 || - packet_ref_dec(new_hdr) == 1) { - ODP_ASSERT((packet_ref_count_set(new_hdr, 0), - 1)); - buf_hdr[nfree++] = &new_hdr->buf_hdr; + /* Free link headers, if those become empty */ + if (odp_unlikely(idx == 0 && seg_is_link(tmp_hdr))) { + link_hdr[links] = &tmp_hdr->buf_hdr; + links++; } } - /* First remaining segment is the new packet descriptor */ - new_hdr = pkt_hdr->buf_hdr.seg[num].hdr; + /* The first remaining header is the new packet descriptor. + * Copy remaining segments from the last to-be-removed header + * to the new header. */ + new_hdr = hdr->buf_hdr.seg[idx].hdr; + num_seg = hdr->buf_hdr.num_seg - idx; + + new_hdr->buf_hdr.next_seg = hdr->buf_hdr.next_seg; + + if (hdr == last_hdr) + new_hdr->buf_hdr.last_seg = new_hdr; + else + new_hdr->buf_hdr.last_seg = last_hdr; + + new_hdr->buf_hdr.num_seg = num_seg; + new_hdr->buf_hdr.segcount = num_remain; + + for (i = 0; i < num_seg; i++) { + seg = seg_entry_next(&hdr, &idx); + new_hdr->buf_hdr.seg[i] = *seg; + } - copy_num_segs(new_hdr, pkt_hdr, num, num_remain); packet_seg_copy_md(new_hdr, pkt_hdr); /* Tailroom not changed */ - new_hdr->tailroom = pkt_hdr->tailroom; - new_hdr->headroom = seg_headroom(new_hdr, 0); - new_hdr->frame_len = pkt_hdr->frame_len - free_len; - new_hdr->unshared_len = pkt_hdr->unshared_len - free_len; + new_hdr->tailroom = pkt_hdr->tailroom; + + /* Link header does not have headroom */ + if (seg_is_link(new_hdr)) + new_hdr->headroom = 0; + else + new_hdr->headroom = seg_headroom(new_hdr, 0); + + new_hdr->frame_len = pkt_hdr->frame_len - free_len; + new_hdr->shared_len = pkt_hdr->shared_len; pull_head(new_hdr, pull_len); pkt_hdr = new_hdr; - if (nfree > 0) - buffer_free_multi(buf_hdr, nfree); + if (odp_unlikely(links)) + packet_free_multi(link_hdr, links); + + packet_free_multi(buf_hdr, num); } else { - /* Free last 'num' bufs */ - free_bufs(pkt_hdr, num_remain, num); + /* Free last 'num' bufs. + * First, find the last remaining header. */ + seg_entry_find_idx(&hdr, &idx, num_remain - 1); + last_hdr = hdr; + num_seg = idx + 1; + + seg_entry_next(&hdr, &idx); + + for (i = 0; i < num; i++) { + tmp_hdr = hdr; + seg = seg_entry_next(&hdr, &idx); + buf_hdr[i] = seg->hdr; + + /* Free link headers, if those become empty */ + if (odp_unlikely(idx == 0 && seg_is_link(tmp_hdr))) { + link_hdr[links] = &tmp_hdr->buf_hdr; + links++; + } + } + + if (odp_unlikely(links)) + packet_free_multi(link_hdr, links); + + packet_free_multi(buf_hdr, num); /* Head segment remains, no need to copy or update majority * of the metadata. */ + last_hdr->buf_hdr.num_seg = num_seg; + last_hdr->buf_hdr.next_seg = NULL; + + pkt_hdr->buf_hdr.last_seg = last_hdr; pkt_hdr->buf_hdr.segcount = num_remain; pkt_hdr->frame_len -= free_len; - pkt_hdr->unshared_len -= free_len; pkt_hdr->tailroom = seg_tailroom(pkt_hdr, num_remain - 1); pull_tail(pkt_hdr, pull_len); @@ -605,80 +849,59 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, return num; } -static inline void packet_free(odp_packet_hdr_t *pkt_hdr) +void odp_packet_free(odp_packet_t pkt) { - odp_packet_hdr_t *ref_hdr; - odp_buffer_hdr_t *buf_hdr; - uint32_t ref_count; - int num_seg; - - do { - buf_hdr = &pkt_hdr->buf_hdr; - ref_count = packet_ref_count(pkt_hdr); - num_seg = pkt_hdr->buf_hdr.segcount; - ref_hdr = pkt_hdr->ref_hdr; - ODP_ASSERT(ref_count >= 1); - - if (odp_likely((CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1) && - ref_count == 1)) { - ODP_ASSERT((packet_ref_count_set(pkt_hdr, 0), 1)); - buffer_free_multi(&buf_hdr, 1); - } else { - free_bufs(pkt_hdr, 0, num_seg); - } + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + int num_seg = pkt_hdr->buf_hdr.segcount; - pkt_hdr = ref_hdr; - } while (pkt_hdr); -} + if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1)) { + odp_buffer_hdr_t *buf_hdr[2]; + int num = 1; -void odp_packet_free(odp_packet_t pkt) -{ - packet_free(packet_hdr(pkt)); + buf_hdr[0] = &pkt_hdr->buf_hdr; + + if (odp_unlikely(seg_is_link(pkt_hdr))) { + num = 2; + buf_hdr[1] = pkt_hdr->buf_hdr.seg[0].hdr; + } + + packet_free_multi(buf_hdr, num); + } else { + free_all_segments(pkt_hdr, num_seg); + } } void odp_packet_free_multi(const odp_packet_t pkt[], int num) { - odp_packet_hdr_t *pkt_hdr, *ref_hdr, *hdr; - int nbufs = num * CONFIG_PACKET_MAX_SEGS * 2; - odp_buffer_hdr_t *buf_hdr[nbufs]; - int num_seg; - int i, j; - uint32_t ref_count; - int nfree = 0; + odp_buffer_hdr_t *buf_hdr[num]; + odp_buffer_hdr_t *buf_hdr2[num]; + int i; + int links = 0; + int num_freed = 0; for (i = 0; i < num; i++) { - pkt_hdr = packet_hdr(pkt[i]); - - do { - num_seg = pkt_hdr->buf_hdr.segcount; - ref_hdr = pkt_hdr->ref_hdr; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt[i]); + int num_seg = pkt_hdr->buf_hdr.segcount; - /* Make sure we have enough space for this pkt's segs */ - if (nfree + num_seg > nbufs) { - buffer_free_multi(buf_hdr, nfree); - nfree = 0; - } + if (odp_unlikely(num_seg > 1)) { + free_all_segments(pkt_hdr, num_seg); + num_freed++; + continue; + } - for (j = 0; j < num_seg; j++) { - hdr = pkt_hdr->buf_hdr.seg[j].hdr; - ref_count = packet_ref_count(hdr); - ODP_ASSERT(ref_count >= 1); - - if (ref_count == 1 || - packet_ref_dec(hdr) == 1) { - ODP_ASSERT - ((packet_ref_count_set(hdr, 0), - 1)); - buf_hdr[nfree++] = &hdr->buf_hdr; - } - } + if (odp_unlikely(seg_is_link(pkt_hdr))) { + buf_hdr2[links] = pkt_hdr->buf_hdr.seg[0].hdr; + links++; + } - pkt_hdr = ref_hdr; - } while (pkt_hdr); + buf_hdr[i - num_freed] = &pkt_hdr->buf_hdr; } - if (nfree > 0) - buffer_free_multi(buf_hdr, nfree); + if (odp_unlikely(links)) + packet_free_multi(buf_hdr2, links); + + if (odp_likely(num - num_freed)) + packet_free_multi(buf_hdr, num - num_freed); } int odp_packet_reset(odp_packet_t pkt, uint32_t len) @@ -690,9 +913,6 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) if (odp_unlikely(len > (pool->max_seg_len * num))) return -1; - if (pkt_hdr->ref_hdr) - packet_free(pkt_hdr->ref_hdr); - reset_seg(pkt_hdr, 0, num); packet_init(pkt_hdr, len); @@ -726,47 +946,13 @@ odp_event_t odp_packet_to_event(odp_packet_t pkt) uint32_t odp_packet_buf_len(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t buf_len = 0; - - do { - buf_len += pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount; - pkt_hdr = pkt_hdr->ref_hdr; - } while (pkt_hdr); - return buf_len; -} - -uint32_t odp_packet_unshared_len(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t pkt_len = 0, offset = 0; - - if (packet_ref_count(pkt_hdr) == 1) - pkt_hdr->unshared_len = pkt_hdr->frame_len; - - do { - if (packet_ref_count(pkt_hdr) > 1) { - if (offset == 0) - pkt_len += pkt_hdr->unshared_len; - break; - } - - pkt_len += pkt_hdr->frame_len - offset; - offset = pkt_hdr->ref_offset; - - if (pkt_hdr->ref_hdr) - offset += (pkt_hdr->ref_hdr->frame_len - - pkt_hdr->ref_len); - - pkt_hdr = pkt_hdr->ref_hdr; - } while (pkt_hdr); - - return pkt_len; + return pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount; } void *odp_packet_tail(odp_packet_t pkt) { - odp_packet_hdr_t *pkt_hdr = packet_last_hdr(pkt, NULL); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); return packet_tail(pkt_hdr); } @@ -782,166 +968,6 @@ void *odp_packet_push_head(odp_packet_t pkt, uint32_t len) return packet_data(pkt_hdr); } -static inline uint32_t pack_seg_head(odp_packet_hdr_t *pkt_hdr, int seg) -{ - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[seg].hdr; - uint32_t len = pkt_hdr->buf_hdr.seg[seg].len; - uint8_t *src = pkt_hdr->buf_hdr.seg[seg].data; - uint8_t *dst = hdr->base_data; - - if (dst != src) { - memmove(dst, src, len); - pkt_hdr->buf_hdr.seg[seg].data = dst; - } - - return len; -} - -static inline uint32_t pack_seg_tail(odp_packet_hdr_t *pkt_hdr, int seg) -{ - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[seg].hdr; - uint32_t len = pkt_hdr->buf_hdr.seg[seg].len; - uint8_t *src = pkt_hdr->buf_hdr.seg[seg].data; - uint8_t *dst = hdr->base_data + BASE_LEN - len; - - if (dst != src) { - memmove(dst, src, len); - pkt_hdr->buf_hdr.seg[seg].data = dst; - } - - return len; -} - -static inline uint32_t fill_seg_head(odp_packet_hdr_t *pkt_hdr, int dst_seg, - int src_seg, uint32_t max_len) -{ - uint32_t len = pkt_hdr->buf_hdr.seg[src_seg].len; - uint8_t *src = pkt_hdr->buf_hdr.seg[src_seg].data; - uint32_t offset = pkt_hdr->buf_hdr.seg[dst_seg].len; - uint8_t *dst = pkt_hdr->buf_hdr.seg[dst_seg].data + offset; - - if (len > max_len) - len = max_len; - - memmove(dst, src, len); - - pkt_hdr->buf_hdr.seg[dst_seg].len += len; - pkt_hdr->buf_hdr.seg[src_seg].len -= len; - pkt_hdr->buf_hdr.seg[src_seg].data += len; - - if (pkt_hdr->buf_hdr.seg[src_seg].len == 0) { - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[src_seg].hdr; - - pkt_hdr->buf_hdr.seg[src_seg].data = hdr->base_data; - } - - return len; -} - -static inline uint32_t fill_seg_tail(odp_packet_hdr_t *pkt_hdr, int dst_seg, - int src_seg, uint32_t max_len) -{ - uint32_t src_len = pkt_hdr->buf_hdr.seg[src_seg].len; - uint8_t *src = pkt_hdr->buf_hdr.seg[src_seg].data; - uint8_t *dst = pkt_hdr->buf_hdr.seg[dst_seg].data; - uint32_t len = src_len; - - if (len > max_len) - len = max_len; - - src += src_len - len; - dst -= len; - - memmove(dst, src, len); - - pkt_hdr->buf_hdr.seg[dst_seg].data -= len; - pkt_hdr->buf_hdr.seg[dst_seg].len += len; - pkt_hdr->buf_hdr.seg[src_seg].len -= len; - - if (pkt_hdr->buf_hdr.seg[src_seg].len == 0) { - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[src_seg].hdr; - - pkt_hdr->buf_hdr.seg[src_seg].data = hdr->base_data; - } - - return len; -} - -static inline int move_data_to_head(odp_packet_hdr_t *pkt_hdr, int segs) -{ - int dst_seg, src_seg; - uint32_t len, free_len; - uint32_t moved = 0; - - for (dst_seg = 0; dst_seg < segs; dst_seg++) { - len = pack_seg_head(pkt_hdr, dst_seg); - moved += len; - - if (len == BASE_LEN) - continue; - - free_len = BASE_LEN - len; - - for (src_seg = dst_seg + 1; CONFIG_PACKET_MAX_SEGS > 1 && - src_seg < segs; src_seg++) { - len = fill_seg_head(pkt_hdr, dst_seg, src_seg, - free_len); - moved += len; - - if (len == free_len) { - /* dst seg is full */ - break; - } - - /* src seg is empty */ - free_len -= len; - } - - if (moved == pkt_hdr->frame_len) - break; - } - - /* last segment which have data */ - return dst_seg; -} - -static inline int move_data_to_tail(odp_packet_hdr_t *pkt_hdr, int segs) -{ - int dst_seg, src_seg; - uint32_t len, free_len; - uint32_t moved = 0; - - for (dst_seg = segs - 1; dst_seg >= 0; dst_seg--) { - len = pack_seg_tail(pkt_hdr, dst_seg); - moved += len; - - if (len == BASE_LEN) - continue; - - free_len = BASE_LEN - len; - - for (src_seg = dst_seg - 1; src_seg >= 0; src_seg--) { - len = fill_seg_tail(pkt_hdr, dst_seg, src_seg, - free_len); - moved += len; - - if (len == free_len) { - /* dst seg is full */ - break; - } - - /* src seg is empty */ - free_len -= len; - } - - if (moved == pkt_hdr->frame_len) - break; - } - - /* first segment which have data */ - return dst_seg; -} - int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len) { @@ -953,109 +979,32 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, if (len > headroom) { pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; int num; - int segs; + void *ptr; if (odp_unlikely((frame_len + len) > pool->max_len)) return -1; - num = num_segments(len - headroom); - segs = pkt_hdr->buf_hdr.segcount; - - if (odp_unlikely((segs + num) > CONFIG_PACKET_MAX_SEGS)) { - /* Corner case: fail request if packet has - * references since we cannot shuffle segments - * since another thread may be accessing them - * concurrently */ - if (packet_ref_count(pkt_hdr) > 1) - return -1; - - /* Cannot directly add new segments */ - odp_packet_hdr_t *new_hdr; - int new_segs = 0; - int free_segs = 0; - uint32_t offset; - - num = num_segments(frame_len + len); - - if (num > segs) { - /* Allocate additional segments */ - new_segs = num - segs; - new_hdr = alloc_segments(pool, new_segs); - - if (new_hdr == NULL) - return -1; - - } else if (num < segs) { - free_segs = segs - num; - } - - /* Pack all data to packet tail */ - move_data_to_tail(pkt_hdr, segs); - reset_seg(pkt_hdr, 0, segs); - - if (new_segs) { - add_all_segs(new_hdr, pkt_hdr); - packet_seg_copy_md(new_hdr, pkt_hdr); - segs += new_segs; - - pkt_hdr = new_hdr; - *pkt = packet_handle(pkt_hdr); - } else if (CONFIG_PACKET_MAX_SEGS > 1 && free_segs) { - new_hdr = pkt_hdr->buf_hdr.seg[free_segs].hdr; - packet_seg_copy_md(new_hdr, pkt_hdr); - - /* Free extra segs */ - free_bufs(pkt_hdr, 0, free_segs); + num = num_segments(len - headroom); + push_head(pkt_hdr, headroom); + ptr = add_segments(pkt_hdr, pool, len - headroom, num, 1); - segs -= free_segs; - pkt_hdr = new_hdr; - *pkt = packet_handle(pkt_hdr); - } - - frame_len += len; - offset = (segs * BASE_LEN) - frame_len; - - pkt_hdr->buf_hdr.seg[0].data += offset; - pkt_hdr->buf_hdr.seg[0].len -= offset; - - pkt_hdr->buf_hdr.segcount = segs; - pkt_hdr->frame_len = frame_len; - pkt_hdr->unshared_len = frame_len; - pkt_hdr->headroom = offset + pool->headroom; - pkt_hdr->tailroom = pool->tailroom; - - /* Data was moved */ - ret = 1; - } else { - void *ptr; - - push_head(pkt_hdr, headroom); - ptr = add_segments(pkt_hdr, pool, len - headroom, - num, 1); - - if (ptr == NULL) { - /* segment alloc failed, rollback changes */ - pull_head(pkt_hdr, headroom); - return -1; - } - - *pkt = packet_handle(ptr); - pkt_hdr = ptr; + if (ptr == NULL) { + /* segment alloc failed, rollback changes */ + pull_head(pkt_hdr, headroom); + return -1; } + + *pkt = packet_handle(ptr); + pkt_hdr = ptr; } else { push_head(pkt_hdr, len); } - if (data_ptr || seg_len) { - uint32_t seg_ln = 0; - void *data = packet_map(pkt_hdr, 0, &seg_ln, NULL); - - if (data_ptr) - *data_ptr = data; + if (data_ptr) + *data_ptr = packet_data(pkt_hdr); - if (seg_len) - *seg_len = seg_ln; - } + if (seg_len) + *seg_len = packet_first_seg_len(pkt_hdr); return ret; } @@ -1067,8 +1016,6 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) if (len > pkt_hdr->frame_len) return NULL; - ODP_ASSERT(len <= pkt_hdr->unshared_len); - pull_head(pkt_hdr, len); return packet_data(pkt_hdr); } @@ -1076,35 +1023,15 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len_out) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt), *nxt_hdr; + odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); uint32_t seg_len = packet_first_seg_len(pkt_hdr); - int ret = 0; - if (len > packet_len(pkt_hdr)) + if (len > pkt_hdr->frame_len) return -1; - ODP_ASSERT(len <= odp_packet_unshared_len(*pkt)); - - /* Special processing for references */ - while (len >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); - nxt_hdr = pkt_hdr->ref_hdr; - len -= pkt_hdr->frame_len; - len += pkt_hdr->ref_offset + - (nxt_hdr->frame_len - pkt_hdr->ref_len); - pkt_hdr->ref_hdr = NULL; - packet_free(pkt_hdr); - pkt_hdr = nxt_hdr; - seg_len = packet_first_seg_len(pkt_hdr); - *pkt = packet_handle(pkt_hdr); - ret = 1; - } - - if (CONFIG_PACKET_MAX_SEGS == 1 || - len < seg_len || - pkt_hdr->buf_hdr.segcount == 1) { + if (len < seg_len) { pull_head(pkt_hdr, len); - } else { + } else if (CONFIG_PACKET_MAX_SEGS != 1) { int num = 0; uint32_t pull_len = 0; @@ -1119,29 +1046,23 @@ int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, *pkt = packet_handle(pkt_hdr); } - if (data_ptr || seg_len_out) { - void *data_head = packet_map(pkt_hdr, 0, &seg_len, NULL); - - if (data_ptr) - *data_ptr = data_head; + if (data_ptr) + *data_ptr = packet_data(pkt_hdr); - if (seg_len_out) - *seg_len_out = seg_len; - } + if (seg_len_out) + *seg_len_out = packet_first_seg_len(pkt_hdr); - return ret; + return 0; } void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) { - odp_packet_hdr_t *pkt_hdr = packet_last_hdr(pkt, NULL); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); void *old_tail; if (len > pkt_hdr->tailroom) return NULL; - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); - old_tail = packet_tail(pkt_hdr); push_tail(pkt_hdr, len); @@ -1151,86 +1072,28 @@ void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len_out) { - odp_packet_hdr_t *pkt_hdr = packet_last_hdr(*pkt, NULL); + odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); uint32_t frame_len = pkt_hdr->frame_len; uint32_t tailroom = pkt_hdr->tailroom; uint32_t tail_off = frame_len; int ret = 0; - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); - if (len > tailroom) { pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; int num; - int segs; + void *ptr; if (odp_unlikely((frame_len + len) > pool->max_len)) return -1; - num = num_segments(len - tailroom); - segs = pkt_hdr->buf_hdr.segcount; - - if (odp_unlikely((segs + num) > CONFIG_PACKET_MAX_SEGS)) { - /* Cannot directly add new segments */ - odp_packet_hdr_t *new_hdr; - int new_segs = 0; - int free_segs = 0; - uint32_t offset; - - num = num_segments(frame_len + len); - - if (num > segs) { - /* Allocate additional segments */ - new_segs = num - segs; - new_hdr = alloc_segments(pool, new_segs); - - if (new_hdr == NULL) - return -1; - - } else if (num < segs) { - free_segs = segs - num; - } - - /* Pack all data to packet head */ - move_data_to_head(pkt_hdr, segs); - reset_seg(pkt_hdr, 0, segs); + num = num_segments(len - tailroom); + push_tail(pkt_hdr, tailroom); + ptr = add_segments(pkt_hdr, pool, len - tailroom, num, 0); - if (new_segs) { - /* Add new segs */ - add_all_segs(pkt_hdr, new_hdr); - segs += new_segs; - } else if (free_segs) { - /* Free extra segs */ - free_bufs(pkt_hdr, segs - free_segs, free_segs); - - segs -= free_segs; - } - - frame_len += len; - offset = (segs * BASE_LEN) - frame_len; - - pkt_hdr->buf_hdr.seg[segs - 1].len -= offset; - - pkt_hdr->buf_hdr.segcount = segs; - pkt_hdr->frame_len = frame_len; - pkt_hdr->headroom = pool->headroom; - pkt_hdr->tailroom = offset + pool->tailroom; - - /* Data was moved */ - ret = 1; - } else { - void *ptr; - - push_tail(pkt_hdr, tailroom); - - ptr = add_segments(pkt_hdr, pool, len - tailroom, - num, 0); - - if (ptr == NULL) { - /* segment alloc failed, rollback changes */ - pull_tail(pkt_hdr, tailroom); - return -1; - } + if (ptr == NULL) { + /* segment alloc failed, rollback changes */ + pull_tail(pkt_hdr, tailroom); + return -1; } } else { push_tail(pkt_hdr, len); @@ -1245,11 +1108,11 @@ int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + seg_entry_t *last_seg = seg_entry_last(pkt_hdr); - if (len > packet_last_seg_len(pkt_hdr)) + if (len > last_seg->len) return NULL; - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); pull_tail(pkt_hdr, len); return packet_tail(pkt_hdr); @@ -1260,34 +1123,19 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, { int last; uint32_t seg_len; - uint32_t offset; - odp_packet_hdr_t *first_hdr = packet_hdr(*pkt); - odp_packet_hdr_t *pkt_hdr, *prev_hdr; + seg_entry_t *last_seg; + odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); - if (len > packet_len(first_hdr)) + if (len > pkt_hdr->frame_len) return -1; - pkt_hdr = packet_last_hdr(*pkt, &offset); + last = packet_last_seg(pkt_hdr); + last_seg = seg_entry_last(pkt_hdr); + seg_len = last_seg->len; - /* Special processing for references */ - while (len >= pkt_hdr->frame_len - offset && first_hdr->ref_hdr) { - len -= (pkt_hdr->frame_len - offset); - prev_hdr = packet_prev_hdr(first_hdr, pkt_hdr, &offset); - ODP_ASSERT(packet_ref_count(prev_hdr) == 1); - prev_hdr->ref_hdr = NULL; - packet_free(pkt_hdr); - pkt_hdr = prev_hdr; - } - - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); - last = packet_last_seg(pkt_hdr); - seg_len = packet_seg_len(pkt_hdr, last); - - if (CONFIG_PACKET_MAX_SEGS == 1 || - len < seg_len || - pkt_hdr->buf_hdr.segcount == 1) { + if (len < seg_len) { pull_tail(pkt_hdr, len); - } else { + } else if (CONFIG_PACKET_MAX_SEGS != 1) { int num = 0; uint32_t pull_len = 0; @@ -1434,46 +1282,6 @@ void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) pkt_hdr->p.input_flags.timestamp = 1; } -int odp_packet_num_segs(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t segcount = 0, i; - uint32_t seg_offset = 0, offset; - - do { - segcount += pkt_hdr->buf_hdr.segcount - seg_offset; - offset = pkt_hdr->ref_offset; - pkt_hdr = pkt_hdr->ref_hdr; - if (pkt_hdr) { - for (i = 0, seg_offset = 0; - i < pkt_hdr->buf_hdr.segcount; - i++, seg_offset++) { - if (offset < pkt_hdr->buf_hdr.seg[i].len) - break; - offset -= pkt_hdr->buf_hdr.seg[i].len; - } - } - } while (pkt_hdr); - - return segcount; -} - -odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) -{ - return _odp_packet_seg_from_ndx(odp_packet_num_segs(pkt) - 1); -} - -odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >= - packet_last_seg(pkt_hdr))) - return ODP_PACKET_SEG_INVALID; - - return seg + 1; -} - /* * * Segment level @@ -1484,53 +1292,23 @@ odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg) void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t seg_offset = 0, offset = 0, i; - uint32_t seg_ndx = _odp_packet_seg_to_ndx(seg); - - while (seg_ndx >= pkt_hdr->buf_hdr.segcount - seg_offset && - pkt_hdr->ref_hdr) { - seg_ndx -= (pkt_hdr->buf_hdr.segcount - seg_offset); - offset = pkt_hdr->ref_offset; - pkt_hdr = pkt_hdr->ref_hdr; - for (i = 0, seg_offset = 0; - i < pkt_hdr->buf_hdr.segcount; - i++, seg_offset++) { - if (offset < pkt_hdr->buf_hdr.seg[i].len) - break; - offset -= pkt_hdr->buf_hdr.seg[i].len; - } - } - if (odp_unlikely(seg_ndx + seg_offset >= pkt_hdr->buf_hdr.segcount)) + if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >= + pkt_hdr->buf_hdr.segcount)) return NULL; - return packet_seg_data(pkt_hdr, seg_ndx + seg_offset) + offset; + return packet_seg_data(pkt_hdr, _odp_packet_seg_to_ndx(seg)); } uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t seg_offset = 0, offset = 0, i; - uint32_t seg_ndx = _odp_packet_seg_to_ndx(seg); - - while (seg_ndx >= pkt_hdr->buf_hdr.segcount - seg_offset && - pkt_hdr->ref_hdr) { - seg_ndx -= (pkt_hdr->buf_hdr.segcount - seg_offset); - offset = pkt_hdr->ref_offset; - pkt_hdr = pkt_hdr->ref_hdr; - for (i = 0, seg_offset = 0; - i < pkt_hdr->buf_hdr.segcount; - i++, seg_offset++) { - if (offset < pkt_hdr->buf_hdr.seg[i].len) - break; - offset -= pkt_hdr->buf_hdr.seg[i].len; - } - } - if (odp_unlikely(seg_ndx + seg_offset >= pkt_hdr->buf_hdr.segcount)) + if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >= + pkt_hdr->buf_hdr.segcount)) return 0; - return packet_seg_len(pkt_hdr, seg_ndx + seg_offset) - offset; + return packet_seg_len(pkt_hdr, _odp_packet_seg_to_ndx(seg)); } /* @@ -1551,8 +1329,6 @@ int odp_packet_add_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) if (offset > pktlen) return -1; - ODP_ASSERT(odp_packet_unshared_len(*pkt_ptr) >= offset); - newpkt = odp_packet_alloc(pool->pool_hdl, pktlen + len); if (newpkt == ODP_PACKET_INVALID) @@ -1576,15 +1352,13 @@ int odp_packet_rem_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) { odp_packet_t pkt = *pkt_ptr; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t pktlen = packet_len(pkt_hdr); + uint32_t pktlen = pkt_hdr->frame_len; pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; odp_packet_t newpkt; - if (offset + len > pktlen) + if (offset > pktlen || offset + len > pktlen) return -1; - ODP_ASSERT(odp_packet_unshared_len(*pkt_ptr) >= offset + len); - newpkt = odp_packet_alloc(pool->pool_hdl, pktlen - len); if (newpkt == ODP_PACKET_INVALID) @@ -1618,8 +1392,6 @@ int odp_packet_align(odp_packet_t *pkt, uint32_t offset, uint32_t len, if (align > ODP_CACHE_LINE_SIZE) return -1; - ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); - if (seglen >= len) { misalign = align <= 1 ? 0 : ROUNDUP_ALIGN(uaddr, align) - uaddr; @@ -1652,20 +1424,13 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) { odp_packet_hdr_t *dst_hdr = packet_hdr(*dst); odp_packet_hdr_t *src_hdr = packet_hdr(src); - int dst_segs = dst_hdr->buf_hdr.segcount; - int src_segs = src_hdr->buf_hdr.segcount; pool_t *dst_pool = dst_hdr->buf_hdr.pool_ptr; pool_t *src_pool = src_hdr->buf_hdr.pool_ptr; uint32_t dst_len = dst_hdr->frame_len; uint32_t src_len = src_hdr->frame_len; - ODP_ASSERT(packet_ref_count(dst_hdr) == 1); - - /* Do a copy if resulting packet would be out of segments or packets - * are from different pools or src is a reference. */ - if (odp_unlikely((dst_segs + src_segs) > CONFIG_PACKET_MAX_SEGS) || - odp_unlikely(dst_pool != src_pool) || - odp_unlikely(packet_ref_count(src_hdr) > 1)) { + /* Do a copy if packets are from different pools. */ + if (odp_unlikely(dst_pool != src_pool)) { if (odp_packet_extend_tail(dst, src_len, NULL, NULL) >= 0) { (void)odp_packet_copy_from_pkt(*dst, dst_len, src, 0, src_len); @@ -1680,9 +1445,8 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) add_all_segs(dst_hdr, src_hdr); - dst_hdr->frame_len = dst_len + src_len; - dst_hdr->unshared_len = dst_len + src_len; - dst_hdr->tailroom = src_hdr->tailroom; + dst_hdr->frame_len = dst_len + src_len; + dst_hdr->tailroom = src_hdr->tailroom; /* Data was not moved in memory */ return 0; @@ -1695,7 +1459,6 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) if (len >= pktlen || tail == NULL) return -1; - ODP_ASSERT(odp_packet_unshared_len(*pkt) >= len); *tail = odp_packet_copy_part(*pkt, len, pktlen - len, odp_packet_pool(*pkt)); @@ -1706,109 +1469,6 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) } /* - * References - */ - -static inline void packet_ref(odp_packet_hdr_t *pkt_hdr) -{ - uint32_t i; - odp_packet_hdr_t *hdr; - - do { - for (i = 0; i < pkt_hdr->buf_hdr.segcount; i++) { - hdr = pkt_hdr->buf_hdr.seg[i].hdr; - packet_ref_inc(hdr); - } - - pkt_hdr = pkt_hdr->ref_hdr; - } while (pkt_hdr); -} - -static inline odp_packet_t packet_splice(odp_packet_hdr_t *pkt_hdr, - uint32_t offset, - odp_packet_hdr_t *ref_hdr) -{ - /* Catch attempted references to stale handles in debug builds */ - ODP_ASSERT(packet_ref_count(pkt_hdr) > 0); - - /* Splicing is from the last section of src pkt */ - while (ref_hdr->ref_hdr) - ref_hdr = ref_hdr->ref_hdr; - - /* Find section where splice begins */ - while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { - offset -= (pkt_hdr->frame_len - pkt_hdr->ref_offset); - offset += (pkt_hdr->ref_hdr->frame_len - pkt_hdr->ref_len); - pkt_hdr = pkt_hdr->ref_hdr; - } - - ref_hdr->ref_hdr = pkt_hdr; - ref_hdr->ref_offset = offset; - ref_hdr->ref_len = pkt_hdr->frame_len; - - if (packet_ref_count(pkt_hdr) == 1 || offset < pkt_hdr->unshared_len) - pkt_hdr->unshared_len = offset; - - packet_ref(pkt_hdr); - return packet_handle(ref_hdr); -} - -odp_packet_t odp_packet_ref_static(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - packet_ref(pkt_hdr); - pkt_hdr->unshared_len = 0; - return pkt; -} - -odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) -{ - odp_packet_t hdr; - odp_packet_hdr_t *pkt_hdr; - - if (pkt == ODP_PACKET_INVALID) - return ODP_PACKET_INVALID; - - pkt_hdr = packet_hdr(pkt); - if (offset >= packet_len(pkt_hdr)) - return ODP_PACKET_INVALID; - - hdr = odp_packet_alloc(odp_packet_pool(pkt), 0); - - if (hdr == ODP_PACKET_INVALID) - return ODP_PACKET_INVALID; - - return packet_splice(pkt_hdr, offset, packet_hdr(hdr)); -} - -odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, - odp_packet_t hdr) -{ - odp_packet_hdr_t *pkt_hdr; - - if (pkt == ODP_PACKET_INVALID || - hdr == ODP_PACKET_INVALID || - pkt == hdr) - return ODP_PACKET_INVALID; - - ODP_ASSERT(odp_packet_has_ref(hdr) == 0); - - pkt_hdr = packet_hdr(pkt); - if (offset >= packet_len(pkt_hdr)) - return ODP_PACKET_INVALID; - - return packet_splice(pkt_hdr, offset, packet_hdr(hdr)); -} - -int odp_packet_has_ref(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - return pkt_hdr->ref_hdr != NULL || packet_ref_count(pkt_hdr) > 1; -} - -/* * * Copy * ******************************************************** @@ -1817,7 +1477,8 @@ int odp_packet_has_ref(odp_packet_t pkt) odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_pool_t pool) { - uint32_t pktlen = odp_packet_len(pkt); + odp_packet_hdr_t *srchdr = packet_hdr(pkt); + uint32_t pktlen = srchdr->frame_len; odp_packet_t newpkt = odp_packet_alloc(pool, pktlen); if (newpkt != ODP_PACKET_INVALID) { @@ -1856,7 +1517,7 @@ int odp_packet_copy_to_mem(odp_packet_t pkt, uint32_t offset, uint8_t *dstaddr = (uint8_t *)dst; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (offset + len > packet_len(pkt_hdr)) + if (offset + len > pkt_hdr->frame_len) return -1; while (len > 0) { @@ -1880,11 +1541,9 @@ int odp_packet_copy_from_mem(odp_packet_t pkt, uint32_t offset, const uint8_t *srcaddr = (const uint8_t *)src; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (offset + len > packet_len(pkt_hdr)) + if (offset + len > pkt_hdr->frame_len) return -1; - ODP_ASSERT(odp_packet_unshared_len(pkt) >= offset + len); - while (len > 0) { mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); cpylen = len > seglen ? seglen : len; @@ -1910,12 +1569,10 @@ int odp_packet_copy_from_pkt(odp_packet_t dst, uint32_t dst_offset, uint32_t src_seglen = 0; /* GCC */ int overlap; - if (dst_offset + len > packet_len(dst_hdr) || - src_offset + len > packet_len(src_hdr)) + if (dst_offset + len > dst_hdr->frame_len || + src_offset + len > src_hdr->frame_len) return -1; - ODP_ASSERT(odp_packet_unshared_len(dst) >= dst_offset + len); - overlap = (dst_hdr == src_hdr && ((dst_offset <= src_offset && dst_offset + len >= src_offset) || @@ -2021,11 +1678,13 @@ int _odp_packet_cmp_data(odp_packet_t pkt, uint32_t offset, * ******************************************************** * */ - void odp_packet_print(odp_packet_t pkt) { odp_packet_seg_t seg; - int max_len = 512; + seg_entry_t *seg_entry; + odp_packet_hdr_t *seg_hdr; + uint8_t idx; + int max_len = 1024; char str[max_len]; int len = 0; int n = max_len - 1; @@ -2048,7 +1707,7 @@ void odp_packet_print(odp_packet_t pkt) len += snprintf(&str[len], n - len, " l4_offset %" PRIu32 "\n", hdr->p.l4_offset); len += snprintf(&str[len], n - len, - " frame_len %" PRIu32 "\n", packet_len(hdr)); + " frame_len %" PRIu32 "\n", hdr->frame_len); len += snprintf(&str[len], n - len, " input %" PRIu64 "\n", odp_pktio_to_u64(hdr->input)); @@ -2061,12 +1720,32 @@ void odp_packet_print(odp_packet_t pkt) len += snprintf(&str[len], n - len, " num_segs %i\n", odp_packet_num_segs(pkt)); + seg_hdr = hdr; + idx = 0; seg = odp_packet_first_seg(pkt); while (seg != ODP_PACKET_SEG_INVALID) { + odp_buffer_hdr_t *buf_hdr; + odp_packet_hdr_t *tmp_hdr; + + tmp_hdr = seg_hdr; + seg_entry = seg_entry_next(&seg_hdr, &idx); + buf_hdr = seg_entry->hdr; + len += snprintf(&str[len], n - len, - " seg_len %" PRIu32 "\n", - odp_packet_seg_data_len(pkt, seg)); + " seg_len %-4" PRIu32 " seg_data %p ", + odp_packet_seg_data_len(pkt, seg), + odp_packet_seg_data(pkt, seg)); + len += snprintf(&str[len], n - len, "ref_cnt %u", + buffer_ref(buf_hdr)); + if (seg_is_link(tmp_hdr)) { + uint32_t ref; + + ref = buffer_ref(&tmp_hdr->buf_hdr); + len += snprintf(&str[len], n - len, "L(%u)\n", ref); + } else { + len += snprintf(&str[len], n - len, "\n"); + } seg = odp_packet_next_seg(pkt, seg); } @@ -2452,6 +2131,170 @@ uint64_t odp_packet_seg_to_u64(odp_packet_seg_t hdl) return _odp_pri(hdl); } +odp_packet_t odp_packet_ref_static(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + packet_ref_inc(pkt_hdr); + pkt_hdr->shared_len = pkt_hdr->frame_len; + + return pkt; +} + +odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) +{ + odp_packet_t ref; + odp_packet_hdr_t *link_hdr; + odp_packet_hdr_t *next_hdr; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + odp_packet_hdr_t *hdr = pkt_hdr; + seg_entry_t *seg; + uint32_t seg_idx = 0; + uint8_t idx = 0; + uint32_t seg_offset = 0; + int i, num_copy, segcount; + uint32_t len; + + if (offset >= pkt_hdr->frame_len) { + ODP_DBG("offset too large\n"); + return ODP_PACKET_INVALID; + } + + /* Allocate link segment */ + if (packet_alloc(pkt_hdr->buf_hdr.pool_ptr, 0, 1, 1, &ref) != 1) { + ODP_DBG("segment alloc failed\n"); + return ODP_PACKET_INVALID; + } + + link_hdr = packet_hdr(ref); + + seg_entry_find_offset(&hdr, &idx, &seg_offset, &seg_idx, offset); + num_copy = hdr->buf_hdr.num_seg - idx; + segcount = pkt_hdr->buf_hdr.segcount; + + /* In addition to segments, update reference count of + * an existing link header. */ + if (seg_is_link(hdr)) + buffer_ref_inc((odp_buffer_hdr_t *)hdr); + + seg = seg_entry_next(&hdr, &idx); + link_hdr->buf_hdr.num_seg = 1; + link_hdr->buf_hdr.seg[0].hdr = seg->hdr; + link_hdr->buf_hdr.seg[0].data = seg->data + seg_offset; + link_hdr->buf_hdr.seg[0].len = seg->len - seg_offset; + buffer_ref_inc(seg->hdr); + + for (i = 1; i < num_copy; i++) { + /* Update link header reference count */ + if (idx == 0 && seg_is_link(hdr)) + buffer_ref_inc((odp_buffer_hdr_t *)hdr); + + seg = seg_entry_next(&hdr, &idx); + + link_hdr->buf_hdr.num_seg++; + link_hdr->buf_hdr.seg[i].hdr = seg->hdr; + link_hdr->buf_hdr.seg[i].data = seg->data; + link_hdr->buf_hdr.seg[i].len = seg->len; + buffer_ref_inc(seg->hdr); + } + + next_hdr = hdr; + + /* Increment ref count for remaining segments */ + for (i = seg_idx + num_copy; i < segcount; i++) { + /* Update link header reference count */ + if (idx == 0 && seg_is_link(hdr)) + buffer_ref_inc((odp_buffer_hdr_t *)hdr); + + seg = seg_entry_next(&hdr, &idx); + buffer_ref_inc(seg->hdr); + } + + len = pkt_hdr->frame_len - offset; + link_hdr->buf_hdr.next_seg = next_hdr; + link_hdr->buf_hdr.last_seg = pkt_hdr->buf_hdr.last_seg; + link_hdr->buf_hdr.segcount = segcount - seg_idx; + link_hdr->frame_len = len; + link_hdr->tailroom = pkt_hdr->tailroom; + link_hdr->shared_len = len; + + /* Link header does not have headroom, it just points to other + * buffers. Zero length headroom ensures that head of the other buffer + * is not pushed through a reference. */ + link_hdr->headroom = 0; + + if (pkt_hdr->shared_len < len) + pkt_hdr->shared_len = len; + + return ref; + +} + +odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, + odp_packet_t hdr) +{ + odp_packet_t ref; + int ret; + odp_packet_hdr_t *new_hdr; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t len = pkt_hdr->frame_len; + + ref = odp_packet_ref(pkt, offset); + + if (ref == ODP_PACKET_INVALID) { + ODP_DBG("reference create failed\n"); + return ODP_PACKET_INVALID; + } + + ret = odp_packet_concat(&hdr, ref); + + if (ret < 0) { + ODP_DBG("concat failed\n"); + odp_packet_free(ref); + return ODP_PACKET_INVALID; + } + + new_hdr = packet_hdr(hdr); + new_hdr->shared_len = len - offset; + + return hdr; + +} + +int odp_packet_has_ref(odp_packet_t pkt) +{ + odp_buffer_hdr_t *buf_hdr; + seg_entry_t *seg; + int i; + uint32_t ref_cnt; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + int seg_count = pkt_hdr->buf_hdr.segcount; + odp_packet_hdr_t *hdr = pkt_hdr; + uint8_t idx = 0; + + for (i = 0; i < seg_count; i++) { + seg = seg_entry_next(&hdr, &idx); + buf_hdr = seg->hdr; + ref_cnt = buffer_ref(buf_hdr); + + if (is_multi_ref(ref_cnt)) + return 1; + } + + return 0; +} + +uint32_t odp_packet_unshared_len(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t len = pkt_hdr->frame_len; + + if (odp_packet_has_ref(pkt)) + return len - pkt_hdr->shared_len; + + return len; +} + /* Include non-inlined versions of API functions */ #if ODP_ABI_COMPAT == 1 #include <odp/api/plat/packet_inlines_api.h> diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 3e8ec6d85..39e010833 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -41,6 +41,9 @@ ODP_STATIC_ASSERT(CONFIG_POOL_CACHE_SIZE > (2 * CACHE_BURST), ODP_STATIC_ASSERT(CONFIG_PACKET_SEG_LEN_MIN >= 256, "ODP Segment size must be a minimum of 256 bytes"); +ODP_STATIC_ASSERT(CONFIG_PACKET_SEG_SIZE < 0xffff, + "Segment size must be less than 64k (16 bit offsets)"); + /* Thread local variables */ typedef struct pool_local_t { pool_cache_t *cache[ODP_CONFIG_POOLS]; @@ -269,12 +272,17 @@ static void init_buffers(pool_t *pool) /* Show user requested size through API */ buf_hdr->uarea_size = pool->params.pkt.uarea_size; buf_hdr->segcount = 1; + buf_hdr->num_seg = 1; + buf_hdr->next_seg = NULL; + buf_hdr->last_seg = buf_hdr; /* Pointer to data start (of the first segment) */ buf_hdr->seg[0].hdr = buf_hdr; buf_hdr->seg[0].data = &data[offset]; buf_hdr->seg[0].len = pool->data_size; + odp_atomic_init_u32(&buf_hdr->ref_cnt, 0); + /* Store base values for fast init */ buf_hdr->base_data = buf_hdr->seg[0].data; buf_hdr->buf_end = &data[offset + pool->data_size + diff --git a/test/common_plat/validation/api/packet/packet.c b/test/common_plat/validation/api/packet/packet.c index 8dde0ecae..d1c42c1fe 100644 --- a/test/common_plat/validation/api/packet/packet.c +++ b/test/common_plat/validation/api/packet/packet.c @@ -1395,7 +1395,7 @@ void packet_test_concat_small(void) int ret; uint8_t *data; uint32_t i; - uint32_t len = 32000; + uint32_t len = PACKET_POOL_NUM / 4; uint8_t buf[len]; CU_ASSERT_FATAL(odp_pool_capability(&capa) == 0); @@ -2091,14 +2091,107 @@ void packet_test_ref(void) { odp_packet_t base_pkt, segmented_base_pkt, hdr_pkt[4], ref_pkt[4], refhdr_pkt[4], hdr_cpy; + odp_packet_t pkt, pkt2, pkt3, ref, ref2; uint32_t pkt_len, segmented_pkt_len, hdr_len[4], offset[4], hr[4], base_hr, ref_len[4]; - int i; + int i, ret; + odp_pool_t pool; + + /* Create references and compare data */ + pool = odp_packet_pool(test_packet); + + pkt = odp_packet_copy(test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + ref = odp_packet_ref_static(pkt); + CU_ASSERT_FATAL(ref != ODP_PACKET_INVALID) + packet_compare_data(pkt, ref); + odp_packet_free(ref); + odp_packet_free(pkt); + + pkt = odp_packet_copy(test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + ref = odp_packet_ref(pkt, 0); + CU_ASSERT_FATAL(ref != ODP_PACKET_INVALID) + packet_compare_data(pkt, ref); + odp_packet_free(ref); + odp_packet_free(pkt); + + pkt = odp_packet_copy(test_packet, pool); + pkt3 = odp_packet_copy(test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + CU_ASSERT_FATAL(pkt3 != ODP_PACKET_INVALID) + ret = odp_packet_concat(&pkt3, pkt); + CU_ASSERT_FATAL(ret >= 0); + + pkt = odp_packet_copy(test_packet, pool); + pkt2 = odp_packet_copy(test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + CU_ASSERT_FATAL(pkt2 != ODP_PACKET_INVALID) + ref = odp_packet_ref_pkt(pkt, 0, pkt2); + CU_ASSERT_FATAL(ref != ODP_PACKET_INVALID) + packet_compare_data(pkt3, ref); + odp_packet_free(ref); + odp_packet_free(pkt); + odp_packet_free(pkt3); + /* Do the same for segmented packets */ + pool = odp_packet_pool(segmented_test_packet); + + pkt = odp_packet_copy(segmented_test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + ref = odp_packet_ref_static(pkt); + CU_ASSERT_FATAL(ref != ODP_PACKET_INVALID) + packet_compare_data(pkt, ref); + odp_packet_free(ref); + odp_packet_free(pkt); + + pkt = odp_packet_copy(segmented_test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + ref = odp_packet_ref(pkt, 0); + CU_ASSERT_FATAL(ref != ODP_PACKET_INVALID) + packet_compare_data(pkt, ref); + odp_packet_free(ref); + odp_packet_free(pkt); + + /* Avoid to create too large packets with concat */ + pool = odp_packet_pool(test_packet); + + pkt = odp_packet_copy(test_packet, pool); + pkt2 = odp_packet_copy(test_packet, pool); + pkt3 = odp_packet_copy(test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + CU_ASSERT_FATAL(pkt2 != ODP_PACKET_INVALID) + CU_ASSERT_FATAL(pkt3 != ODP_PACKET_INVALID) + ret = odp_packet_concat(&pkt3, pkt2); + CU_ASSERT_FATAL(ret >= 0); + ret = odp_packet_concat(&pkt3, pkt); + CU_ASSERT_FATAL(ret >= 0); + + pkt = odp_packet_copy(test_packet, pool); + pkt2 = odp_packet_copy(test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + CU_ASSERT_FATAL(pkt2 != ODP_PACKET_INVALID) + ref = odp_packet_ref_pkt(pkt, 0, pkt2); + CU_ASSERT_FATAL(ref != ODP_PACKET_INVALID) + pkt2 = odp_packet_copy(test_packet, pool); + CU_ASSERT_FATAL(pkt2 != ODP_PACKET_INVALID) + ref2 = odp_packet_ref_pkt(ref, 0, pkt2); + CU_ASSERT_FATAL(ref2 != ODP_PACKET_INVALID) + packet_compare_data(pkt3, ref2); + + /* Try print function on a reference */ + odp_packet_print(ref2); + + odp_packet_free(ref); + odp_packet_free(ref2); + odp_packet_free(pkt); + odp_packet_free(pkt3); + + /* Test has_ref, unshared_len, lengths, etc */ base_pkt = odp_packet_copy(test_packet, odp_packet_pool(test_packet)); - base_hr = odp_packet_headroom(base_pkt); - pkt_len = odp_packet_len(test_packet); CU_ASSERT_FATAL(base_pkt != ODP_PACKET_INVALID); + base_hr = odp_packet_headroom(base_pkt); + pkt_len = odp_packet_len(test_packet); segmented_base_pkt = odp_packet_copy(segmented_test_packet, @@ -2243,8 +2336,14 @@ void packet_test_ref(void) CU_ASSERT(ref_pkt[0] != ODP_PACKET_INVALID); if (odp_packet_has_ref(base_pkt) == 1) { - /* CU_ASSERT needs braces */ CU_ASSERT(odp_packet_has_ref(ref_pkt[0]) == 1); + CU_ASSERT(odp_packet_unshared_len(base_pkt) == 0); + CU_ASSERT(odp_packet_unshared_len(ref_pkt[0]) == 0); + } else { + CU_ASSERT(odp_packet_unshared_len(base_pkt) == + odp_packet_len(base_pkt)); + CU_ASSERT(odp_packet_unshared_len(ref_pkt[0]) == + odp_packet_len(ref_pkt[0])); } CU_ASSERT(odp_packet_len(ref_pkt[0]) == odp_packet_len(base_pkt)); |