aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMatias Elo <matias.elo@nokia.com>2022-05-25 12:50:34 +0300
committerGitHub <noreply@github.com>2022-05-25 12:50:34 +0300
commit9951d549ce6c8dec08af9ab04ae2bd61cf489855 (patch)
tree50c987b4844596abe30134000c4a77ad469cd0ee
parent2f869d2698349bf75ec2404d4ab7d7f3b5b4c88b (diff)
parent62c95b2e71aab0e3af543e0df55ebf14ea306a11 (diff)
Merge ODP v1.36.0.0v1.36.0.0_DPDK_19.11
Merge ODP linux-generic v1.36.0.0 into linux-dpdk
-rw-r--r--.github/workflows/ci-pipeline-arm64.yml34
-rw-r--r--.github/workflows/ci-pipeline.yml41
-rw-r--r--CHANGELOG33
-rw-r--r--DEPENDENCIES13
-rw-r--r--configure.ac53
-rw-r--r--doc/users-guide/users-guide-crypto.adoc132
-rw-r--r--doc/users-guide/users-guide-pktio.adoc2
-rw-r--r--example/generator/odp_generator.c10
-rw-r--r--example/ipfragreass/odp_ipfragreass_reassemble.c3
-rwxr-xr-xexample/ipsec_api/odp_ipsec_api_run_esp_tun_out.sh1
-rw-r--r--example/ipsec_crypto/odp_ipsec_stream.c6
-rw-r--r--helper/include/odp/helper/cli.h2
-rw-r--r--helper/include/odp/helper/linux/process.h1
-rw-r--r--helper/include/odp/helper/linux/pthread.h1
-rw-r--r--helper/include/odp/helper/threads.h1
-rw-r--r--include/Makefile.am16
-rw-r--r--include/odp/api/abi-default/event.h38
-rw-r--r--include/odp/api/abi-default/event_types.h58
-rw-r--r--include/odp/api/abi-default/timer.h20
-rw-r--r--include/odp/api/abi-default/traffic_mngr.h5
-rw-r--r--include/odp/api/buffer.h2
-rw-r--r--include/odp/api/comp.h2
-rw-r--r--include/odp/api/dma_types.h3
-rw-r--r--include/odp/api/event_types.h28
-rw-r--r--include/odp/api/packet.h6
-rw-r--r--include/odp/api/queue.h5
-rw-r--r--include/odp/api/spec/buffer.h2
-rw-r--r--include/odp/api/spec/classification.h133
-rw-r--r--include/odp/api/spec/comp.h5
-rw-r--r--include/odp/api/spec/crypto.h39
-rw-r--r--include/odp/api/spec/dma_types.h2
-rw-r--r--include/odp/api/spec/event.h79
-rw-r--r--include/odp/api/spec/event_types.h110
-rw-r--r--include/odp/api/spec/packet.h7
-rw-r--r--include/odp/api/spec/packet_flags.h2
-rw-r--r--include/odp/api/spec/packet_io.h5
-rw-r--r--include/odp/api/spec/packet_io_types.h8
-rw-r--r--include/odp/api/spec/pool_types.h10
-rw-r--r--include/odp/api/spec/queue.h3
-rw-r--r--include/odp/api/spec/schedule.h2
-rw-r--r--include/odp/api/spec/shared_memory.h2
-rw-r--r--include/odp/api/spec/timer.h2
-rw-r--r--include/odp/api/spec/timer_types.h2
-rw-r--r--include/odp/api/spec/traffic_mngr.h249
-rw-r--r--include/odp/api/timer.h2
-rw-r--r--include/odp/arch/arm32-linux/odp/api/abi/event_types.h7
-rw-r--r--include/odp/arch/arm32-linux/odp/api/abi/timer.h7
-rw-r--r--include/odp/arch/arm64-linux/odp/api/abi/event_types.h7
-rw-r--r--include/odp/arch/arm64-linux/odp/api/abi/timer.h7
-rw-r--r--include/odp/arch/default-linux/odp/api/abi/event_types.h7
-rw-r--r--include/odp/arch/default-linux/odp/api/abi/timer.h7
-rw-r--r--include/odp/arch/power64-linux/odp/api/abi/event_types.h7
-rw-r--r--include/odp/arch/power64-linux/odp/api/abi/timer.h7
-rw-r--r--include/odp/arch/x86_32-linux/odp/api/abi/event_types.h7
-rw-r--r--include/odp/arch/x86_32-linux/odp/api/abi/timer.h7
-rw-r--r--include/odp/arch/x86_64-linux/odp/api/abi/event_types.h7
-rw-r--r--include/odp/arch/x86_64-linux/odp/api/abi/timer.h7
-rw-r--r--platform/linux-dpdk/Makefile.am14
-rw-r--r--platform/linux-dpdk/README21
l---------platform/linux-dpdk/include-abi/odp/api/abi/event_types.h1
l---------platform/linux-dpdk/include-abi/odp/api/abi/timer.h1
-rw-r--r--platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h2
-rw-r--r--platform/linux-dpdk/include/odp/api/plat/packet_flag_inlines.h194
-rw-r--r--platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h7
-rw-r--r--platform/linux-dpdk/include/odp/api/plat/packet_inlines.h32
-rw-r--r--platform/linux-dpdk/include/odp/api/plat/time_inlines.h12
-rw-r--r--platform/linux-dpdk/include/odp/api/plat/timer_inline_types.h37
-rw-r--r--platform/linux-dpdk/include/odp/api/plat/timer_inlines.h60
-rw-r--r--platform/linux-dpdk/include/odp_packet_dpdk.h55
-rw-r--r--platform/linux-dpdk/include/odp_packet_internal.h44
-rw-r--r--platform/linux-dpdk/include/odp_packet_io_internal.h20
-rw-r--r--platform/linux-dpdk/odp_crypto.c4
-rw-r--r--platform/linux-dpdk/odp_packet.c575
-rw-r--r--platform/linux-dpdk/odp_packet_dpdk.c164
-rw-r--r--platform/linux-dpdk/odp_packet_flags.c138
-rw-r--r--platform/linux-dpdk/odp_time.c11
-rw-r--r--platform/linux-dpdk/odp_timer.c44
-rw-r--r--platform/linux-generic/Makefile.am12
-rw-r--r--platform/linux-generic/arch/aarch64/odp_atomic.h95
-rw-r--r--platform/linux-generic/arch/aarch64/odp_crypto_armv8.c265
-rw-r--r--platform/linux-generic/arch/aarch64/odp_llsc.h8
-rw-r--r--platform/linux-generic/arch/arm/odp_atomic.h3
-rw-r--r--platform/linux-generic/arch/default/odp_atomic.h14
-rw-r--r--platform/linux-generic/include-abi/odp/api/abi/event.h34
-rw-r--r--platform/linux-generic/include-abi/odp/api/abi/event_types.h61
-rw-r--r--platform/linux-generic/include-abi/odp/api/abi/timer.h8
-rw-r--r--platform/linux-generic/include/odp/api/plat/buffer_inlines.h2
-rw-r--r--platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h195
-rw-r--r--platform/linux-generic/include/odp/api/plat/packet_inline_types.h7
-rw-r--r--platform/linux-generic/include/odp/api/plat/packet_inlines.h32
-rw-r--r--platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h2
-rw-r--r--platform/linux-generic/include/odp/api/plat/queue_inline_types.h1
-rw-r--r--platform/linux-generic/include/odp/api/plat/time_inlines.h12
-rw-r--r--platform/linux-generic/include/odp/api/plat/timer_inline_types.h37
-rw-r--r--platform/linux-generic/include/odp/api/plat/timer_inlines.h78
-rw-r--r--platform/linux-generic/include/odp_atomic_internal.h13
-rw-r--r--platform/linux-generic/include/odp_classification_datamodel.h4
-rw-r--r--platform/linux-generic/include/odp_classification_internal.h7
-rw-r--r--platform/linux-generic/include/odp_llqueue.h3
-rw-r--r--platform/linux-generic/include/odp_macros_internal.h2
-rw-r--r--platform/linux-generic/include/odp_packet_dpdk.h14
-rw-r--r--platform/linux-generic/include/odp_packet_internal.h48
-rw-r--r--platform/linux-generic/include/odp_packet_io_internal.h18
-rw-r--r--platform/linux-generic/include/odp_parse_internal.h112
-rw-r--r--platform/linux-generic/include/odp_schedule_if.h2
-rw-r--r--platform/linux-generic/include/odp_types_internal.h24
-rw-r--r--platform/linux-generic/odp_classification.c119
-rw-r--r--platform/linux-generic/odp_crypto_null.c7
-rw-r--r--platform/linux-generic/odp_crypto_openssl.c10
-rw-r--r--platform/linux-generic/odp_dma.c1
-rw-r--r--platform/linux-generic/odp_event.c5
-rw-r--r--platform/linux-generic/odp_ipsec.c7
-rw-r--r--platform/linux-generic/odp_ipsec_sad.c5
-rw-r--r--platform/linux-generic/odp_packet.c568
-rw-r--r--platform/linux-generic/odp_packet_flags.c138
-rw-r--r--platform/linux-generic/odp_packet_io.c122
-rw-r--r--platform/linux-generic/odp_parse.c471
-rw-r--r--platform/linux-generic/odp_pool.c13
-rw-r--r--platform/linux-generic/odp_queue_lf.c17
-rw-r--r--platform/linux-generic/odp_schedule_scalable.c16
-rw-r--r--platform/linux-generic/odp_schedule_sp.c1
-rw-r--r--platform/linux-generic/odp_time.c9
-rw-r--r--platform/linux-generic/odp_timer.c127
-rw-r--r--platform/linux-generic/odp_timer_api.c11
-rw-r--r--platform/linux-generic/odp_traffic_mngr.c123
-rw-r--r--platform/linux-generic/pktio/dpdk.c254
-rw-r--r--platform/linux-generic/pktio/dpdk_parse.c28
-rw-r--r--platform/linux-generic/pktio/loop.c96
-rw-r--r--platform/linux-generic/pktio/netmap.c53
-rw-r--r--platform/linux-generic/pktio/null.c4
-rw-r--r--platform/linux-generic/pktio/pcap.c69
-rw-r--r--platform/linux-generic/pktio/pktio_common.c12
-rw-r--r--platform/linux-generic/pktio/socket.c67
-rw-r--r--platform/linux-generic/pktio/socket_mmap.c44
-rw-r--r--platform/linux-generic/pktio/tap.c42
-rwxr-xr-xscripts/ci/build.sh3
-rwxr-xr-xscripts/ci/build_arm64.sh29
-rw-r--r--test/performance/odp_bench_packet.c6
-rw-r--r--test/performance/odp_crypto.c1
-rw-r--r--test/performance/odp_ipsec.c415
-rwxr-xr-xtest/performance/odp_ipsec_run.sh4
-rw-r--r--test/performance/odp_pktio_ordered.c2
-rw-r--r--test/performance/odp_random.c445
-rw-r--r--test/performance/odp_sched_pktio.c3
-rw-r--r--test/validation/api/classification/classification.h7
-rw-r--r--test/validation/api/classification/odp_classification_basic.c45
-rw-r--r--test/validation/api/classification/odp_classification_tests.c76
-rw-r--r--test/validation/api/classification/odp_classification_testsuites.h3
-rw-r--r--test/validation/api/crypto/odp_crypto_test_inp.c14
-rw-r--r--test/validation/api/ipsec/ipsec.c9
-rw-r--r--test/validation/api/pktio/pktio.c11
-rw-r--r--test/validation/api/traffic_mngr/traffic_mngr.c218
152 files changed, 4254 insertions, 3289 deletions
diff --git a/.github/workflows/ci-pipeline-arm64.yml b/.github/workflows/ci-pipeline-arm64.yml
index 6f32208dc..b16417ade 100644
--- a/.github/workflows/ci-pipeline-arm64.yml
+++ b/.github/workflows/ci-pipeline-arm64.yml
@@ -31,6 +31,9 @@ jobs:
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}"
-e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/build_${ARCH}.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Build_OS:
if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }}
@@ -39,12 +42,15 @@ jobs:
fail-fast: false
matrix:
cc: [gcc, clang]
- os: ['ubuntu_16.04', 'ubuntu_18.04']
+ os: ['ubuntu_18.04', 'rocky_linux_8']
steps:
- uses: AutoModality/action-clean@v1.1.0
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}"
-e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${{matrix.os}}-${ARCH}-native /odp/scripts/ci/build_${ARCH}.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Build_gcc-10:
if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }}
@@ -58,8 +64,11 @@ jobs:
steps:
- uses: AutoModality/action-clean@v1.1.0
- uses: actions/checkout@v2
- - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}"
+ - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e CXX=g++-10
-e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/build_${ARCH}.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Build_out-of-tree:
if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }}
@@ -69,6 +78,9 @@ jobs:
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}"
-e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/out_of_tree.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Run_distcheck:
if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }}
@@ -130,11 +142,11 @@ jobs:
strategy:
fail-fast: false
matrix:
- os: ['ubuntu_18.04']
+ os: ['ubuntu_18.04', 'ubuntu_22.04-openssl']
steps:
- uses: AutoModality/action-clean@v1.1.0
- uses: actions/checkout@v2
- - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e ARCH="${ARCH}"
+ - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}"
-e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${{matrix.os}}-${ARCH}-native /odp/scripts/ci/check.sh
- name: Failure log
if: ${{ failure() }}
@@ -177,3 +189,17 @@ jobs:
- name: Failure log
if: ${{ failure() }}
run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done
+
+ Run_dpdk-21_11:
+ if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }}
+ runs-on: [self-hosted, ARM64]
+ env:
+ OS: ubuntu_20.04
+ steps:
+ - uses: AutoModality/action-clean@v1.1.0
+ - uses: actions/checkout@v2
+ - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}"
+ -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native-dpdk_21.11 /odp/scripts/ci/check.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done
diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml
index 56dd65fcc..fd76a2f82 100644
--- a/.github/workflows/ci-pipeline.yml
+++ b/.github/workflows/ci-pipeline.yml
@@ -78,6 +78,9 @@ jobs:
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}"
-e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Build_arm64:
runs-on: ubuntu-18.04
@@ -118,6 +121,9 @@ jobs:
OS: ubuntu_20.04
run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.compiler}}"
-e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Build_ppc64el:
runs-on: ubuntu-18.04
@@ -126,12 +132,14 @@ jobs:
strategy:
fail-fast: false
matrix:
- cc: [gcc, clang]
conf: ['', '--enable-abi-compat']
steps:
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}"
-e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Build_i386:
runs-on: ubuntu-18.04
@@ -146,6 +154,9 @@ jobs:
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}"
-e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Build_OS:
runs-on: ubuntu-18.04
@@ -153,12 +164,15 @@ jobs:
fail-fast: false
matrix:
cc: [gcc, clang]
- os: ['centos_7', 'centos_8', 'ubuntu_16.04']
+ os: ['centos_7', 'rocky_linux_8']
conf: ['--enable-abi-compat']
steps:
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}"
-e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${{matrix.os}}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Build_gcc-10:
runs-on: ubuntu-18.04
@@ -173,6 +187,9 @@ jobs:
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e CXX=g++-10
-e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Build_out-of-tree:
runs-on: ubuntu-18.04
@@ -180,6 +197,9 @@ jobs:
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}"
-e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/out_of_tree.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Build_linux-generic:
runs-on: ubuntu-18.04
@@ -189,6 +209,9 @@ jobs:
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ODP_LIB_NAME="libodp-linux"
-e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name config.log -exec cat {} \;
Run_coverage:
runs-on: ubuntu-18.04
@@ -241,7 +264,7 @@ jobs:
fail-fast: false
matrix:
cc: [gcc, clang]
- os: ['ubuntu_20.04']
+ os: ['ubuntu_20.04', 'ubuntu_22.04-openssl']
steps:
- uses: actions/checkout@v2
- run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e ARCH="${ARCH}"
@@ -303,6 +326,18 @@ jobs:
if: ${{ failure() }}
run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done
+ Run_dpdk-21_11:
+ runs-on: ubuntu-18.04
+ env:
+ OS: ubuntu_20.04
+ steps:
+ - uses: actions/checkout@v2
+ - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}"
+ -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-dpdk_21.11 /odp/scripts/ci/check.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done
+
Run_crypto:
runs-on: ubuntu-18.04
env:
diff --git a/CHANGELOG b/CHANGELOG
index e5624a903..e182f0ad5 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,36 @@
+== OpenDataPlane (1.36.0.0)
+
+=== Backward incompatible API changes
+
+==== Classifier
+* Add an action parameter `odp_cos_action_t` to CoS parameters
+(`odp_cls_cos_param_t`). Action may be to enqueue or drop the packet classified
+to the CoS. The old methods of creating a drop CoS have been replaced by the
+new drop action.
+
+==== Crypto
+* Deprecate `odp_crypto_operation()`, the associated data structures, and the
+completion event. Use `odp_crypto_op()` or `odp_crypto_op_enq()` instead.
+
+==== Traffic Manager
+* Split `odp_tm_capabilities_t.tm_queue_threshold` capability into byte and
+packet modes.
+* Split `odp_tm_level_capabilities_t.tm_node_threshold` capability into byte and
+packet modes.
+
+=== Backward compatible API changes
+==== Classifier
+* Add CoS specific statistics counters (`odp_cls_cos_stats_t`) and
+matching capabilities (`odp_cls_stats_capability_t`). Statistics counters can be
+read with `odp_cls_cos_stats()`.
+
+==== Common
+* Convert `unsigned int` types to `uint32_t`.
+
+==== Traffic Manager
+* Remove unused TM shaper color enum `odp_tm_shaper_color_t` and
+`ODP_NUM_SHAPER_COLORS` define.
+
== OpenDataPlane (1.35.0.0)
=== Backward incompatible API changes
diff --git a/DEPENDENCIES b/DEPENDENCIES
index f319eb36e..85a4cf242 100644
--- a/DEPENDENCIES
+++ b/DEPENDENCIES
@@ -1,9 +1,8 @@
Prerequisites for building the OpenDataPlane (ODP) API
1. Linux
- CentOS 7 (kernel v3.10) and Ubuntu 16.04 (kernel v4.4) are the oldest
- Linux distributions tested by the ODP CI. Earlier versions may or may not
- work.
+ CentOS 7 (kernel v3.10) is the oldest Linux distributions tested by the ODP
+ CI. Earlier versions may or may not work.
For CentOS/RedHat distros, configure the system to use Fedora EPEL repos and
third-party packages:
@@ -206,8 +205,8 @@ Prerequisites for building the OpenDataPlane (ODP) API
3.5 DPDK packet I/O support (optional)
- Use DPDK for ODP packet I/O. Currently supported DPDK versions are v19.11
- (recommended) and v20.11.
+ Use DPDK for ODP packet I/O. Currently supported DPDK versions are v19.11,
+ v20.11 (recommended), v21.11.
Note: only packet I/O is accelerated with DPDK. See
https://github.com/OpenDataPlane/odp-dpdk.git
@@ -244,8 +243,8 @@ Prerequisites for building the OpenDataPlane (ODP) API
# Configure ODP
$ ./configure --with-dpdk-path=<dpdk-dir>
-3.5.4 Build DPDK v20.11 from source
- $ git clone https://dpdk.org/git/dpdk-stable --branch 20.11 --depth 1 ./<dpdk-dir>
+3.5.4 Build DPDK v20.11 and onwards from source
+ $ git clone https://dpdk.org/git/dpdk-stable --branch <version, e.g. 21.11> --depth 1 ./<dpdk-dir>
# Prepare the build directory
$ cd <dpdk-dir>
diff --git a/configure.ac b/configure.ac
index 1a3904d53..9a4cce34a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3,7 +3,7 @@ AC_PREREQ([2.5])
# ODP API version
##########################################################################
m4_define([odpapi_generation_version], [1])
-m4_define([odpapi_major_version], [35])
+m4_define([odpapi_major_version], [36])
m4_define([odpapi_minor_version], [0])
m4_define([odpapi_point_version], [0])
m4_define([odpapi_version],
@@ -90,57 +90,6 @@ AM_PROG_LIBTOOL
PKG_PROG_PKG_CONFIG
-# Checks for library functions.
-dnl breaks cross-compilation and malloc(0) behaviour is not that important
-dnl AC_FUNC_MALLOC
-AC_FUNC_MMAP
-AC_CHECK_FUNCS(m4_normalize([
- bzero
- clock_gettime
- gethostbyname
- getpagesize
- gettimeofday
- malloc
- memset
- munmap
- socket
- strchr
- strerror
- strrchr
- strstr
- strtoull
-]))
-
-# Checks for header files.
-AC_HEADER_RESOLV
-AC_CHECK_HEADERS(m4_normalize([
- arpa/inet.h
- fcntl.h
- inttypes.h
- limits.h
- netdb.h
- netinet/in.h
- stddef.h
- stdint.h
- stdlib.h
- string.h
- sys/ioctl.h
- sys/socket.h
- sys/time.h
- unistd.h
-]))
-
-# Checks for typedefs, structures, and compiler characteristics.
-AC_HEADER_STDBOOL
-AC_C_INLINE
-AC_TYPE_SIZE_T
-AC_TYPE_SSIZE_T
-AC_TYPE_UINT8_T
-AC_TYPE_UINT16_T
-AC_TYPE_INT32_T
-AC_TYPE_UINT32_T
-AC_TYPE_UINT64_T
-
##########################################################################
# Default warning setup
##########################################################################
diff --git a/doc/users-guide/users-guide-crypto.adoc b/doc/users-guide/users-guide-crypto.adoc
index 6393a5e78..0f33d6548 100644
--- a/doc/users-guide/users-guide-crypto.adoc
+++ b/doc/users-guide/users-guide-crypto.adoc
@@ -13,11 +13,6 @@ ODP provides APIs for following cryptographic services:
* Random number generation
* Crypto capability inquiries
-Ciphering and authentication services are accessible via two complementary
-sets of related APIs. The original ODP crypto APIs, and a newer
-_packet-oriented_ set of crypto APIs that are designed to be consistent with
-the protocol-aware cryptographic services offered by the IPsec API set.
-
=== Crypto Sessions
To apply a cryptographic operation to a packet a session must be created. All
@@ -49,112 +44,35 @@ with an `odp_crypto_session_t`.
=== Crypto operations
After session creation, a cryptographic operation can be applied to a packet
-in one of two ways.
-
-==== Parameter-based Crypto Operations
-This is the original ODP support for cryptographic operations. The
-`odp_crypto_operation()` API takes an input `odp_crypto_op_param_t` struct
-that describes the cryptographic operation to be performed. This struct
-contains the session to use as well as the input packet the operation is to be
-performed on. The caller may either specify an output packet to receive the
-operation results or may request that the ODP implementation allocate a new
-packet to receive these results from the output pool associated with the
-`odp_crypto_session_t`. If the input packet is also used as the output packet,
-then an "in place" operation is requested.
-
-When using the `odp_crypto_operation()` API. Applications may indicate a
-preference for synchronous or asynchronous processing in the session's
-`pref_mode` parameter. However crypto operations may complete synchronously
-even if an asynchronous preference is indicated, and applications must examine
-the `posted` output parameter from `odp_crypto_operation()` to determine
-whether the operation has completed or if an `ODP_EVENT_CRYPTO_COMPL`
-notification is expected. In the case of an async operation, the `posted`
-output parameter will be set to true.
-
-The operation arguments specify for each packet the areas that are to be
-encrypted or decrypted and authenticated. The parameters include also
-pointers to cipher and authentication initialization vectors as needed,
-depending on the initialization vector lengths specified at session creation.
-
-An operation can be executed in in-place, out-of-place or new buffer mode.
-In in-place mode output packet is same as the input packet.
-In case of out-of-place mode output packet is different from input packet as
-specified by the application, while in new buffer mode implementation allocates
-a new output buffer from the session’s output pool.
-
-The application can also specify a context associated with a given operation
-that will be retained during async operation and can be retrieved via the
-completion event.
-
-Results of an asynchronous session will be posted as completion events to the
-session’s completion queue, which can be accessed directly or via the ODP
-scheduler. The completion event contains the status of the operation and the
-result. The application has the responsibility to free the completion event.
-
-Upon receipt of an `ODP_EVENT_CRYPTO_COMPL` event, the
-`odp_crypto_compl_result()` API is used to retrieve the
-`odp_crypto_op_result_t` associated with the event. This result struct in turn
-contains:
-
-* An indication of the success or failure of the crypto operation
-* The user context associated with the event
-* The output `odp_packet_t`.
-* The `odp_crypto_op_status_t` for the requested cipher operation
-* The `odp_crypto_op_status_t` for the requested authentication operation
-
-==== Packet-based Crypto Operations
-To simplify the original cryptographic operation request API, as well as to
-be more flexible and consistent with the protocol-aware APIs introduced for
-IPsec support, a newer packet-oriented set of cryptographic operation
-APIs is also provided. Applications may use either API set, but going forward
-it is expected that these newer APIs will be the focus of continued
-development.
-
-Instead of a single `odp_crypto_operation()` API, the packet-based form
-provides two APIs: `odp_crypto_op()` is the synchronous form while
-`odp_crypto_op_enq()` is the asynchronous form. To check which of these are
-supported by the ODP implementation, examine the `sync_mode` and `async_mode`
-fields in the `odp_crypto_capability_t` struct returned by the
+synchronously or asynchronously. `odp_crypto_op()` is the synchronous API
+while `odp_crypto_op_enq()` is the asynchronous API. To check which of these
+are supported by the ODP implementation, examine the `sync_mode` and
+`async_mode` fields in the `odp_crypto_capability_t` struct returned by the
`odp_crypto_capability()` API.
Both forms take an input array of packets, an optional output array of packets
to receive the results, and an array of `odp_crypto_packet_op_param_t` structs
-that describe the operation to be performed on each input packet. As with the
-original APIs, the output array may be the same packets to request in-place
-operation, or may be specified as `ODP_PACKET_INVALID` to request that ODP
-allocate output packets from the pool associated with the
-`odp_crypto_session_t` being used.
-
-The key differences between the `odp_crypto_op_param_t` used by the original
-APIs and the `odp_crypto_packet_op_param_t` used by the new APIs are:
-
-* The original API takes a single `odp_crypto_op_param_t` since it operates on
-a single packet whereas the new forms take an array of
-`odp_crypto_packet_op_param_t` structs, one for each input packet.
-
-* The `odp_crypto_packet_op_param_t` does not contain any packet information
-since the input and output packets are supplied as API parameters rather than
-being encoded in this struct.
-
-* The `odp_crypto_packet_op_param_t` does not contain a user context field.
-
-In addition, the `odp_crypto_session_t` field `op_mode` is used instead of
-the `pref_mode` field when the packet-oriented APIs are used. If the
-`op_mode` is set to `ODP_CRYPTO_SYNC` then the synchronous form of the API
-must be used and if `op_mode` is set to `ODP_CRYPTO_ASYNC` then the
-asynchronous form of the API must be used. It is an error to attempt to use
-a form of the API not properly matched to the mode of the crypto session.
-
-The output of a packet-based crypto operation is an `odp_packet_t` (one for
-each input packet) that is returned either synchronously or
-asynchronously. Asynchronous return is in the form of `ODP_EVENT_PACKET`
-events that have event subtype `ODP_EVENT_PACKET_CRYPTO`. The packet
-associated with such events is obtained via the
-`odp_crypto_packet_from_event()` API. The `odp_crypto_result()` API, in turn,
-retrieves the `odp_crypto_packet_result_t` from this `odp_packet_t` that
-contains:
-
-* An indication of whether the crypto packet operation was successful or not
+that describe the operation to be performed on each input packet. The output
+array may be the same packets to request in-place operation, or may be
+specified as `ODP_PACKET_INVALID` to request that ODP allocate output packets
+from the pool associated with the `odp_crypto_session_t` being used.
+
+The op_mode field of `odp_crypto_session_t` indicates whether asynchronous
+or synchronous operations are used with the session. If `op_mode` is set
+to `ODP_CRYPTO_SYNC` then the synchronous API must be used and if `op_mode`
+is set to `ODP_CRYPTO_ASYNC` then the asynchronous API must be used. It is
+an error to use a form of the API that does not match the mode of the crypto
+session.
+
+The output of a crypto operation is an `odp_packet_t` (one for each input
+packet) that is returned either synchronously or asynchronously. Asynchronous
+return is in the form of `ODP_EVENT_PACKET` events that have event subtype
+`ODP_EVENT_PACKET_CRYPTO`. The packet associated with such events is obtained
+via the `odp_crypto_packet_from_event()` API. The `odp_crypto_result()` API,
+in turn, retrieves the `odp_crypto_packet_result_t` from this `odp_packet_t`
+that contains:
+
+* An indication of whether the crypto operation was successful or not
* The `odp_crypto_op_status_t` for the requested cipher operation
* The `odp_crypto_op_status_t` for the requested authentication operation
diff --git a/doc/users-guide/users-guide-pktio.adoc b/doc/users-guide/users-guide-pktio.adoc
index 43df729fa..1698060e9 100644
--- a/doc/users-guide/users-guide-pktio.adoc
+++ b/doc/users-guide/users-guide-pktio.adoc
@@ -468,7 +468,7 @@ specified independently at *open* time.
.PktIO Input Modes
* `ODP_PKTIN_MODE_DIRECT`
* `ODP_PKTIN_MODE_QUEUE`
-* `ODP_OKTIN_MODE_SCHED`
+* `ODP_PKTIN_MODE_SCHED`
* `ODP_PKTIN_MODE_DISABLED`
.PktIO Output Modes
diff --git a/example/generator/odp_generator.c b/example/generator/odp_generator.c
index 7a070235b..b980ee180 100644
--- a/example/generator/odp_generator.c
+++ b/example/generator/odp_generator.c
@@ -55,9 +55,9 @@ typedef struct {
odp_pktio_t pktio;
odp_pktio_config_t config;
odp_pktout_queue_t pktout[MAX_WORKERS];
- unsigned pktout_count;
+ uint32_t pktout_count;
odp_pktin_queue_t pktin[MAX_WORKERS];
- unsigned pktin_count;
+ uint32_t pktin_count;
} interface_t;
/**
@@ -537,8 +537,8 @@ static int setup_icmp_pkt(odp_packet_t pkt,
* @warning This routine aborts if the create is unsuccessful.
*/
static int create_pktio(const char *dev, odp_pool_t pool,
- unsigned num_rx_queues,
- unsigned num_tx_queues,
+ uint32_t num_rx_queues,
+ uint32_t num_tx_queues,
interface_t *itf)
{
odp_pktio_capability_t capa;
@@ -1104,7 +1104,7 @@ int main(int argc, char *argv[])
odph_thread_t thread_tbl[MAX_WORKERS];
odp_pool_t pool;
int num_workers;
- unsigned num_rx_queues, num_tx_queues;
+ uint32_t num_rx_queues, num_tx_queues;
int i;
odp_shm_t shm;
odp_cpumask_t cpumask;
diff --git a/example/ipfragreass/odp_ipfragreass_reassemble.c b/example/ipfragreass/odp_ipfragreass_reassemble.c
index 57e0f375a..2542a3b97 100644
--- a/example/ipfragreass/odp_ipfragreass_reassemble.c
+++ b/example/ipfragreass/odp_ipfragreass_reassemble.c
@@ -183,11 +183,9 @@ static struct packet *extract_complete_packet(struct packet *tail,
struct packet *current = tail;
odph_ipv4hdr_t tail_hdr;
uint16_t final_frag_offset;
- uint16_t expected_frag_offset;
tail_hdr = *(odph_ipv4hdr_t *)odp_packet_data(tail->handle);
final_frag_offset = ipv4hdr_fragment_offset_oct(tail_hdr);
- expected_frag_offset = final_frag_offset;
while (current) {
odph_ipv4hdr_t curr_hdr;
uint16_t curr_offset_oct;
@@ -258,7 +256,6 @@ static struct packet *extract_complete_packet(struct packet *tail,
break;
}
- expected_frag_offset -= prev_oct;
current = prev;
}
diff --git a/example/ipsec_api/odp_ipsec_api_run_esp_tun_out.sh b/example/ipsec_api/odp_ipsec_api_run_esp_tun_out.sh
index a4c62e226..2bda78665 100755
--- a/example/ipsec_api/odp_ipsec_api_run_esp_tun_out.sh
+++ b/example/ipsec_api/odp_ipsec_api_run_esp_tun_out.sh
@@ -22,6 +22,7 @@ setup_interfaces
-r 192.168.222.2/32,$ROUTE_IF_OUTB,08:00:27:F5:8B:DB \
-p 192.168.111.0/24,192.168.222.0/24,out,esp \
-e 192.168.111.2,192.168.222.2,3des,201,656c8523255ccc23a66c1917aa0cf30991fce83532a4b224 \
+ -t 192.168.111.2,192.168.222.2,10.0.111.2,10.0.222.2 \
-s 192.168.111.2,192.168.222.2,$IN_IF,$OUT_IF,10,100 \
-c 2 "$@"
diff --git a/example/ipsec_crypto/odp_ipsec_stream.c b/example/ipsec_crypto/odp_ipsec_stream.c
index a24d4b4f9..0a0710a13 100644
--- a/example/ipsec_crypto/odp_ipsec_stream.c
+++ b/example/ipsec_crypto/odp_ipsec_stream.c
@@ -15,6 +15,7 @@
#include <openssl/des.h>
#include <openssl/rand.h>
#include <openssl/hmac.h>
+#include <openssl/opensslv.h>
#include <odp_api.h>
@@ -26,6 +27,11 @@
#define LOOP_DEQ_COUNT 32 /**< packets to dequeue at once */
+/* Ignore warnings about APIs deprecated in OpenSSL 3.0 */
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#endif
+
/**
* Stream packet header
*/
diff --git a/helper/include/odp/helper/cli.h b/helper/include/odp/helper/cli.h
index b2f88d7bc..e9351147f 100644
--- a/helper/include/odp/helper/cli.h
+++ b/helper/include/odp/helper/cli.h
@@ -21,8 +21,6 @@
extern "C" {
#endif
-#include <odp_api.h>
-#include <odp/helper/ip.h>
#include <stdint.h>
#include <stdarg.h>
diff --git a/helper/include/odp/helper/linux/process.h b/helper/include/odp/helper/linux/process.h
index 679c6c907..620db7405 100644
--- a/helper/include/odp/helper/linux/process.h
+++ b/helper/include/odp/helper/linux/process.h
@@ -18,6 +18,7 @@
#define ODPH_LINUX_PROCESS_H_
#include <odp/helper/threads.h>
+#include <odp_api.h>
#ifdef __cplusplus
extern "C" {
diff --git a/helper/include/odp/helper/linux/pthread.h b/helper/include/odp/helper/linux/pthread.h
index 33109ec65..298536893 100644
--- a/helper/include/odp/helper/linux/pthread.h
+++ b/helper/include/odp/helper/linux/pthread.h
@@ -18,6 +18,7 @@
#define ODPH_LINUX_PTHREAD_H_
#include <odp/helper/threads.h>
+#include <odp_api.h>
#ifdef __cplusplus
extern "C" {
diff --git a/helper/include/odp/helper/threads.h b/helper/include/odp/helper/threads.h
index cfde92830..53ce8cd07 100644
--- a/helper/include/odp/helper/threads.h
+++ b/helper/include/odp/helper/threads.h
@@ -24,6 +24,7 @@ extern "C" {
#endif
#include <odp/helper/deprecated.h>
+#include <odp_api.h>
#include <pthread.h>
#include <getopt.h>
diff --git a/include/Makefile.am b/include/Makefile.am
index 89dfddebc..510154753 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -26,6 +26,7 @@ odpapiinclude_HEADERS = \
odp/api/dma_types.h \
odp/api/errno.h \
odp/api/event.h \
+ odp/api/event_types.h \
odp/api/hash.h \
odp/api/hints.h \
odp/api/init.h \
@@ -86,6 +87,7 @@ odpapispecinclude_HEADERS = \
odp/api/spec/dma_types.h \
odp/api/spec/errno.h \
odp/api/spec/event.h \
+ odp/api/spec/event_types.h \
odp/api/spec/hash.h \
odp/api/spec/hints.h \
odp/api/spec/init.h \
@@ -148,6 +150,7 @@ odpapiabidefaultinclude_HEADERS = \
odp/api/abi-default/dma_types.h \
odp/api/abi-default/errno.h \
odp/api/abi-default/event.h \
+ odp/api/abi-default/event_types.h \
odp/api/abi-default/hash.h \
odp/api/abi-default/init.h \
odp/api/abi-default/ipsec.h \
@@ -178,6 +181,7 @@ odpapiabidefaultinclude_HEADERS = \
odp/api/abi-default/thrmask.h \
odp/api/abi-default/ticketlock.h \
odp/api/abi-default/time.h \
+ odp/api/abi-default/timer.h \
odp/api/abi-default/timer_types.h \
odp/api/abi-default/traffic_mngr.h \
odp/api/abi-default/version.h
@@ -202,6 +206,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/arm32-linux/odp/api/abi/dma_types.h \
odp/arch/arm32-linux/odp/api/abi/errno.h \
odp/arch/arm32-linux/odp/api/abi/event.h \
+ odp/arch/arm32-linux/odp/api/abi/event_types.h \
odp/arch/arm32-linux/odp/api/abi/hash.h \
odp/arch/arm32-linux/odp/api/abi/init.h \
odp/arch/arm32-linux/odp/api/abi/ipsec.h \
@@ -232,6 +237,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/arm32-linux/odp/api/abi/thrmask.h \
odp/arch/arm32-linux/odp/api/abi/ticketlock.h \
odp/arch/arm32-linux/odp/api/abi/time.h \
+ odp/arch/arm32-linux/odp/api/abi/timer.h \
odp/arch/arm32-linux/odp/api/abi/timer_types.h \
odp/arch/arm32-linux/odp/api/abi/traffic_mngr.h \
odp/arch/arm32-linux/odp/api/abi/version.h
@@ -252,6 +258,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/arm64-linux/odp/api/abi/dma_types.h \
odp/arch/arm64-linux/odp/api/abi/errno.h \
odp/arch/arm64-linux/odp/api/abi/event.h \
+ odp/arch/arm64-linux/odp/api/abi/event_types.h \
odp/arch/arm64-linux/odp/api/abi/hash.h \
odp/arch/arm64-linux/odp/api/abi/init.h \
odp/arch/arm64-linux/odp/api/abi/ipsec.h \
@@ -282,6 +289,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/arm64-linux/odp/api/abi/thrmask.h \
odp/arch/arm64-linux/odp/api/abi/ticketlock.h \
odp/arch/arm64-linux/odp/api/abi/time.h \
+ odp/arch/arm64-linux/odp/api/abi/timer.h \
odp/arch/arm64-linux/odp/api/abi/timer_types.h \
odp/arch/arm64-linux/odp/api/abi/traffic_mngr.h \
odp/arch/arm64-linux/odp/api/abi/version.h
@@ -302,6 +310,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/default-linux/odp/api/abi/dma_types.h \
odp/arch/default-linux/odp/api/abi/errno.h \
odp/arch/default-linux/odp/api/abi/event.h \
+ odp/arch/default-linux/odp/api/abi/event_types.h \
odp/arch/default-linux/odp/api/abi/hash.h \
odp/arch/default-linux/odp/api/abi/init.h \
odp/arch/default-linux/odp/api/abi/ipsec.h \
@@ -332,6 +341,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/default-linux/odp/api/abi/thrmask.h \
odp/arch/default-linux/odp/api/abi/ticketlock.h \
odp/arch/default-linux/odp/api/abi/time.h \
+ odp/arch/default-linux/odp/api/abi/timer.h \
odp/arch/default-linux/odp/api/abi/timer_types.h \
odp/arch/default-linux/odp/api/abi/traffic_mngr.h \
odp/arch/default-linux/odp/api/abi/version.h
@@ -352,6 +362,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/power64-linux/odp/api/abi/dma_types.h \
odp/arch/power64-linux/odp/api/abi/errno.h \
odp/arch/power64-linux/odp/api/abi/event.h \
+ odp/arch/power64-linux/odp/api/abi/event_types.h \
odp/arch/power64-linux/odp/api/abi/hash.h \
odp/arch/power64-linux/odp/api/abi/init.h \
odp/arch/power64-linux/odp/api/abi/ipsec.h \
@@ -382,6 +393,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/power64-linux/odp/api/abi/thrmask.h \
odp/arch/power64-linux/odp/api/abi/ticketlock.h \
odp/arch/power64-linux/odp/api/abi/time.h \
+ odp/arch/power64-linux/odp/api/abi/timer.h \
odp/arch/power64-linux/odp/api/abi/timer_types.h \
odp/arch/power64-linux/odp/api/abi/traffic_mngr.h \
odp/arch/power64-linux/odp/api/abi/version.h
@@ -402,6 +414,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/x86_32-linux/odp/api/abi/dma_types.h \
odp/arch/x86_32-linux/odp/api/abi/errno.h \
odp/arch/x86_32-linux/odp/api/abi/event.h \
+ odp/arch/x86_32-linux/odp/api/abi/event_types.h \
odp/arch/x86_32-linux/odp/api/abi/hash.h \
odp/arch/x86_32-linux/odp/api/abi/init.h \
odp/arch/x86_32-linux/odp/api/abi/ipsec.h \
@@ -432,6 +445,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/x86_32-linux/odp/api/abi/thrmask.h \
odp/arch/x86_32-linux/odp/api/abi/ticketlock.h \
odp/arch/x86_32-linux/odp/api/abi/time.h \
+ odp/arch/x86_32-linux/odp/api/abi/timer.h \
odp/arch/x86_32-linux/odp/api/abi/timer_types.h \
odp/arch/x86_32-linux/odp/api/abi/traffic_mngr.h \
odp/arch/x86_32-linux/odp/api/abi/version.h
@@ -452,6 +466,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/x86_64-linux/odp/api/abi/dma_types.h \
odp/arch/x86_64-linux/odp/api/abi/errno.h \
odp/arch/x86_64-linux/odp/api/abi/event.h \
+ odp/arch/x86_64-linux/odp/api/abi/event_types.h \
odp/arch/x86_64-linux/odp/api/abi/hash.h \
odp/arch/x86_64-linux/odp/api/abi/init.h \
odp/arch/x86_64-linux/odp/api/abi/ipsec.h \
@@ -482,6 +497,7 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/x86_64-linux/odp/api/abi/thrmask.h \
odp/arch/x86_64-linux/odp/api/abi/ticketlock.h \
odp/arch/x86_64-linux/odp/api/abi/time.h \
+ odp/arch/x86_64-linux/odp/api/abi/timer.h \
odp/arch/x86_64-linux/odp/api/abi/timer_types.h \
odp/arch/x86_64-linux/odp/api/abi/traffic_mngr.h \
odp/arch/x86_64-linux/odp/api/abi/version.h
diff --git a/include/odp/api/abi-default/event.h b/include/odp/api/abi-default/event.h
index ecedda3bd..a1563ac9b 100644
--- a/include/odp/api/abi-default/event.h
+++ b/include/odp/api/abi-default/event.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2018, Linaro Limited
+/* Copyright (c) 2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -11,41 +11,7 @@
extern "C" {
#endif
-#include <stdint.h>
-
-/** @internal Dummy type for strong typing */
-typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_event_t;
-
-/** @ingroup odp_event
- * @{
- */
-
-typedef _odp_abi_event_t *odp_event_t;
-
-#define ODP_EVENT_INVALID ((odp_event_t)0)
-
-typedef enum {
- ODP_EVENT_BUFFER = 1,
- ODP_EVENT_PACKET = 2,
- ODP_EVENT_TIMEOUT = 3,
- ODP_EVENT_CRYPTO_COMPL = 4,
- ODP_EVENT_IPSEC_STATUS = 5,
- ODP_EVENT_PACKET_VECTOR = 6,
- ODP_EVENT_PACKET_TX_COMPL = 7,
- ODP_EVENT_DMA_COMPL = 8,
-} odp_event_type_t;
-
-typedef enum {
- ODP_EVENT_NO_SUBTYPE = 0,
- ODP_EVENT_PACKET_BASIC = 1,
- ODP_EVENT_PACKET_CRYPTO = 2,
- ODP_EVENT_PACKET_IPSEC = 3,
- ODP_EVENT_PACKET_COMP = 4
-} odp_event_subtype_t;
-
-/**
- * @}
- */
+/* Empty header required due to the inline functions */
#ifdef __cplusplus
}
diff --git a/include/odp/api/abi-default/event_types.h b/include/odp/api/abi-default/event_types.h
new file mode 100644
index 000000000..fc685cfd9
--- /dev/null
+++ b/include/odp/api/abi-default/event_types.h
@@ -0,0 +1,58 @@
+/* Copyright (c) 2017-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_ABI_EVENT_TYPES_H_
+#define ODP_ABI_EVENT_TYPES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+#include <odp/api/deprecated.h>
+
+/** @internal Dummy type for strong typing */
+typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_event_t;
+
+/** @ingroup odp_event
+ * @{
+ */
+
+typedef _odp_abi_event_t *odp_event_t;
+
+#define ODP_EVENT_INVALID ((odp_event_t)0)
+
+typedef enum {
+ ODP_EVENT_BUFFER = 1,
+ ODP_EVENT_PACKET = 2,
+ ODP_EVENT_TIMEOUT = 3,
+#if ODP_DEPRECATED_API
+ ODP_EVENT_CRYPTO_COMPL = 4,
+#endif
+ ODP_EVENT_IPSEC_STATUS = 5,
+ ODP_EVENT_PACKET_VECTOR = 6,
+ ODP_EVENT_PACKET_TX_COMPL = 7,
+ ODP_EVENT_DMA_COMPL = 8,
+} odp_event_type_t;
+
+typedef enum {
+ ODP_EVENT_NO_SUBTYPE = 0,
+ ODP_EVENT_PACKET_BASIC = 1,
+ ODP_EVENT_PACKET_CRYPTO = 2,
+ ODP_EVENT_PACKET_IPSEC = 3,
+ ODP_EVENT_PACKET_COMP = 4
+} odp_event_subtype_t;
+
+/**
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/include/odp/api/abi-default/timer.h b/include/odp/api/abi-default/timer.h
new file mode 100644
index 000000000..faf690ae3
--- /dev/null
+++ b/include/odp/api/abi-default/timer.h
@@ -0,0 +1,20 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_ABI_TIMER_H_
+#define ODP_ABI_TIMER_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Empty header required due to the timer inline functions */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/include/odp/api/abi-default/traffic_mngr.h b/include/odp/api/abi-default/traffic_mngr.h
index 06dc6c3c7..78985313d 100644
--- a/include/odp/api/abi-default/traffic_mngr.h
+++ b/include/odp/api/abi-default/traffic_mngr.h
@@ -76,11 +76,6 @@ extern "C" {
*/
#define ODP_TM_MAX_TM_NODE_FANIN (4 * 1024)
-/** The ODP_NUM_SHAPER_COLORS constant just counts the number of enumeration
- * values defined in the odp_tm_shaper_color_t type.
- */
-#define ODP_NUM_SHAPER_COLORS 3
-
/** The INVALID_PRIORITY constant is used when one needs to indicate an
* invalid priority value.
*/
diff --git a/include/odp/api/buffer.h b/include/odp/api/buffer.h
index 27622c2a3..a56cd0660 100644
--- a/include/odp/api/buffer.h
+++ b/include/odp/api/buffer.h
@@ -17,8 +17,6 @@
extern "C" {
#endif
-#include <odp/api/std_types.h>
-#include <odp/api/abi/event.h>
#include <odp/api/abi/buffer.h>
#include <odp/api/spec/buffer.h>
diff --git a/include/odp/api/comp.h b/include/odp/api/comp.h
index c3294e79e..300d12a61 100644
--- a/include/odp/api/comp.h
+++ b/include/odp/api/comp.h
@@ -18,8 +18,6 @@ extern "C" {
#endif
#include <odp/api/abi/comp.h>
-#include <odp/api/abi/event.h>
-#include <odp/api/abi/queue_types.h>
#include <odp/api/spec/comp.h>
diff --git a/include/odp/api/dma_types.h b/include/odp/api/dma_types.h
index 5a726a319..eeaa7bee8 100644
--- a/include/odp/api/dma_types.h
+++ b/include/odp/api/dma_types.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2021, Nokia
+/* Copyright (c) 2021-2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -17,7 +17,6 @@
extern "C" {
#endif
-#include <odp/api/abi/event.h>
#include <odp/api/abi/dma_types.h>
#include <odp/api/spec/dma_types.h>
diff --git a/include/odp/api/event_types.h b/include/odp/api/event_types.h
new file mode 100644
index 000000000..a4d20a341
--- /dev/null
+++ b/include/odp/api/event_types.h
@@ -0,0 +1,28 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODP event API type definitions
+ */
+
+#ifndef ODP_API_EVENT_TYPES_H_
+#define ODP_API_EVENT_TYPES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp/api/abi/event_types.h>
+
+#include <odp/api/spec/event_types.h>
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/include/odp/api/packet.h b/include/odp/api/packet.h
index eb75d18fa..1f04c026d 100644
--- a/include/odp/api/packet.h
+++ b/include/odp/api/packet.h
@@ -1,4 +1,5 @@
/* Copyright (c) 2013-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -17,12 +18,7 @@
extern "C" {
#endif
-#include <odp/api/std_types.h>
-#include <odp/api/abi/event.h>
-#include <odp/api/abi/packet_io_types.h>
-#include <odp/api/abi/packet_types.h>
#include <odp/api/abi/packet.h>
-#include <odp/api/abi/buffer.h>
#include <odp/api/spec/packet.h>
diff --git a/include/odp/api/queue.h b/include/odp/api/queue.h
index 7897fff86..666d5eaf9 100644
--- a/include/odp/api/queue.h
+++ b/include/odp/api/queue.h
@@ -1,4 +1,5 @@
/* Copyright (c) 2013-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -17,11 +18,7 @@
extern "C" {
#endif
-#include <odp/api/std_types.h>
-#include <odp/api/abi/event.h>
-#include <odp/api/abi/queue_types.h>
#include <odp/api/abi/queue.h>
-#include <odp/api/abi/buffer.h>
#include <odp/api/spec/queue.h>
diff --git a/include/odp/api/spec/buffer.h b/include/odp/api/spec/buffer.h
index 7945b59f9..c11a3c7b4 100644
--- a/include/odp/api/spec/buffer.h
+++ b/include/odp/api/spec/buffer.h
@@ -18,7 +18,9 @@
extern "C" {
#endif
+#include <odp/api/event_types.h>
#include <odp/api/pool_types.h>
+#include <odp/api/std_types.h>
/** @defgroup odp_buffer ODP BUFFER
* Buffer event metadata and operations.
diff --git a/include/odp/api/spec/classification.h b/include/odp/api/spec/classification.h
index b9682c7cb..1c76ea192 100644
--- a/include/odp/api/spec/classification.h
+++ b/include/odp/api/spec/classification.h
@@ -36,9 +36,7 @@ extern "C" {
/**
* @def ODP_COS_INVALID
- * This value is returned from odp_cls_cos_create() on failure,
- * May also be used as a sink class of service that
- * results in packets being discarded.
+ * This value is returned from odp_cls_cos_create() on failure.
*/
/**
@@ -179,6 +177,30 @@ typedef struct odp_bp_param_t {
} odp_bp_param_t;
/**
+ * Classifier CoS specific statistics counters
+ *
+ * Counters are incremented per packet classified to the CoS. In a CoS chain,
+ * counters are incremented in every CoS for which counters are enabled.
+ */
+typedef struct odp_cls_cos_stats_t {
+ /** Number of octets in classified packets. In case of Ethernet, packet
+ * size includes MAC header. */
+ uint64_t octets;
+
+ /** Number of classified packets, including packets dropped due to drop
+ * action. */
+ uint64_t packets;
+
+ /** Number of discarded packets due to other reasons than packet
+ * errors or drop action. */
+ uint64_t discards;
+
+ /** Number of packets with errors. */
+ uint64_t errors;
+
+} odp_cls_cos_stats_t;
+
+/**
* Classifier queue specific statistics counters
*
* Counters are incremented per packet destined to the queue per originating
@@ -206,6 +228,34 @@ typedef struct odp_cls_queue_stats_t {
* Classifier statistics capabilities
*/
typedef struct odp_cls_stats_capability_t {
+ /** CoS level capabilities */
+ struct {
+ /** Supported counters */
+ union {
+ /** Statistics counters in a bit field structure */
+ struct {
+ /** @see odp_cls_cos_stats_t::octets */
+ uint64_t octets : 1;
+
+ /** @see odp_cls_cos_stats_t::packets */
+ uint64_t packets : 1;
+
+ /** @see odp_cls_cos_stats_t::discards */
+ uint64_t discards : 1;
+
+ /** @see odp_cls_cos_stats_t::errors */
+ uint64_t errors : 1;
+
+ } counter;
+
+ /** All bits of the bit field structure
+ *
+ * This field can be used to set/clear all flags, or
+ * for bitwise operations over the entire structure. */
+ uint64_t all_counters;
+ };
+ } cos;
+
/** Queue level capabilities */
struct {
/** Supported counters */
@@ -247,17 +297,22 @@ typedef struct odp_cls_capability_t {
odp_cls_pmr_terms_t supported_terms;
/** Maximum number of PMR terms */
- unsigned int max_pmr_terms;
+ uint32_t max_pmr_terms;
/** Number of PMR terms available for use now */
- unsigned int available_pmr_terms;
+ uint32_t available_pmr_terms;
/** Maximum number of CoS supported */
- unsigned int max_cos;
+ uint32_t max_cos;
+
+ /** Maximum number of CoSes that can have statistics enabled at the same
+ * time. If this value is zero, then CoS level statistics are not
+ * supported. */
+ uint32_t max_cos_stats;
/** Maximun number of queues supported per CoS
* if the value is 1, then hashing is not supported*/
- unsigned int max_hash_queues;
+ uint32_t max_hash_queues;
/** Protocol header combination supported for Hashing */
odp_pktin_hash_proto_t hash_protocols;
@@ -315,10 +370,48 @@ typedef enum {
} odp_cos_hdr_flow_fields_t;
/**
+ * Enumeration of actions for CoS.
+ */
+typedef enum {
+ /**
+ * Enqueue packet
+ *
+ * Packets that arrive in the CoS are enqueued to a destination queue.
+ */
+ ODP_COS_ACTION_ENQUEUE,
+
+ /**
+ * Drop packet
+ *
+ * Packets that arrive in the CoS are dropped. Packets are freed into
+ * their originating pool.
+ */
+ ODP_COS_ACTION_DROP,
+} odp_cos_action_t;
+
+/**
* Class of service parameters
* Used to communicate class of service creation options
*/
typedef struct odp_cls_cos_param {
+ /** Action to take. When action is ODP_COS_ACTION_DROP, all the other
+ * parameters are ignored.
+ *
+ * The final match in the CoS chain defines the action for a packet.
+ * I.e. packet is dropped only when the CoS of the last matching rule
+ * has drop action. Actions in the previous CoSes in the chain are
+ * ignored.
+ *
+ * Default is ODP_COS_ACTION_ENQUEUE.
+ */
+ odp_cos_action_t action;
+
+ /** Enable statistics. If true, counters are incremented when packets
+ * are classified to the CoS. Default is false. @see
+ * odp_cls_cos_stats().
+ */
+ odp_bool_t stats_enable;
+
/** Number of queues to be linked to this CoS.
* If the number is greater than 1 then hashing is enabled.
* If number is equal to 1 then hashing is disabled.
@@ -391,6 +484,9 @@ int odp_cls_capability(odp_cls_capability_t *capability);
/**
* Create a class-of-service
*
+ * Depending on the action parameter, packets to the CoS are either enqueued to
+ * a destination queue, or dropped.
+ *
* The use of class-of-service name is optional. Unique names are not required.
* Use odp_cls_cos_param_init() to initialize parameters into their default
* values.
@@ -401,10 +497,6 @@ int odp_cls_capability(odp_cls_capability_t *capability);
*
* @retval Class-of-service handle
* @retval ODP_COS_INVALID on failure.
- *
- * @note ODP_QUEUE_INVALID and ODP_POOL_INVALID are valid values for queue
- * and pool associated with a class of service. When either of these values
- * is configured as INVALID packets assigned to the CoS get dropped.
*/
odp_cos_t odp_cls_cos_create(const char *name,
const odp_cls_cos_param_t *param);
@@ -542,6 +634,25 @@ int odp_cos_with_l3_qos(odp_pktio_t pktio_in,
odp_bool_t l3_preference);
/**
+ * Get statistics for a CoS
+ *
+ * The statistics counters are incremented for packets classified to the
+ * given CoS.
+ *
+ * Counters that are not supported are set to zero.
+ *
+ * It's implementation defined if odp_pktio_stats_reset() call affects these
+ * counters.
+ *
+ * @param cos CoS handle
+ * @param[out] stats Statistics structure for output
+ *
+ * @retval 0 on success
+ * @retval <0 on failure
+ */
+int odp_cls_cos_stats(odp_cos_t cos, odp_cls_cos_stats_t *stats);
+
+/**
* Get statistics for a queue assigned to a CoS
*
* The statistics counters are incremented only for packets originating from the
diff --git a/include/odp/api/spec/comp.h b/include/odp/api/spec/comp.h
index 7f64cca5d..39ecbfde4 100644
--- a/include/odp/api/spec/comp.h
+++ b/include/odp/api/spec/comp.h
@@ -14,8 +14,11 @@
#define ODP_API_SPEC_COMP_H_
#include <odp/visibility_begin.h>
+
+#include <odp/api/event_types.h>
+#include <odp/api/packet_types.h>
+#include <odp/api/queue_types.h>
#include <odp/api/std_types.h>
-#include <odp/api/packet.h>
#ifdef __cplusplus
extern "C" {
diff --git a/include/odp/api/spec/crypto.h b/include/odp/api/spec/crypto.h
index c0ff3710d..7cd69f4d2 100644
--- a/include/odp/api/spec/crypto.h
+++ b/include/odp/api/spec/crypto.h
@@ -16,6 +16,7 @@
#include <odp/visibility_begin.h>
#include <odp/api/deprecated.h>
+#include <odp/api/packet_types.h>
#include <odp/api/pool_types.h>
#include <odp/api/std_types.h>
@@ -23,8 +24,6 @@
extern "C" {
#endif
-#include <odp/api/packet.h>
-
/** @defgroup odp_crypto ODP CRYPTO
* Data ciphering and authentication.
* @{
@@ -42,7 +41,7 @@ extern "C" {
/**
* @typedef odp_crypto_compl_t
-* Crypto API completion event (platform dependent).
+* @deprecated Crypto API completion event (platform dependent).
*/
/**
@@ -559,8 +558,10 @@ typedef struct odp_crypto_session_param_t {
/** Preferred sync vs. async for odp_crypto_operation()
*
* The default value is ODP_CRYPTO_SYNC.
+ *
+ * @deprecated Used only with deprecated odp_crypto_operation()
*/
- odp_crypto_op_mode_t pref_mode;
+ odp_crypto_op_mode_t ODP_DEPRECATE(pref_mode);
/** Operation mode when using packet interface: sync or async
*
@@ -672,8 +673,8 @@ typedef struct odp_crypto_session_param_t {
/** Async mode completion event queue
*
* The completion queue is used to return completions from
- * odp_crypto_operation() or odp_crypto_op_enq() results to the
- * application.
+ * odp_crypto_op_enq() (and the deprecated odp_crypto_operation())
+ * to the application.
*/
odp_queue_t compl_queue;
@@ -689,6 +690,8 @@ typedef struct odp_crypto_session_param_t {
/**
* Crypto API per packet operation parameters
+ *
+ * @deprecated Use odp_crypto_packet_op_param_t instead.
*/
typedef struct odp_crypto_op_param_t {
/** Session handle from creation */
@@ -760,7 +763,7 @@ typedef struct odp_crypto_op_param_t {
*/
odp_packet_data_range_t auth_range;
-} odp_crypto_op_param_t;
+} ODP_DEPRECATE(odp_crypto_op_param_t);
/**
* Crypto packet API per packet operation parameters
@@ -895,6 +898,8 @@ typedef struct odp_crypto_op_status {
/**
* Crypto API operation result
+ *
+ * @deprecated Use odp_crypto_packet_result_t instead.
*/
typedef struct odp_crypto_op_result {
/** Request completed successfully */
@@ -912,7 +917,7 @@ typedef struct odp_crypto_op_result {
/** Authentication status */
odp_crypto_op_status_t auth_status;
-} odp_crypto_op_result_t;
+} ODP_DEPRECATE(odp_crypto_op_result_t);
/**
* Crypto packet API operation result
@@ -1141,9 +1146,13 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param,
*/
int odp_crypto_session_destroy(odp_crypto_session_t session);
+#if ODP_DEPRECATED_API
+
/**
* Return crypto completion handle that is associated with event
*
+ * @deprecated Used only with the deprecated odp_crypto_operation()
+ *
* Note: any invalid parameters will cause undefined behavior and may cause
* the application to abort or crash.
*
@@ -1156,6 +1165,8 @@ odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev);
/**
* Convert crypto completion handle to event handle
*
+ * @deprecated Used only with the deprecated odp_crypto_operation()
+ *
* @param completion_event Completion event to convert to generic event
*
* @return Event handle
@@ -1165,6 +1176,8 @@ odp_event_t odp_crypto_compl_to_event(odp_crypto_compl_t completion_event);
/**
* Release crypto completion event
*
+ * @deprecated Used only with the deprecated odp_crypto_operation()
+ *
* @param completion_event Completion event we are done accessing
*/
void odp_crypto_compl_free(odp_crypto_compl_t completion_event);
@@ -1172,6 +1185,8 @@ void odp_crypto_compl_free(odp_crypto_compl_t completion_event);
/**
* Crypto per packet operation
*
+ * @deprecated Use odp_crypto_op() or odp_crypto_op_enq() instead.
+ *
* Performs the cryptographic operations specified during session creation
* on the packet. If the operation is performed synchronously, "posted"
* will return FALSE and the result of the operation is immediately available.
@@ -1192,12 +1207,16 @@ int odp_crypto_operation(odp_crypto_op_param_t *param,
/**
* Crypto per packet operation query result from completion event
*
+ * @deprecated Used only with the deprecated odp_crypto_operation()
+ *
* @param completion_event Event containing operation results
* @param result Pointer to result structure
*/
void odp_crypto_compl_result(odp_crypto_compl_t completion_event,
odp_crypto_op_result_t *result);
+#endif /* ODP_DEPRECATED_API */
+
/**
* Get printable value for an odp_crypto_session_t
*
@@ -1211,9 +1230,12 @@ void odp_crypto_compl_result(odp_crypto_compl_t completion_event,
*/
uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl);
+#if ODP_DEPRECATED_API
/**
* Get printable value for an odp_crypto_compl_t
*
+ * @deprecated Used only with the deprecated odp_crypto_operation()
+ *
* @param hdl odp_crypto_compl_t handle to be printed
* @return uint64_t value that can be used to print/display this
* handle
@@ -1223,6 +1245,7 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl);
* an odp_crypto_compl_t handle.
*/
uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl);
+#endif
/**
* Initialize crypto session parameters
diff --git a/include/odp/api/spec/dma_types.h b/include/odp/api/spec/dma_types.h
index d7ccc6d56..ae3d05287 100644
--- a/include/odp/api/spec/dma_types.h
+++ b/include/odp/api/spec/dma_types.h
@@ -18,7 +18,9 @@
extern "C" {
#endif
+#include <odp/api/event_types.h>
#include <odp/api/packet_types.h>
+#include <odp/api/queue_types.h>
#include <odp/api/std_types.h>
/** @defgroup odp_dma ODP DMA
diff --git a/include/odp/api/spec/event.h b/include/odp/api/spec/event.h
index dc99f35c7..c36d9cef1 100644
--- a/include/odp/api/spec/event.h
+++ b/include/odp/api/spec/event.h
@@ -1,4 +1,5 @@
/* Copyright (c) 2015-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -18,7 +19,8 @@
extern "C" {
#endif
-#include <odp/api/packet.h>
+#include <odp/api/event_types.h>
+#include <odp/api/packet_types.h>
/** @defgroup odp_event ODP EVENT
* Generic event metadata and operations.
@@ -26,81 +28,6 @@ extern "C" {
*/
/**
- * @typedef odp_event_t
- * ODP event
- */
-
-/**
- * @def ODP_EVENT_INVALID
- * Invalid event
- */
-
-/**
- * @typedef odp_event_type_t
- * Event type
- *
- * Event type specifies purpose and general format of an event. It can be
- * checked with odp_event_type() or odp_event_types(). Each event type has
- * functions (e.g. odp_buffer_from_event()) to convert between the generic event
- * handle (odp_event_t) and the type specific handle (e.g. odp_buffer_t).
- * Results are undefined, if conversion function of a wrong event type is used.
- * Application cannot change event type by chaining conversion functions.
- *
- * List of event types:
- * - ODP_EVENT_BUFFER
- * - Buffer event (odp_buffer_t) for simple data storage and message passing
- * - ODP_EVENT_PACKET
- * - Packet event (odp_packet_t) containing packet data and plenty of
- * packet processing related metadata
- * - ODP_EVENT_TIMEOUT
- * - Timeout event (odp_timeout_t) from a timer
- * - ODP_EVENT_CRYPTO_COMPL
- * - Crypto completion event (odp_crypto_compl_t)
- * - ODP_EVENT_IPSEC_STATUS
- * - IPSEC status update event (odp_ipsec_status_t)
- * - ODP_EVENT_PACKET_VECTOR
- * - Vector of packet events (odp_packet_t) as odp_packet_vector_t
- * - ODP_EVENT_PACKET_TX_COMPL
- * - Packet Tx completion event (odp_packet_tx_compl_t) generated as a result of a Packet Tx
- * completion.
- * - ODP_EVENT_DMA_COMPL
- * - DMA completion event (odp_dma_compl_t) indicates that a DMA transfer has finished
- */
-
-/**
- * @typedef odp_event_subtype_t
- * Event subtype
- *
- * Event subtype expands event type specification by providing more detailed
- * purpose and format of an event. It can be checked with odp_event_subtype() or
- * odp_event_types(). Each event subtype may define specific functions
- * (e.g. odp_ipsec_packet_from_event()) to convert between the generic event
- * handle (odp_event_t) and event type specific handle (e.g. odp_packet_t). When
- * subtype is known, these subtype specific functions should be preferred over
- * the event type general function (e.g. odp_packet_from_event()). Results are
- * undefined, if conversion function of a wrong event subtype is used.
- * Application cannot change event subtype by chaining conversion functions.
- *
- * List of event subtypes:
- * - ODP_EVENT_PACKET_BASIC
- * - Packet event (odp_packet_t) with basic packet metadata
- * - ODP_EVENT_PACKET_COMP
- * - Packet event (odp_packet_t) generated as a result of a compression/
- * decompression operation. It contains compression specific metadata in
- * addition to the basic packet metadata.
- * - ODP_EVENT_PACKET_CRYPTO
- * - Packet event (odp_packet_t) generated as a result of a Crypto
- * operation. It contains crypto specific metadata in addition to the
- * basic packet metadata.
- * - ODP_EVENT_PACKET_IPSEC
- * - Packet event (odp_packet_t) generated as a result of an IPsec
- * operation. It contains IPSEC specific metadata in addition to the basic
- * packet metadata.
- * - ODP_EVENT_NO_SUBTYPE
- * - An event type does not have any subtypes defined
- */
-
-/**
* Event type of an event
*
* Event type specifies purpose and general format of an event.
diff --git a/include/odp/api/spec/event_types.h b/include/odp/api/spec/event_types.h
new file mode 100644
index 000000000..529d031e7
--- /dev/null
+++ b/include/odp/api/spec/event_types.h
@@ -0,0 +1,110 @@
+/* Copyright (c) 2015-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODP event API type definitions
+ */
+
+#ifndef ODP_API_SPEC_EVENT_TYPES_H_
+#define ODP_API_SPEC_EVENT_TYPES_H_
+#include <odp/visibility_begin.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** @addtogroup odp_event
+ * @{
+ */
+
+/**
+ * @typedef odp_event_t
+ * ODP event
+ */
+
+/**
+ * @def ODP_EVENT_INVALID
+ * Invalid event
+ */
+
+/**
+ * @typedef odp_event_type_t
+ * Event type
+ *
+ * Event type specifies purpose and general format of an event. It can be
+ * checked with odp_event_type() or odp_event_types(). Each event type has
+ * functions (e.g. odp_buffer_from_event()) to convert between the generic event
+ * handle (odp_event_t) and the type specific handle (e.g. odp_buffer_t).
+ * Results are undefined, if conversion function of a wrong event type is used.
+ * Application cannot change event type by chaining conversion functions.
+ *
+ * List of event types:
+ * - ODP_EVENT_BUFFER
+ * - Buffer event (odp_buffer_t) for simple data storage and message passing
+ * - ODP_EVENT_PACKET
+ * - Packet event (odp_packet_t) containing packet data and plenty of
+ * packet processing related metadata
+ * - ODP_EVENT_TIMEOUT
+ * - Timeout event (odp_timeout_t) from a timer
+ * - ODP_EVENT_CRYPTO_COMPL
+ * - Crypto completion event (odp_crypto_compl_t)
+ * - ODP_EVENT_IPSEC_STATUS
+ * - IPSEC status update event (odp_ipsec_status_t)
+ * - ODP_EVENT_PACKET_VECTOR
+ * - Vector of packet events (odp_packet_t) as odp_packet_vector_t
+ * - ODP_EVENT_PACKET_TX_COMPL
+ * - Packet Tx completion event (odp_packet_tx_compl_t) generated as a result of a Packet Tx
+ * completion.
+ * - ODP_EVENT_DMA_COMPL
+ * - DMA completion event (odp_dma_compl_t) indicates that a DMA transfer has finished
+ */
+
+/**
+ * @typedef odp_event_subtype_t
+ * Event subtype
+ *
+ * Event subtype expands event type specification by providing more detailed
+ * purpose and format of an event. It can be checked with odp_event_subtype() or
+ * odp_event_types(). Each event subtype may define specific functions
+ * (e.g. odp_ipsec_packet_from_event()) to convert between the generic event
+ * handle (odp_event_t) and event type specific handle (e.g. odp_packet_t). When
+ * subtype is known, these subtype specific functions should be preferred over
+ * the event type general function (e.g. odp_packet_from_event()). Results are
+ * undefined, if conversion function of a wrong event subtype is used.
+ * Application cannot change event subtype by chaining conversion functions.
+ *
+ * List of event subtypes:
+ * - ODP_EVENT_PACKET_BASIC
+ * - Packet event (odp_packet_t) with basic packet metadata
+ * - ODP_EVENT_PACKET_COMP
+ * - Packet event (odp_packet_t) generated as a result of a compression/
+ * decompression operation. It contains compression specific metadata in
+ * addition to the basic packet metadata.
+ * - ODP_EVENT_PACKET_CRYPTO
+ * - Packet event (odp_packet_t) generated as a result of a Crypto
+ * operation. It contains crypto specific metadata in addition to the
+ * basic packet metadata.
+ * - ODP_EVENT_PACKET_IPSEC
+ * - Packet event (odp_packet_t) generated as a result of an IPsec
+ * operation. It contains IPSEC specific metadata in addition to the basic
+ * packet metadata.
+ * - ODP_EVENT_NO_SUBTYPE
+ * - An event type does not have any subtypes defined
+ */
+
+/**
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <odp/visibility_end.h>
+#endif
diff --git a/include/odp/api/spec/packet.h b/include/odp/api/spec/packet.h
index 8dc0a77bb..c745d29ad 100644
--- a/include/odp/api/spec/packet.h
+++ b/include/odp/api/spec/packet.h
@@ -19,10 +19,13 @@
extern "C" {
#endif
-#include <odp/api/proto_stats_types.h>
-#include <odp/api/time.h>
+#include <odp/api/event_types.h>
#include <odp/api/packet_types.h>
+#include <odp/api/packet_io_types.h>
#include <odp/api/pool_types.h>
+#include <odp/api/proto_stats_types.h>
+#include <odp/api/std_types.h>
+#include <odp/api/time.h>
/** @defgroup odp_packet ODP PACKET
* Packet event metadata and operations.
diff --git a/include/odp/api/spec/packet_flags.h b/include/odp/api/spec/packet_flags.h
index 7c1386cb6..98bf2b7bf 100644
--- a/include/odp/api/spec/packet_flags.h
+++ b/include/odp/api/spec/packet_flags.h
@@ -19,7 +19,7 @@ extern "C" {
#endif
#include <odp/api/std_types.h>
-#include <odp/api/packet.h>
+#include <odp/api/packet_types.h>
/** @addtogroup odp_packet
* Operations on packet metadata flags.
diff --git a/include/odp/api/spec/packet_io.h b/include/odp/api/spec/packet_io.h
index 3be45446d..4a8635d0a 100644
--- a/include/odp/api/spec/packet_io.h
+++ b/include/odp/api/spec/packet_io.h
@@ -431,9 +431,8 @@ int odp_pktin_recv_tmo(odp_pktin_queue_t queue, odp_packet_t packets[],
* @return Number of packets received
* @retval <0 on failure
*/
-int odp_pktin_recv_mq_tmo(const odp_pktin_queue_t queues[], unsigned int num_q,
- unsigned int *from, odp_packet_t packets[], int num,
- uint64_t wait);
+int odp_pktin_recv_mq_tmo(const odp_pktin_queue_t queues[], uint32_t num_q, uint32_t *from,
+ odp_packet_t packets[], int num, uint64_t wait);
/**
* Packet input wait time
diff --git a/include/odp/api/spec/packet_io_types.h b/include/odp/api/spec/packet_io_types.h
index 2d43822fe..153e188b7 100644
--- a/include/odp/api/spec/packet_io_types.h
+++ b/include/odp/api/spec/packet_io_types.h
@@ -257,7 +257,7 @@ typedef struct odp_pktin_queue_param_t {
* More than one input queues require flow hashing configured.
* The maximum value is defined by pktio capability 'max_input_queues'.
* Queue type is defined by the input mode. The default value is 1. */
- unsigned int num_queues;
+ uint32_t num_queues;
/** Queue parameters
*
@@ -304,7 +304,7 @@ typedef struct odp_pktout_queue_param_t {
/** Number of output queues to be created. The value must be between
* 1 and interface capability. The default value is 1. */
- unsigned int num_queues;
+ uint32_t num_queues;
/** Output queue size array
*
@@ -820,12 +820,12 @@ typedef struct odp_pktin_vector_capability_t {
*/
typedef struct odp_pktio_capability_t {
/** Maximum number of input queues */
- unsigned int max_input_queues;
+ uint32_t max_input_queues;
/** Maximum number of output queues
*
* Value does not exceed ODP_PKTOUT_MAX_QUEUES. */
- unsigned int max_output_queues;
+ uint32_t max_output_queues;
/** Minimum output queue size
*
diff --git a/include/odp/api/spec/pool_types.h b/include/odp/api/spec/pool_types.h
index b0d5b37c9..ef272b67d 100644
--- a/include/odp/api/spec/pool_types.h
+++ b/include/odp/api/spec/pool_types.h
@@ -127,12 +127,12 @@ typedef struct odp_pool_stats_t {
*/
typedef struct odp_pool_capability_t {
/** Maximum number of pools of any type */
- unsigned int max_pools;
+ uint32_t max_pools;
/** Buffer pool capabilities */
struct {
/** Maximum number of buffer pools */
- unsigned int max_pools;
+ uint32_t max_pools;
/** Maximum buffer data alignment in bytes */
uint32_t max_align;
@@ -162,7 +162,7 @@ typedef struct odp_pool_capability_t {
/** Packet pool capabilities */
struct {
/** Maximum number of packet pools */
- unsigned int max_pools;
+ uint32_t max_pools;
/** Maximum packet data length in bytes
*
@@ -250,7 +250,7 @@ typedef struct odp_pool_capability_t {
/** Timeout pool capabilities */
struct {
/** Maximum number of timeout pools */
- unsigned int max_pools;
+ uint32_t max_pools;
/** Maximum number of timeout events in a pool
*
@@ -271,7 +271,7 @@ typedef struct odp_pool_capability_t {
/** Vector pool capabilities */
struct {
/** Maximum number of vector pools */
- unsigned int max_pools;
+ uint32_t max_pools;
/** Maximum number of vector events in a pool
*
diff --git a/include/odp/api/spec/queue.h b/include/odp/api/spec/queue.h
index 9c677c926..65cf3d1ee 100644
--- a/include/odp/api/spec/queue.h
+++ b/include/odp/api/spec/queue.h
@@ -18,8 +18,9 @@
extern "C" {
#endif
-#include <odp/api/event.h>
+#include <odp/api/event_types.h>
#include <odp/api/queue_types.h>
+#include <odp/api/std_types.h>
/** @defgroup odp_queue ODP QUEUE
* Queues for event passing and scheduling.
diff --git a/include/odp/api/spec/schedule.h b/include/odp/api/spec/schedule.h
index 12da88f7d..524d9e2bf 100644
--- a/include/odp/api/spec/schedule.h
+++ b/include/odp/api/spec/schedule.h
@@ -19,7 +19,7 @@ extern "C" {
#endif
#include <odp/api/std_types.h>
-#include <odp/api/event.h>
+#include <odp/api/event_types.h>
#include <odp/api/queue_types.h>
#include <odp/api/schedule_types.h>
#include <odp/api/thrmask.h>
diff --git a/include/odp/api/spec/shared_memory.h b/include/odp/api/spec/shared_memory.h
index 6ba6f7fc2..d1955db26 100644
--- a/include/odp/api/spec/shared_memory.h
+++ b/include/odp/api/spec/shared_memory.h
@@ -133,7 +133,7 @@ typedef struct odp_shm_capability_t {
*
* This number of separate shared memory blocks can be
* reserved concurrently. */
- unsigned int max_blocks;
+ uint32_t max_blocks;
/** Maximum memory block size in bytes
*
diff --git a/include/odp/api/spec/timer.h b/include/odp/api/spec/timer.h
index 9df43d034..b51db7dbc 100644
--- a/include/odp/api/spec/timer.h
+++ b/include/odp/api/spec/timer.h
@@ -21,7 +21,7 @@ extern "C" {
#endif
#include <odp/api/timer_types.h>
-#include <odp/api/event.h>
+#include <odp/api/event_types.h>
#include <odp/api/pool_types.h>
#include <odp/api/queue_types.h>
diff --git a/include/odp/api/spec/timer_types.h b/include/odp/api/spec/timer_types.h
index 56c118ce3..754c831b2 100644
--- a/include/odp/api/spec/timer_types.h
+++ b/include/odp/api/spec/timer_types.h
@@ -21,7 +21,7 @@ extern "C" {
#endif
#include <odp/api/std_types.h>
-#include <odp/api/event.h>
+#include <odp/api/event_types.h>
/** @defgroup odp_timer ODP TIMER
* Timer generating timeout events.
diff --git a/include/odp/api/spec/traffic_mngr.h b/include/odp/api/spec/traffic_mngr.h
index 64f2eac83..b22887fc6 100644
--- a/include/odp/api/spec/traffic_mngr.h
+++ b/include/odp/api/spec/traffic_mngr.h
@@ -98,11 +98,6 @@ extern "C" {
*/
/**
- * @def ODP_NUM_SHAPER_COLORS
- * The number of enumeration values defined in the odp_tm_shaper_color_t type.
- */
-
-/**
* @def ODP_TM_INVALID_PRIORITY
* Used to indicate an invalid priority value.
*/
@@ -344,9 +339,19 @@ typedef struct {
* the legal range of such weights. */
odp_bool_t weights_supported;
- /** tm_node_threshold indicates that the tm_nodes at this
- * level support threshold profiles. */
- odp_bool_t tm_node_threshold;
+ /** TM node threshold profile support */
+ struct {
+ /** Threshold given as bytes */
+ uint8_t byte : 1;
+
+ /** Threshold given as packets */
+ uint8_t packet : 1;
+
+ /** Threshold given as bytes and packets simultaneously */
+ uint8_t byte_and_packet : 1;
+
+ } tm_node_threshold;
+
} odp_tm_level_capabilities_t;
/** The tm_pkt_prio_mode_t enumeration type is used to indicate different
@@ -515,11 +520,18 @@ typedef struct {
/** TM queue statistics counter capabilities */
odp_tm_queue_stats_capability_t queue_stats;
- /** tm_queue_threshold indicates support for threshold profile on a
- * TM queue. When TRUE, users can set/clear/update threshold profile
- * on a TM queue. When false none of it is supported.
- */
- odp_bool_t tm_queue_threshold;
+ /** TM queue threshold profile support */
+ struct {
+ /** Threshold given as bytes */
+ uint8_t byte : 1;
+
+ /** Threshold given as packets */
+ uint8_t packet : 1;
+
+ /** Threshold given as bytes and packets simultaneously */
+ uint8_t byte_and_packet : 1;
+
+ } tm_queue_threshold;
/** tm_queue_query_flags indicates supported types of TM queue query.
* Types of TM queue query are same as query_flags that are passed to
@@ -719,7 +731,6 @@ typedef struct {
*
* odp_tm_requirements_init() must be called to initialize any
* odp_tm_requirements_t record before it is first used or assigned to.
- * This is done to allow for vendor specific additions to this record.
*
* @param requirements A pointer to an odp_tm_requirements_t record which
* is to be initialized.
@@ -730,7 +741,6 @@ void odp_tm_requirements_init(odp_tm_requirements_t *requirements);
*
* odp_tm_egress_init() must be called to initialize any odp_tm_egress_t
* record before it is first used or assigned to.
- * This is done to allow for vendor specific additions to this record.
*
* @param egress A pointer to an odp_tm_egress_t record which
* is to be initialized.
@@ -940,9 +950,9 @@ int odp_tm_destroy(odp_tm_t tm);
* The odp_tm_vlan_marking() function allows one to configure the TM egress
* so as to have it set the one bit VLAN Drop Eligibility Indicator (DEI)
* field (but only for pkts that already carry a VLAN tag) of a pkt based upon
- * the final pkt (or shaper?) color assigned to the pkt when it reaches the
- * egress node. When drop_eligible_enabled is false, then the given color has
- * no effect on the VLAN fields. See IEEE 802.1q for more details.
+ * the final pkt color assigned to the pkt when it reaches the egress node. When
+ * drop_eligible_enabled is false, then the given color has no effect on the
+ * VLAN fields. See IEEE 802.1q for more details.
*
* Note that ALL ODP implementations are required to SUCCESSFULLY handle all
* calls to this function with drop_eligible_enabled == FALSE - i.e. must
@@ -1037,24 +1047,11 @@ int odp_tm_drop_prec_marking(odp_tm_t tm,
/* Shaper profile types and functions
* -------------------------------------------------------- */
-/** Possible values of running the shaper algorithm. ODP_TM_SHAPER_GREEN
- * means that the traffic is within the commit specification (rate and burst
- * size), ODP_TM_SHAPER_YELLOW means that the traffic is within the peak
- * specification (rate and burst size) and ODP_TM_SHAPER_RED means that the
- * traffic is exceeding both its commit and peak specifications. Note that
- * packets can also have an assigned <b> packet color</b> of ODP_PACKET_GREEN,
- * ODP_PACKET_YELLOW or ODP_PACKET_RED which has a different meaning and
- * purpose than the shaper colors.
- */
-typedef enum {
- ODP_TM_SHAPER_GREEN, ODP_TM_SHAPER_YELLOW, ODP_TM_SHAPER_RED
-} odp_tm_shaper_color_t;
-
-/** The odp_tm_shaper_params_t record type is used to supply the parameters
- * associated with a shaper profile. Since it is expected that
- * implementations might augment this record type with platform specific
- * additional fields - it is required that odp_tm_shaper_params_init() be
- * called on variables of this type before any of the fields are filled in.
+/**
+ * TM shaper parameters
+ *
+ * Use odp_tm_shaper_params_init() to initialize parameters into their default
+ * values.
*/
typedef struct {
/** The committed information rate for this shaper profile. The units
@@ -1123,12 +1120,12 @@ typedef struct {
odp_bool_t packet_mode;
} odp_tm_shaper_params_t;
-/** odp_tm_shaper_params_init() must be called to initialize any
- * odp_tm_shaper_params_t record before it is first used or assigned to.
- * The parameters are initialized to their default values.
+/**
+ * Initialize TM shaper parameters
*
- * @param params A pointer to an odp_tm_shaper_params_t record which
- * is to be initialized.
+ * Initialize an odp_tm_shaper_params_t to its default values for all fields.
+ *
+ * @param params Address of the odp_tm_shaper_params_t to be initialized
*/
void odp_tm_shaper_params_init(odp_tm_shaper_params_t *params);
@@ -1176,8 +1173,8 @@ int odp_tm_shaper_params_read(odp_tm_shaper_t shaper_profile,
/** odp_tm_shaper_params_update() "sets" the current set of values associated
* with the specified shaper profile object. In addition, this call has the
- * effect that all tm_input's and tm_nodes that are associated (attached?)
- * with this shaper profile object will be updated with the new values.
+ * effect that all tm_input's and tm_nodes that are associated with this shaper
+ * profile object will be updated with the new values.
*
* @param shaper_profile Specifies the shaper profile object whose
* values are to be set.
@@ -1215,11 +1212,11 @@ typedef enum {
ODP_TM_FRAME_BASED_WEIGHTS /**< Ignore the packet length */
} odp_tm_sched_mode_t;
-/** The odp_tm_sched_params_t record type is used to supply the parameters
- * associated with a scheduler profile. Since it is expected that
- * implementations might augment this record type with platform specific
- * additional fields - it is required that odp_tm_sched_params_init() be
- * called on variables of this type before any of the fields are filled in.
+/**
+ * TM scheduler parameters
+ *
+ * Use odp_tm_sched_params_init() to initialize parameters into their default
+ * values.
*/
typedef struct {
/** sched_modes indicates whether weighted scheduling should be used
@@ -1237,12 +1234,12 @@ typedef struct {
uint32_t sched_weights[ODP_TM_MAX_PRIORITIES];
} odp_tm_sched_params_t;
-/** odp_tm_sched_params_init() must be called to initialize any
- * odp_tm_sched_params_t record before it is first used or assigned to.
- * The parameters are initialized to their default values.
+/**
+ * Initialize TM scheduler parameters
*
- * @param params A pointer to an odp_tm_sched_params_t record which
- * is to be initialized.
+ * Initialize an odp_tm_sched_params_t to its default values for all fields.
+ *
+ * @param params Address of the odp_tm_sched_params_t to be initialized
*/
void odp_tm_sched_params_init(odp_tm_sched_params_t *params);
@@ -1289,8 +1286,8 @@ int odp_tm_sched_params_read(odp_tm_sched_t sched_profile,
/** odp_tm_sched_params_update() "sets" the current set of values associated
* with the specified scheduler profile object. In addition, this call has
- * the effect that all tm_nodes that are associated (attached?) with this
- * Scheduler profile object will be updated with the new values.
+ * the effect that all tm_nodes that are associated with this scheduler profile
+ * object will be updated with the new values.
*
* @param sched_profile Specifies the Scheduler profile object whose
* values are to be set.
@@ -1315,11 +1312,11 @@ odp_tm_sched_t odp_tm_sched_lookup(const char *name);
/* Queue Threshold Profiles - types and functions
* -------------------------------------------------------- */
-/** The odp_tm_threshold_params_t record type is used to supply the parameters
- * associated with a queue thresholds profile. Since it is expected that
- * implementations might augment this record type with platform specific
- * additional fields - it is required that odp_tm_threshold_params_init() be
- * called on variables of this type before any of the fields are filled in
+/**
+ * TM threshold parameters
+ *
+ * Use odp_tm_threshold_params_init() to initialize parameters into their
+ * default values.
*/
typedef struct {
uint64_t max_pkts; /**< max pkt cnt for this threshold profile */
@@ -1332,12 +1329,12 @@ typedef struct {
odp_bool_t enable_max_bytes;
} odp_tm_threshold_params_t;
-/** odp_tm_threshold_params_init() must be called to initialize any
- * odp_tm_threshold_params_t record before it is first used or assigned to.
- * The parameters are initialized to their default values.
+/**
+ * Initialize TM threshold parameters
*
- * @param params A pointer to an odp_tm_threshold_params_t record which
- * is to be initialized.
+ * Initialize an odp_tm_threshold_params_t to its default values for all fields.
+ *
+ * @param params Address of the odp_tm_threshold_params_t to be initialized
*/
void odp_tm_threshold_params_init(odp_tm_threshold_params_t *params);
@@ -1387,8 +1384,8 @@ int odp_tm_thresholds_params_read(odp_tm_threshold_t threshold_profile,
/** odp_tm_thresholds_params_update() "sets" the current set of values
* associated with the specified queue thresholds profile object. In addition,
* this call has the effect that all tm_input's and tm_nodes that are
- * associated (attached?) with this queue thresholds profile object will be
- * updated with the new values.
+ * associated with this queue thresholds profile object will be updated with the
+ * new values.
*
* @param threshold_profile Specifies the queue thresholds profile
* object whose values are to be set.
@@ -1413,11 +1410,11 @@ odp_tm_threshold_t odp_tm_thresholds_lookup(const char *name);
/* WRED Profiles - types and functions
* -------------------------------------------------------- */
-/** The odp_tm_wred_params_t record type is used to supply the parameters
- * associated with a Random Early Detection profile. Since it is expected that
- * implementations might augment this record type with platform specific
- * additional fields - it is required that odp_tm_wred_params_init() be called
- * on variables of this type before any of the fields are filled in.
+/**
+ * TM WRED parameters
+ *
+ * Use odp_tm_wred_params_init() to initialize parameters into their default
+ * values.
*/
typedef struct {
/** When min_threshold is set to zero then single-slope WRED is
@@ -1471,12 +1468,12 @@ typedef struct {
odp_bool_t use_byte_fullness;
} odp_tm_wred_params_t;
-/** odp_tm_wred_params_init() must be called to initialize any
- * odp_tm_wred_params_t record before it is first used or assigned to.
- * The parameters are initialized to their default values.
+/**
+ * Initialize TM WRED parameters
*
- * @param params A pointer to an odp_tm_wred_params_t record which
- * is to be initialized.
+ * Initialize an odp_tm_wred_params_t to its default values for all fields.
+ *
+ * @param params Address of the odp_tm_wred_params_t to be initialized
*/
void odp_tm_wred_params_init(odp_tm_wred_params_t *params);
@@ -1524,8 +1521,8 @@ int odp_tm_wred_params_read(odp_tm_wred_t wred_profile,
/** odp_tm_wred_params_update() "sets" the current set of values associated
* with the specified WRED profile object. In addition, this call has the
- * effect that all tm_input's and tm_nodes that are associated (attached?)
- * with this WRED profile object will be updated with the new values.
+ * effect that all tm_input's and tm_nodes that are associated with this WRED
+ * profile object will be updated with the new values.
*
* @param wred_profile Specifies the WRED profile object whose
* values are to be set.
@@ -1547,12 +1544,11 @@ int odp_tm_wred_params_update(odp_tm_wred_t wred_profile,
*/
odp_tm_wred_t odp_tm_wred_lookup(const char *name);
-/** The odp_tm_node_params_t record type is used to hold extra parameters when
- * calling the odp_tm_node_create() function. Many of these fields are
- * optional EXCEPT for max_fanin and level. Also since it is expected that
- * implementations might augment this record type with platform specific
- * additional fields - it is required that odp_tm_node_params_init() be called
- * on variables of this type before any of the fields are filled in.
+/**
+ * TM node parameters
+ *
+ * Many of these fields are optional EXCEPT for max_fanin and level. Use
+ * odp_tm_node_params_init() to initialize parameters into their default values.
*/
typedef struct {
/** The user_context field is an generic pointer that the user can
@@ -1571,9 +1567,9 @@ typedef struct {
odp_tm_shaper_t shaper_profile;
/** The threshold profile to be used in setting the max queue fullness
- * for WRED and/or tail drop? Can be ODP_TM_INVALID and can also be
- * set and changed post-creation via odp_tm_node_threshold_config().
- * The default value is ODP_TM_INVALID. */
+ * for WRED and/or tail drop. Can be ODP_TM_INVALID and can also be set
+ * and changed post-creation via odp_tm_node_threshold_config(). The
+ * default value is ODP_TM_INVALID. */
odp_tm_threshold_t threshold_profile;
/** The WRED profile(s) to be associated with this tm_node. Any or
@@ -1606,12 +1602,12 @@ typedef struct {
uint8_t priority;
} odp_tm_node_params_t;
-/** odp_tm_node_params_init() must be called to initialize any
- * odp_tm_node_params_t record before it is first used or assigned to.
- * The parameters are initialized to their default values.
+/**
+ * Initialize TM node parameters
*
- * @param params A pointer to an odp_tm_node_params_t record which
- * is to be initialized.
+ * Initialize an odp_tm_node_params_t to its default values for all fields.
+ *
+ * @param params Address of the odp_tm_node_params_t to be initialized
*/
void odp_tm_node_params_init(odp_tm_node_params_t *params);
@@ -1627,8 +1623,7 @@ void odp_tm_node_params_init(odp_tm_node_params_t *params);
* @param name Optional name that can be used later later to find this
* same odp_tm_node_t. Can be NULL, otherwise must be
* unique across all odp_tm_node objects.
- * @param params A pointer to a record holding (an extensible) set of
- * properties/attributes of this tm_node.
+ * @param params TM node parameters.
*
* @return Returns ODP_TM_INVALID upon failure, otherwise returns
* a valid odp_tm_node_t handle if successful.
@@ -1739,12 +1734,11 @@ void *odp_tm_node_context(odp_tm_node_t tm_node);
*/
int odp_tm_node_context_set(odp_tm_node_t tm_node, void *user_context);
-/** The odp_tm_queue_params_t record type is used to hold extra parameters
- * when calling the odp_tm_queue_create() function. Many of these fields are
- * optional EXCEPT for priority. Also since it is expected that
- * implementations might augment this record type with platform specific
- * additional fields - it is required that odp_tm_queue_params_init() be
- * called on variables of this type before any of the fields are filled in.
+/**
+ * TM queue parameters
+ *
+ * Use odp_tm_queue_params_init() to initialize parameters into their default
+ * values.
*/
typedef struct {
/** The user_context field is an generic pointer that the user can
@@ -1758,9 +1752,13 @@ typedef struct {
odp_tm_shaper_t shaper_profile;
/** The threshold profile to be used in setting the max queue fullness
- * for WRED and/or tail drop? Can be ODP_TM_INVALID and can also be
- * set and changed post-creation via odp_tm_queue_threshold_config().
- * The default value is ODP_TM_INVALID. */
+ * for WRED and/or tail drop. Can be ODP_TM_INVALID and can also be set
+ * and changed post-creation via odp_tm_queue_threshold_config(). The
+ * default value is ODP_TM_INVALID.
+ *
+ * One can specify the maximum queue limits either as a maximum number
+ * of packets in the queue or as a maximum number of bytes in the queue,
+ * or if both are specified, then whichever limit is hit first. */
odp_tm_threshold_t threshold_profile;
/** The WRED profile(s) to be associated with this tm_queue. Any or
@@ -1782,27 +1780,24 @@ typedef struct {
odp_bool_t ordered_enqueue;
} odp_tm_queue_params_t;
-/** odp_tm_queue_params_init() must be called to initialize any
- * odp_tm_queue_params_t record before it is first used or assigned to.
- * The parameters are initialized to their default values.
+/**
+ * Initialize TM queue parameters
*
- * @param params A pointer to an odp_tm_queue_params_t record which
- * is to be initialized.
+ * Initialize an odp_tm_queue_params_t to its default values for all fields.
+ *
+ * @param params Address of the odp_tm_queue_params_t to be initialized
*/
void odp_tm_queue_params_init(odp_tm_queue_params_t *params);
-/** Create an tm_queue object. One can specify the maximum queue limits
- * either as a maximum number of packets in the queue OR as a maximum number
- * of bytes in the queue, or if both are specified, then whichever limit is
- * hit first. Note that in the case of specifying the maximum queue memory
- * size as bytes, the system is free to instead convert this byte value into a
- * number of buffers and instead limit the queue memory usage by buffer counts
- * versus strictly using byte counts.
+/**
+ * TM queue create
+ *
+ * Create a TM queue according to the queue parameters. Use
+ * odp_tm_queue_params_init() to initialize parameters into their default values.
*
* @param tm Handle of the TM system into which this odp_tm_queue object is
* created.
- * @param params A pointer to a record holding (an extensible) set of
- * properties/attributes of this tm_queue.
+ * @param params TM queue parameters.
*
* @return Returns ODP_TM_INVALID upon failure, otherwise a valid
* odp_tm_queue_t handle.
@@ -1965,18 +1960,16 @@ int odp_tm_queue_disconnect(odp_tm_queue_t tm_queue);
/* Input API
* -------------------------------------------------------- */
-/** The odp_tm_enq() function is used to add packets to a given TM system.
- * Note that the System Metadata associated with the pkt needed by the TM
- * system is (a) a drop_eligible bit, (b) a two bit "pkt_color", (c) a 16-bit
- * pkt_len, and MAYBE? (d) a signed 8-bit shaper_len_adjust.
+/** Send packet to TM system
+ *
+ * Note that the packet metadata utilized by the TM system is (a)
+ * drop_eligible, (b) pkt_color, (c) pkt_len, and (d) shaper_len_adjust.
*
* If there is a non-zero shaper_len_adjust, then it is added to the pkt_len
* after any non-zero shaper_len_adjust that is part of the shaper profile.
*
* The pkt_color bits are a result of some earlier Metering/Marking/Policing
- * processing (typically ingress based), and should not be confused with the
- * shaper_color produced from the TM shaper entities within the tm_inputs and
- * tm_nodes.
+ * processing.
*
* @param tm_queue Specifies the tm_queue (and indirectly the TM system).
* @param pkt Handle to a packet.
@@ -2023,8 +2016,8 @@ int odp_tm_enq_multi_lso(odp_tm_queue_t tm_queue, const odp_packet_t packets[],
const odp_packet_lso_opt_t *lso_opt);
/** The odp_tm_enq_with_cnt() function behaves identically to odp_tm_enq(),
- * except that it also returns (an approximation to?) the current tm_queue
- * packet queue count.
+ * except that it also returns the current tm_queue packet queue count (may be
+ * an approximation).
*
* @param tm_queue Specifies the tm_queue (and indirectly the TM system).
* @param pkt Handle to a packet.
@@ -2194,7 +2187,7 @@ int odp_tm_queue_info(odp_tm_queue_t tm_queue, odp_tm_queue_info_t *info);
*/
#define ODP_TM_QUERY_PKT_CNT 0x01 /**< The total_pkt_cnt value */
#define ODP_TM_QUERY_BYTE_CNT 0x02 /**< The total_byte_cnt value */
-#define ODP_TM_QUERY_THRESHOLDS 0x04 /**< The thresholds??? */
+#define ODP_TM_QUERY_THRESHOLDS 0x04 /**< The threshold values */
/** The odp_tm_query_info_t record type is used to return the various counts
* as requested by functions like odp_tm_queue_query() and
diff --git a/include/odp/api/timer.h b/include/odp/api/timer.h
index 3041594f5..356f7e951 100644
--- a/include/odp/api/timer.h
+++ b/include/odp/api/timer.h
@@ -17,6 +17,8 @@
extern "C" {
#endif
+#include <odp/api/abi/timer.h>
+
#include <odp/api/spec/timer.h>
#ifdef __cplusplus
diff --git a/include/odp/arch/arm32-linux/odp/api/abi/event_types.h b/include/odp/arch/arm32-linux/odp/api/abi/event_types.h
new file mode 100644
index 000000000..a8321920c
--- /dev/null
+++ b/include/odp/arch/arm32-linux/odp/api/abi/event_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/event_types.h>
diff --git a/include/odp/arch/arm32-linux/odp/api/abi/timer.h b/include/odp/arch/arm32-linux/odp/api/abi/timer.h
new file mode 100644
index 000000000..0351468d3
--- /dev/null
+++ b/include/odp/arch/arm32-linux/odp/api/abi/timer.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/timer.h>
diff --git a/include/odp/arch/arm64-linux/odp/api/abi/event_types.h b/include/odp/arch/arm64-linux/odp/api/abi/event_types.h
new file mode 100644
index 000000000..a8321920c
--- /dev/null
+++ b/include/odp/arch/arm64-linux/odp/api/abi/event_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/event_types.h>
diff --git a/include/odp/arch/arm64-linux/odp/api/abi/timer.h b/include/odp/arch/arm64-linux/odp/api/abi/timer.h
new file mode 100644
index 000000000..0351468d3
--- /dev/null
+++ b/include/odp/arch/arm64-linux/odp/api/abi/timer.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/timer.h>
diff --git a/include/odp/arch/default-linux/odp/api/abi/event_types.h b/include/odp/arch/default-linux/odp/api/abi/event_types.h
new file mode 100644
index 000000000..a8321920c
--- /dev/null
+++ b/include/odp/arch/default-linux/odp/api/abi/event_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/event_types.h>
diff --git a/include/odp/arch/default-linux/odp/api/abi/timer.h b/include/odp/arch/default-linux/odp/api/abi/timer.h
new file mode 100644
index 000000000..0351468d3
--- /dev/null
+++ b/include/odp/arch/default-linux/odp/api/abi/timer.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/timer.h>
diff --git a/include/odp/arch/power64-linux/odp/api/abi/event_types.h b/include/odp/arch/power64-linux/odp/api/abi/event_types.h
new file mode 100644
index 000000000..a8321920c
--- /dev/null
+++ b/include/odp/arch/power64-linux/odp/api/abi/event_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/event_types.h>
diff --git a/include/odp/arch/power64-linux/odp/api/abi/timer.h b/include/odp/arch/power64-linux/odp/api/abi/timer.h
new file mode 100644
index 000000000..0351468d3
--- /dev/null
+++ b/include/odp/arch/power64-linux/odp/api/abi/timer.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/timer.h>
diff --git a/include/odp/arch/x86_32-linux/odp/api/abi/event_types.h b/include/odp/arch/x86_32-linux/odp/api/abi/event_types.h
new file mode 100644
index 000000000..a8321920c
--- /dev/null
+++ b/include/odp/arch/x86_32-linux/odp/api/abi/event_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/event_types.h>
diff --git a/include/odp/arch/x86_32-linux/odp/api/abi/timer.h b/include/odp/arch/x86_32-linux/odp/api/abi/timer.h
new file mode 100644
index 000000000..0351468d3
--- /dev/null
+++ b/include/odp/arch/x86_32-linux/odp/api/abi/timer.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/timer.h>
diff --git a/include/odp/arch/x86_64-linux/odp/api/abi/event_types.h b/include/odp/arch/x86_64-linux/odp/api/abi/event_types.h
new file mode 100644
index 000000000..a8321920c
--- /dev/null
+++ b/include/odp/arch/x86_64-linux/odp/api/abi/event_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/event_types.h>
diff --git a/include/odp/arch/x86_64-linux/odp/api/abi/timer.h b/include/odp/arch/x86_64-linux/odp/api/abi/timer.h
new file mode 100644
index 000000000..0351468d3
--- /dev/null
+++ b/include/odp/arch/x86_64-linux/odp/api/abi/timer.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/timer.h>
diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am
index bb2f733bc..70ce4b980 100644
--- a/platform/linux-dpdk/Makefile.am
+++ b/platform/linux-dpdk/Makefile.am
@@ -48,7 +48,9 @@ odpapiplatinclude_HEADERS = \
include/odp/api/plat/sync_inlines.h \
include/odp/api/plat/thread_inlines.h \
include/odp/api/plat/ticketlock_inlines.h \
- include/odp/api/plat/time_inlines.h
+ include/odp/api/plat/time_inlines.h \
+ include/odp/api/plat/timer_inlines.h \
+ include/odp/api/plat/timer_inline_types.h
odpapiabiarchinclude_HEADERS += \
include-abi/odp/api/abi/align.h \
@@ -64,6 +66,7 @@ odpapiabiarchinclude_HEADERS += \
include-abi/odp/api/abi/dma_types.h \
include-abi/odp/api/abi/errno.h \
include-abi/odp/api/abi/event.h \
+ include-abi/odp/api/abi/event_types.h \
include-abi/odp/api/abi/hash.h \
include-abi/odp/api/abi/init.h \
include-abi/odp/api/abi/ipsec.h \
@@ -94,6 +97,7 @@ odpapiabiarchinclude_HEADERS += \
include-abi/odp/api/abi/thrmask.h \
include-abi/odp/api/abi/ticketlock.h \
include-abi/odp/api/abi/time.h \
+ include-abi/odp/api/abi/timer.h \
include-abi/odp/api/abi/timer_types.h \
include-abi/odp/api/abi/traffic_mngr.h \
include-abi/odp/api/abi/version.h
@@ -121,9 +125,10 @@ noinst_HEADERS = \
include/odp_packet_internal.h \
${top_srcdir}/platform/linux-generic/include/odp_name_table_internal.h \
include/odp_packet_io_internal.h \
+ ${top_srcdir}/platform/linux-generic/include/odp_parse_internal.h \
include/odp_errno_define.h \
include/odp_event_internal.h \
- include/odp_packet_dpdk.h \
+ ${top_srcdir}/platform/linux-generic/include/odp_packet_dpdk.h \
${top_srcdir}/platform/linux-generic/include/odp_pcapng.h \
${top_srcdir}/platform/linux-generic/include/odp_pkt_queue_internal.h \
include/odp_pool_internal.h \
@@ -150,6 +155,7 @@ noinst_HEADERS = \
include/odp_timer_internal.h \
${top_srcdir}/platform/linux-generic/include/odp_timer_wheel_internal.h \
${top_srcdir}/platform/linux-generic/include/odp_traffic_mngr_internal.h \
+ ${top_srcdir}/platform/linux-generic/include/odp_types_internal.h \
include/odp_event_vector_internal.h \
include/protocols/eth.h \
include/protocols/ip.h \
@@ -192,6 +198,7 @@ __LIB__libodp_dpdk_la_SOURCES = \
../linux-generic/pktio/dpdk_parse.c \
odp_packet_flags.c \
../linux-generic/odp_packet_io.c \
+ ../linux-generic/odp_parse.c \
../linux-generic/pktio/loop.c \
../linux-generic/pktio/null.c \
../linux-generic/odp_pkt_queue.c \
@@ -243,7 +250,8 @@ __LIB__libodp_dpdk_la_SOURCES += \
../linux-generic/odp_sync_api.c \
../linux-generic/odp_thread_api.c \
../linux-generic/odp_ticketlock_api.c \
- ../linux-generic/odp_time_api.c
+ ../linux-generic/odp_time_api.c \
+ ../linux-generic/odp_timer_api.c
endif
if ARCH_IS_ARM
diff --git a/platform/linux-dpdk/README b/platform/linux-dpdk/README
index d2d2bec8e..4f76b6448 100644
--- a/platform/linux-dpdk/README
+++ b/platform/linux-dpdk/README
@@ -1,5 +1,5 @@
Copyright (c) 2018-2019, Linaro Limited
-Copyright (c) 2019-2021, Nokia
+Copyright (c) 2019-2022, Nokia
All rights reserved.
SPDX-License-Identifier: BSD-3-Clause
@@ -23,27 +23,26 @@ Prerequisites and considerations:
(4.15.0 kernel).
- DPDK only works on a selected range of network cards. The list of known and
supported devices can be found in the DPDK documentation:
- http://doc.dpdk.org/guides-19.11/nics/index.html
- http://doc.dpdk.org/guides-20.11/nics/index.html
+ https://doc.dpdk.org/guides/nics/index.html
2. Preparing DPDK
=================================================
-Please refer to http://dpdk.org/doc/guides/linux_gsg/build_dpdk.html for more
+Please refer to https://dpdk.org/doc/guides/linux_gsg/build_dpdk.html for more
details on how to build DPDK. Best effort is done to provide some help on DPDK
cmds below for Ubuntu, where it has been compiled and tested.
On Ubuntu install pcap development library:
sudo apt-get install libpcap-dev
-Right now odp-dpdk supports DPDK v19.11 (recommended version) and
-v20.11. v18.11 may also work.
+Right now ODP-DPDK supports DPDK v19.11 (recommended version), v20.11, and
+v21.11.
Compile DPDK 19.11
------------------
Fetch the DPDK code:
- git clone http://dpdk.org/git/dpdk-stable --branch 19.11 --depth 1 ./<dpdk-dir>
+ git clone https://dpdk.org/git/dpdk-stable --branch 19.11 --depth 1 ./<dpdk-dir>
This has to be done only once:
cd <dpdk-dir>
@@ -72,10 +71,10 @@ STATIC libraries:
This only ensures building DPDK, but traffic is not tested with this build yet.
-Compile DPDK 20.11
-------------------
+Compile DPDK from v20.11 onwards
+--------------------------------
Fetch the DPDK code:
- git clone http://dpdk.org/git/dpdk-stable --branch 20.11 --depth 1 ./<dpdk-dir>
+ git clone https://dpdk.org/git/dpdk-stable --branch 20.11 --depth 1 ./<dpdk-dir>
Prepare the build directory:
cd <dpdk-dir>
@@ -291,7 +290,7 @@ what it does.
======================================================
Refer to the DPDK crypto documentation for detailed crypto PMD build instructions:
-http://dpdk.org/doc/guides/cryptodevs/index.html
+https://dpdk.org/doc/guides/cryptodevs/index.html
To build odp-dpdk with DPDK virtual crypto devices, we need to build supporting
Intel Multi-Buffer Crypto for IPsec library prior to DPDK build.
diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/event_types.h b/platform/linux-dpdk/include-abi/odp/api/abi/event_types.h
new file mode 120000
index 000000000..0e5c90929
--- /dev/null
+++ b/platform/linux-dpdk/include-abi/odp/api/abi/event_types.h
@@ -0,0 +1 @@
+../../../../../linux-generic/include-abi/odp/api/abi/event_types.h \ No newline at end of file
diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/timer.h b/platform/linux-dpdk/include-abi/odp/api/abi/timer.h
new file mode 120000
index 000000000..e65d4faf5
--- /dev/null
+++ b/platform/linux-dpdk/include-abi/odp/api/abi/timer.h
@@ -0,0 +1 @@
+../../../../../linux-generic/include-abi/odp/api/abi/timer.h \ No newline at end of file
diff --git a/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h b/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h
index 632ba0e69..9e0ae3f50 100644
--- a/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h
+++ b/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h
@@ -8,7 +8,7 @@
#define ODP_PLAT_BUFFER_INLINES_H_
#include <odp/api/abi/buffer.h>
-#include <odp/api/abi/event.h>
+#include <odp/api/abi/event_types.h>
#include <odp/api/plat/buffer_inline_types.h>
diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_flag_inlines.h b/platform/linux-dpdk/include/odp/api/plat/packet_flag_inlines.h
index e9e6cf028..7482fca93 100644
--- a/platform/linux-dpdk/include/odp/api/plat/packet_flag_inlines.h
+++ b/platform/linux-dpdk/include/odp/api/plat/packet_flag_inlines.h
@@ -33,12 +33,33 @@ static inline uint64_t _odp_packet_input_flags(odp_packet_t pkt)
/* Inline functions by default */
#define _ODP_INLINE static inline
#define odp_packet_has_l2 __odp_packet_has_l2
+ #define odp_packet_has_l3 __odp_packet_has_l3
+ #define odp_packet_has_l4 __odp_packet_has_l4
#define odp_packet_has_eth __odp_packet_has_eth
#define odp_packet_has_jumbo __odp_packet_has_jumbo
#define odp_packet_has_flow_hash __odp_packet_has_flow_hash
#define odp_packet_has_flow_hash_clr __odp_packet_has_flow_hash_clr
#define odp_packet_has_ts __odp_packet_has_ts
#define odp_packet_has_ipsec __odp_packet_has_ipsec
+ #define odp_packet_has_eth_bcast __odp_packet_has_eth_bcast
+ #define odp_packet_has_eth_mcast __odp_packet_has_eth_mcast
+ #define odp_packet_has_vlan __odp_packet_has_vlan
+ #define odp_packet_has_vlan_qinq __odp_packet_has_vlan_qinq
+ #define odp_packet_has_arp __odp_packet_has_arp
+ #define odp_packet_has_ipv4 __odp_packet_has_ipv4
+ #define odp_packet_has_ipv6 __odp_packet_has_ipv6
+ #define odp_packet_has_ip_bcast __odp_packet_has_ip_bcast
+ #define odp_packet_has_ip_mcast __odp_packet_has_ip_mcast
+ #define odp_packet_has_ipfrag __odp_packet_has_ipfrag
+ #define odp_packet_has_ipopt __odp_packet_has_ipopt
+ #define odp_packet_has_udp __odp_packet_has_udp
+ #define odp_packet_has_tcp __odp_packet_has_tcp
+ #define odp_packet_has_sctp __odp_packet_has_sctp
+ #define odp_packet_has_icmp __odp_packet_has_icmp
+ #define odp_packet_has_error __odp_packet_has_error
+ #define odp_packet_has_l2_error __odp_packet_has_l2_error
+ #define odp_packet_has_l3_error __odp_packet_has_l3_error
+ #define odp_packet_has_l4_error __odp_packet_has_l4_error
#else
#undef _ODP_INLINE
#define _ODP_INLINE
@@ -52,6 +73,22 @@ _ODP_INLINE int odp_packet_has_l2(odp_packet_t pkt)
return flags.l2;
}
+_ODP_INLINE int odp_packet_has_l3(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.l3;
+}
+
+_ODP_INLINE int odp_packet_has_l4(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.l4;
+}
+
_ODP_INLINE int odp_packet_has_eth(odp_packet_t pkt)
{
_odp_packet_input_flags_t flags;
@@ -96,6 +133,163 @@ _ODP_INLINE int odp_packet_has_ipsec(odp_packet_t pkt)
return flags.ipsec;
}
+_ODP_INLINE int odp_packet_has_eth_bcast(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.eth_bcast;
+}
+
+_ODP_INLINE int odp_packet_has_eth_mcast(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.eth_mcast;
+}
+
+_ODP_INLINE int odp_packet_has_vlan(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.vlan;
+}
+
+_ODP_INLINE int odp_packet_has_vlan_qinq(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.vlan_qinq;
+}
+
+_ODP_INLINE int odp_packet_has_arp(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.arp;
+}
+
+_ODP_INLINE int odp_packet_has_ipv4(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ipv4;
+}
+
+_ODP_INLINE int odp_packet_has_ipv6(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ipv6;
+}
+
+_ODP_INLINE int odp_packet_has_ip_bcast(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ip_bcast;
+}
+
+_ODP_INLINE int odp_packet_has_ip_mcast(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ip_mcast;
+}
+
+_ODP_INLINE int odp_packet_has_ipfrag(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ipfrag;
+}
+
+_ODP_INLINE int odp_packet_has_ipopt(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ipopt;
+}
+
+_ODP_INLINE int odp_packet_has_udp(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.udp;
+}
+
+_ODP_INLINE int odp_packet_has_tcp(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.tcp;
+}
+
+_ODP_INLINE int odp_packet_has_sctp(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.sctp;
+}
+
+_ODP_INLINE int odp_packet_has_icmp(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.icmp;
+}
+
+_ODP_INLINE int odp_packet_has_error(odp_packet_t pkt)
+{
+ _odp_packet_flags_t flags;
+
+ flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags);
+ return flags.all.error != 0;
+}
+
+_ODP_INLINE int odp_packet_has_l2_error(odp_packet_t pkt)
+{
+ _odp_packet_flags_t flags;
+
+ flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags);
+
+ /* L2 parsing is always done by default and hence
+ no additional check is required. */
+ return flags.snap_len_err;
+}
+
+_ODP_INLINE int odp_packet_has_l3_error(odp_packet_t pkt)
+{
+ _odp_packet_flags_t flags;
+
+ flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags);
+
+ return flags.ip_err;
+}
+
+_ODP_INLINE int odp_packet_has_l4_error(odp_packet_t pkt)
+{
+ _odp_packet_flags_t flags;
+
+ flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags);
+
+ return flags.tcp_err | flags.udp_err;
+}
+
/** @endcond */
#ifdef __cplusplus
diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h b/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h
index 2b8e849e6..06e322066 100644
--- a/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h
+++ b/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h
@@ -113,7 +113,7 @@ typedef union {
uint32_t all_flags;
struct {
- uint32_t reserved1: 7;
+ uint32_t reserved1: 6;
/*
* Init flags
@@ -131,6 +131,7 @@ typedef union {
uint32_t l4_chksum: 1; /* L4 chksum override */
uint32_t ts_set: 1; /* Set Tx timestamp */
uint32_t tx_compl: 1; /* Tx completion event requested */
+ uint32_t tx_aging: 1; /* Packet aging at Tx requested */
uint32_t shaper_len_adj: 8; /* Adjustment for traffic mgr */
/*
@@ -148,8 +149,8 @@ typedef union {
/* Flag groups */
struct {
- uint32_t reserved2: 7;
- uint32_t other: 17; /* All other flags */
+ uint32_t reserved2: 6;
+ uint32_t other: 18; /* All other flags */
uint32_t error: 8; /* All error flags */
} all;
diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h
index 5d8ef2e26..9f3f8923e 100644
--- a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h
+++ b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h
@@ -24,7 +24,7 @@ extern "C" {
#include <odp/api/time.h>
#include <odp/api/abi/buffer.h>
-#include <odp/api/abi/event.h>
+#include <odp/api/abi/event_types.h>
#include <odp/api/abi/packet.h>
#include <odp/api/abi/packet_io.h>
@@ -101,6 +101,9 @@ extern "C" {
#define odp_packet_seg_data_len __odp_packet_seg_data_len
#define odp_packet_ref_static __odp_packet_ref_static
#define odp_packet_has_ref __odp_packet_has_ref
+ #define odp_packet_color __odp_packet_color
+ #define odp_packet_drop_eligible __odp_packet_drop_eligible
+ #define odp_packet_shaper_len_adjust __odp_packet_shaper_len_adjust
#else
#undef _ODP_INLINE
#define _ODP_INLINE
@@ -483,6 +486,33 @@ _ODP_INLINE int odp_packet_has_ref(odp_packet_t pkt)
return (rte_mbuf_refcnt_read((struct rte_mbuf *)(pkt)) > 1);
}
+_ODP_INLINE odp_packet_color_t odp_packet_color(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t input_flags;
+
+ input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags);
+
+ return (odp_packet_color_t)input_flags.color;
+}
+
+_ODP_INLINE odp_bool_t odp_packet_drop_eligible(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t input_flags;
+
+ input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags);
+
+ return !input_flags.nodrop;
+}
+
+_ODP_INLINE int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt)
+{
+ _odp_packet_flags_t flags;
+
+ flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags);
+
+ return (int8_t)flags.shaper_len_adj;
+}
+
#ifdef __cplusplus
}
#endif
diff --git a/platform/linux-dpdk/include/odp/api/plat/time_inlines.h b/platform/linux-dpdk/include/odp/api/plat/time_inlines.h
index c007c5ce9..0ab2b37c0 100644
--- a/platform/linux-dpdk/include/odp/api/plat/time_inlines.h
+++ b/platform/linux-dpdk/include/odp/api/plat/time_inlines.h
@@ -1,5 +1,5 @@
/* Copyright (c) 2018, Linaro Limited
- * Copyright (c) 2020-2021, Nokia
+ * Copyright (c) 2020-2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -123,6 +123,7 @@ static inline uint64_t _odp_time_convert_to_ns(odp_time_t time)
#define odp_time_cmp __odp_time_cmp
#define odp_time_diff __odp_time_diff
+ #define odp_time_diff_ns __odp_time_diff_ns
#define odp_time_sum __odp_time_sum
#else
@@ -194,6 +195,15 @@ _ODP_INLINE odp_time_t odp_time_diff(odp_time_t t2, odp_time_t t1)
return time;
}
+_ODP_INLINE uint64_t odp_time_diff_ns(odp_time_t t2, odp_time_t t1)
+{
+ odp_time_t time;
+
+ time.u64 = t2.u64 - t1.u64;
+
+ return odp_time_to_ns(time);
+}
+
_ODP_INLINE odp_time_t odp_time_sum(odp_time_t t1, odp_time_t t2)
{
odp_time_t time;
diff --git a/platform/linux-dpdk/include/odp/api/plat/timer_inline_types.h b/platform/linux-dpdk/include/odp/api/plat/timer_inline_types.h
new file mode 100644
index 000000000..e3397c4df
--- /dev/null
+++ b/platform/linux-dpdk/include/odp/api/plat/timer_inline_types.h
@@ -0,0 +1,37 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_PLAT_TIMER_INLINE_TYPES_H_
+#define ODP_PLAT_TIMER_INLINE_TYPES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+/** @cond _ODP_HIDE_FROM_DOXYGEN_ */
+
+/* Timeout header field accessor */
+#define _odp_timeout_hdr_field(hdr, cast, field) \
+ (*(cast *)(uintptr_t)((uint8_t *)hdr + \
+ _odp_timeout_inline_offset.field))
+
+/* Timeout header field offsets for inline functions */
+typedef struct _odp_timeout_inline_offset_t {
+ uint16_t expiration;
+ uint16_t timer;
+ uint16_t user_ptr;
+
+} _odp_timeout_inline_offset_t;
+
+/** @endcond */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h b/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h
new file mode 100644
index 000000000..270a6769b
--- /dev/null
+++ b/platform/linux-dpdk/include/odp/api/plat/timer_inlines.h
@@ -0,0 +1,60 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_PLAT_TIMER_INLINES_H_
+#define ODP_PLAT_TIMER_INLINES_H_
+
+#include <odp/api/event_types.h>
+#include <odp/api/timer_types.h>
+
+#include <odp/api/plat/timer_inline_types.h>
+
+#include <stdint.h>
+
+/** @cond _ODP_HIDE_FROM_DOXYGEN_ */
+
+extern const _odp_timeout_inline_offset_t _odp_timeout_inline_offset;
+
+#ifndef _ODP_NO_INLINE
+ /* Inline functions by default */
+ #define _ODP_INLINE static inline
+ #define odp_timeout_timer __odp_timeout_timer
+ #define odp_timeout_tick __odp_timeout_tick
+ #define odp_timeout_user_ptr __odp_timeout_user_ptr
+ #define odp_timeout_from_event __odp_timeout_from_event
+ #define odp_timeout_to_event __odp_timeout_to_event
+#else
+ #define _ODP_INLINE
+#endif
+
+_ODP_INLINE odp_timer_t odp_timeout_timer(odp_timeout_t tmo)
+{
+ return _odp_timeout_hdr_field(tmo, odp_timer_t, timer);
+}
+
+_ODP_INLINE uint64_t odp_timeout_tick(odp_timeout_t tmo)
+{
+ return _odp_timeout_hdr_field(tmo, uint64_t, expiration);
+}
+
+_ODP_INLINE void *odp_timeout_user_ptr(odp_timeout_t tmo)
+{
+ return _odp_timeout_hdr_field(tmo, void *, user_ptr);
+}
+
+_ODP_INLINE odp_timeout_t odp_timeout_from_event(odp_event_t ev)
+{
+ return (odp_timeout_t)ev;
+}
+
+_ODP_INLINE odp_event_t odp_timeout_to_event(odp_timeout_t tmo)
+{
+ return (odp_event_t)tmo;
+}
+
+/** @endcond */
+
+#endif
diff --git a/platform/linux-dpdk/include/odp_packet_dpdk.h b/platform/linux-dpdk/include/odp_packet_dpdk.h
deleted file mode 100644
index 7bb431fd3..000000000
--- a/platform/linux-dpdk/include/odp_packet_dpdk.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/* Copyright (c) 2018, Linaro Limited
- * Copyright (c) 2019, Nokia
- * All rights reserved.
- *
- * SPDX-License-Identifier: BSD-3-Clause
- */
-
-#ifndef ODP_PACKET_DPDK_H_
-#define ODP_PACKET_DPDK_H_
-
-#include <odp/api/packet.h>
-#include <odp/api/packet_io.h>
-
-#include <odp/api/plat/packet_inlines.h>
-
-#include <odp_packet_internal.h>
-
-#include <stdint.h>
-
-/* Flags for pkt_dpdk_t.supported_ptypes */
-#define PTYPE_VLAN 0x01
-#define PTYPE_VLAN_QINQ 0x02
-#define PTYPE_ARP 0x04
-#define PTYPE_IPV4 0x08
-#define PTYPE_IPV6 0x10
-#define PTYPE_UDP 0x20
-#define PTYPE_TCP 0x40
-
-/** Packet parser using DPDK interface */
-int _odp_dpdk_packet_parse_common(packet_parser_t *prs,
- const uint8_t *ptr,
- uint32_t pkt_len,
- uint32_t seg_len,
- struct rte_mbuf *mbuf,
- int layer,
- uint32_t supported_ptypes,
- odp_pktin_config_opt_t pktin_cfg);
-
-static inline int _odp_dpdk_packet_parse_layer(odp_packet_hdr_t *pkt_hdr,
- struct rte_mbuf *mbuf,
- odp_proto_layer_t layer,
- uint32_t supported_ptypes,
- odp_pktin_config_opt_t pktin_cfg)
-{
- odp_packet_t pkt = packet_handle(pkt_hdr);
- uint32_t seg_len = odp_packet_seg_len(pkt);
- uint32_t len = odp_packet_len(pkt);
- uint8_t *base = odp_packet_data(pkt);
-
- return _odp_dpdk_packet_parse_common(&pkt_hdr->p, base, len, seg_len,
- mbuf, layer, supported_ptypes,
- pktin_cfg);
-}
-
-#endif
diff --git a/platform/linux-dpdk/include/odp_packet_internal.h b/platform/linux-dpdk/include/odp_packet_internal.h
index 0af8f7d95..a381386fc 100644
--- a/platform/linux-dpdk/include/odp_packet_internal.h
+++ b/platform/linux-dpdk/include/odp_packet_internal.h
@@ -47,15 +47,24 @@ extern "C" {
#undef vector
#endif
-/** Minimum segment length expected by _odp_packet_parse_common() */
-#define PACKET_PARSE_SEG_LEN 96
-
ODP_STATIC_ASSERT(sizeof(_odp_packet_input_flags_t) == sizeof(uint64_t),
"INPUT_FLAGS_SIZE_ERROR");
ODP_STATIC_ASSERT(sizeof(_odp_packet_flags_t) == sizeof(uint32_t),
"PACKET_FLAGS_SIZE_ERROR");
+/* Maximum number of segments per packet */
+#define PKT_MAX_SEGS 60
+
+ODP_STATIC_ASSERT(PKT_MAX_SEGS < 256, "Maximum of 255 segments supported");
+
+ODP_STATIC_ASSERT(CONFIG_PACKET_SEG_LEN_MIN >= 256, "Segment size must be a minimum of 256 bytes");
+
+ODP_STATIC_ASSERT(CONFIG_PACKET_MAX_SEG_LEN <= UINT16_MAX, "Segment size must fit in uint16_t");
+
+/* We can't enforce tailroom reservation for received packets */
+ODP_STATIC_ASSERT(CONFIG_PACKET_TAILROOM == 0, "Tailroom has to be 0, DPDK doesn't support this");
+
/**
* Packet output flags
*/
@@ -98,16 +107,6 @@ typedef struct {
uint16_t l4_offset;
} packet_parser_t;
-/* Maximum number of segments per packet */
-#define PKT_MAX_SEGS 60
-
-ODP_STATIC_ASSERT(PKT_MAX_SEGS < 256, "Maximum of 255 segments supported");
-
-ODP_STATIC_ASSERT(CONFIG_PACKET_SEG_LEN_MIN >= 256,
- "ODP Segment size must be a minimum of 256 bytes");
-ODP_STATIC_ASSERT(CONFIG_PACKET_MAX_SEG_LEN <= UINT16_MAX,
- "ODP Segment size must fit in uint16_t");
-
/**
* Internal Packet header
*
@@ -148,6 +147,10 @@ typedef struct odp_packet_hdr_t {
/* Max payload size in a LSO segment */
uint16_t lso_max_payload;
+ /* Packet aging drop timeout before enqueue. Once enqueued holds the maximum age (time of
+ * request + requested drop timeout). */
+ uint64_t tx_aging_ns;
+
/* LSO profile index */
uint8_t lso_profile_idx;
@@ -339,17 +342,11 @@ static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len)
rte_pktmbuf_pkt_len(&pkt_hdr->event_hdr.mb) = len;
}
-/* Perform packet parse up to a given protocol layer */
-int _odp_packet_parse_layer(odp_packet_hdr_t *pkt_hdr,
- odp_proto_layer_t layer,
- odp_proto_chksums_t chksums);
-
/* Reset parser metadata for a new parse */
static inline void packet_parse_reset(odp_packet_hdr_t *pkt_hdr, int all)
{
/* Reset parser metadata before new parse */
pkt_hdr->p.input_flags.all = 0;
- pkt_hdr->p.flags.all.error = 0;
pkt_hdr->p.l2_offset = ODP_PACKET_OFFSET_INVALID;
pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID;
pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID;
@@ -388,10 +385,6 @@ static inline void packet_set_ts(odp_packet_hdr_t *pkt_hdr, odp_time_t *ts)
}
}
-int _odp_packet_parse_common(packet_parser_t *pkt_hdr, const uint8_t *ptr,
- uint32_t pkt_len, uint32_t seg_len, int layer,
- odp_proto_chksums_t chksums);
-
int _odp_packet_set_data(odp_packet_t pkt, uint32_t offset,
uint8_t c, uint32_t len);
@@ -403,9 +396,8 @@ int _odp_packet_tcp_chksum_insert(odp_packet_t pkt);
int _odp_packet_udp_chksum_insert(odp_packet_t pkt);
int _odp_packet_sctp_chksum_insert(odp_packet_t pkt);
-/* We can't enforce tailroom reservation for received packets */
-ODP_STATIC_ASSERT(CONFIG_PACKET_TAILROOM == 0,
- "ERROR: Tailroom has to be 0, DPDK doesn't support this");
+int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
+ odp_proto_chksums_t chksums, uint64_t l4_part_sum);
#ifdef __cplusplus
}
diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h b/platform/linux-dpdk/include/odp_packet_io_internal.h
index 9ef95e7e4..8e06703fb 100644
--- a/platform/linux-dpdk/include/odp_packet_io_internal.h
+++ b/platform/linux-dpdk/include/odp_packet_io_internal.h
@@ -60,6 +60,7 @@ struct pktio_entry {
/* These two locks together lock the whole pktio device */
odp_ticketlock_t rxl; /**< RX ticketlock */
odp_ticketlock_t txl; /**< TX ticketlock */
+ odp_proto_layer_t parse_layer;
uint16_t pktin_frame_offset;
struct {
@@ -75,6 +76,8 @@ struct pktio_entry {
uint8_t tx_ts : 1;
/* Tx completion events */
uint8_t tx_compl : 1;
+ /* Packet aging */
+ uint8_t tx_aging : 1;
};
};
} enabled;
@@ -125,8 +128,8 @@ struct pktio_entry {
/* Storage for queue handles
* Multi-queue support is pktio driver specific */
- unsigned num_in_queue;
- unsigned num_out_queue;
+ uint32_t num_in_queue;
+ uint32_t num_out_queue;
struct {
odp_queue_t queue;
@@ -208,8 +211,8 @@ typedef struct pktio_if_ops {
int num);
int (*recv_tmo)(pktio_entry_t *entry, int index, odp_packet_t packets[],
int num, uint64_t wait_usecs);
- int (*recv_mq_tmo)(pktio_entry_t *entry[], int index[], int num_q,
- odp_packet_t packets[], int num, unsigned *from,
+ int (*recv_mq_tmo)(pktio_entry_t *entry[], int index[], uint32_t num_q,
+ odp_packet_t packets[], int num, uint32_t *from,
uint64_t wait_usecs);
int (*fd_set)(pktio_entry_t *entry, int index, fd_set *readfds);
int (*send)(pktio_entry_t *entry, int index,
@@ -276,6 +279,11 @@ static inline int _odp_pktio_tx_compl_enabled(const pktio_entry_t *entry)
return entry->s.enabled.tx_compl;
}
+static inline int _odp_pktio_tx_aging_enabled(pktio_entry_t *entry)
+{
+ return entry->s.enabled.tx_aging;
+}
+
static inline void _odp_pktio_tx_ts_set(pktio_entry_t *entry)
{
odp_time_t ts_val = odp_time_global();
@@ -291,8 +299,8 @@ extern const pktio_if_ops_t * const _odp_pktio_if_ops[];
/* Dummy function required by odp_pktin_recv_mq_tmo() */
static inline int
_odp_sock_recv_mq_tmo_try_int_driven(const struct odp_pktin_queue_t queues[],
- unsigned int num_q ODP_UNUSED,
- unsigned int *from ODP_UNUSED,
+ uint32_t num_q ODP_UNUSED,
+ uint32_t *from ODP_UNUSED,
odp_packet_t packets[] ODP_UNUSED,
int num ODP_UNUSED,
uint64_t usecs ODP_UNUSED,
diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c
index da2714077..21c9f00ee 100644
--- a/platform/linux-dpdk/odp_crypto.c
+++ b/platform/linux-dpdk/odp_crypto.c
@@ -1541,6 +1541,7 @@ int _odp_crypto_term_global(void)
return rc;
}
+#if ODP_DEPRECATED_API
odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev)
{
/* This check not mandated by the API specification */
@@ -1575,6 +1576,7 @@ uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl)
{
return _odp_pri(hdl);
}
+#endif
void odp_crypto_session_param_init(odp_crypto_session_param_t *param)
{
@@ -2001,6 +2003,7 @@ err:
return -1;
}
+#if ODP_DEPRECATED_API
int odp_crypto_operation(odp_crypto_op_param_t *param,
odp_bool_t *posted,
odp_crypto_op_result_t *result)
@@ -2054,6 +2057,7 @@ int odp_crypto_operation(odp_crypto_op_param_t *param,
return 0;
}
+#endif
int odp_crypto_op(const odp_packet_t pkt_in[],
odp_packet_t pkt_out[],
diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c
index 0963bd570..1af9bf0c5 100644
--- a/platform/linux-dpdk/odp_packet.c
+++ b/platform/linux-dpdk/odp_packet.c
@@ -28,6 +28,9 @@
#include <odp_macros_internal.h>
#include <odp_packet_internal.h>
#include <odp_packet_io_internal.h>
+#include <odp_parse_internal.h>
+
+#include <rte_version.h>
#include <protocols/eth.h>
#include <protocols/ip.h>
@@ -40,6 +43,10 @@
#include <stddef.h>
#include <inttypes.h>
+#if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0)
+ #define RTE_MBUF_F_RX_RSS_HASH PKT_RX_RSS_HASH
+#endif
+
#include <odp/visibility_begin.h>
/* Fill in packet header field offsets for inline functions */
@@ -64,7 +71,7 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = {
.user_area = offsetof(odp_packet_hdr_t, uarea_addr),
.rss = offsetof(odp_packet_hdr_t, event_hdr.mb.hash.rss),
.ol_flags = offsetof(odp_packet_hdr_t, event_hdr.mb.ol_flags),
- .rss_flag = PKT_RX_RSS_HASH
+ .rss_flag = RTE_MBUF_F_RX_RSS_HASH
};
#include <odp/visibility_end.h>
@@ -1291,500 +1298,6 @@ static uint32_t packet_sum_crc32c(odp_packet_hdr_t *pkt_hdr,
return sum;
}
-/*
- * In the worst case we look at the Ethernet header, 8 bytes of LLC/SNAP
- * header and two VLAN tags in the same packet.
- */
-#define PARSE_ETH_BYTES (sizeof(_odp_ethhdr_t) + 8 + 2 * sizeof(_odp_vlanhdr_t))
-/** Parser helper function for Ethernet packets
- *
- * Requires up to PARSE_ETH_BYTES bytes of contiguous packet data.
- */
-static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr,
- uint32_t *offset, uint32_t frame_len)
-{
- uint16_t ethtype;
- const _odp_ethhdr_t *eth;
- uint16_t macaddr0, macaddr2, macaddr4;
- const _odp_vlanhdr_t *vlan;
- _odp_packet_input_flags_t input_flags;
-
- input_flags.all = 0;
- input_flags.l2 = 1;
- input_flags.eth = 1;
-
- eth = (const _odp_ethhdr_t *)*parseptr;
-
- /* Detect jumbo frames */
- if (odp_unlikely(frame_len - *offset > _ODP_ETH_LEN_MAX))
- input_flags.jumbo = 1;
-
- /* Handle Ethernet broadcast/multicast addresses */
- macaddr0 = odp_be_to_cpu_16(*((const uint16_t *)(const void *)eth));
- if (odp_unlikely((macaddr0 & 0x0100) == 0x0100))
- input_flags.eth_mcast = 1;
-
- if (odp_unlikely(macaddr0 == 0xffff)) {
- macaddr2 =
- odp_be_to_cpu_16(*((const uint16_t *)
- (const void *)eth + 1));
- macaddr4 =
- odp_be_to_cpu_16(*((const uint16_t *)
- (const void *)eth + 2));
-
- if ((macaddr2 == 0xffff) && (macaddr4 == 0xffff))
- input_flags.eth_bcast = 1;
- }
-
- /* Get Ethertype */
- ethtype = odp_be_to_cpu_16(eth->type);
- *offset += sizeof(*eth);
- *parseptr += sizeof(*eth);
-
- /* Check for SNAP vs. DIX */
- if (odp_unlikely(ethtype < _ODP_ETH_LEN_MAX)) {
- input_flags.snap = 1;
- if (ethtype > frame_len - *offset) {
- prs->flags.snap_len_err = 1;
- ethtype = 0;
- goto error;
- }
- ethtype = odp_be_to_cpu_16(*((const uint16_t *)(uintptr_t)
- (*parseptr + 6)));
- *offset += 8;
- *parseptr += 8;
- }
-
- /* Parse the VLAN header(s), if present */
- if (odp_unlikely(ethtype == _ODP_ETHTYPE_VLAN_OUTER)) {
- input_flags.vlan_qinq = 1;
- input_flags.vlan = 1;
-
- vlan = (const _odp_vlanhdr_t *)*parseptr;
- ethtype = odp_be_to_cpu_16(vlan->type);
- *offset += sizeof(_odp_vlanhdr_t);
- *parseptr += sizeof(_odp_vlanhdr_t);
- }
-
- if (ethtype == _ODP_ETHTYPE_VLAN) {
- input_flags.vlan = 1;
- vlan = (const _odp_vlanhdr_t *)*parseptr;
- ethtype = odp_be_to_cpu_16(vlan->type);
- *offset += sizeof(_odp_vlanhdr_t);
- *parseptr += sizeof(_odp_vlanhdr_t);
- }
-
- /*
- * The packet was too short for what we parsed. We just give up
- * entirely without trying to parse what fits in the packet.
- */
- if (odp_unlikely(*offset > frame_len)) {
- input_flags.all = 0;
- input_flags.l2 = 1;
- ethtype = 0;
- }
-
-error:
- prs->input_flags.all |= input_flags.all;
-
- return ethtype;
-}
-
-#define PARSE_IPV4_BYTES (0xfU * 4) /* max IPv4 header length with options */
-/**
- * Parser helper function for IPv4
- *
- * Requires up to PARSE_IPV4_BYTES bytes of contiguous packet data.
- */
-static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr,
- uint32_t *offset, uint32_t frame_len,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- const _odp_ipv4hdr_t *ipv4 = (const _odp_ipv4hdr_t *)*parseptr;
- uint32_t dstaddr = odp_be_to_cpu_32(ipv4->dst_addr);
- uint32_t l3_len = odp_be_to_cpu_16(ipv4->tot_len);
- uint16_t frag_offset = odp_be_to_cpu_16(ipv4->frag_offset);
- uint8_t ver = _ODP_IPV4HDR_VER(ipv4->ver_ihl);
- uint8_t ihl = _ODP_IPV4HDR_IHL(ipv4->ver_ihl);
-
- if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN ||
- ver != 4 ||
- sizeof(*ipv4) > frame_len - *offset ||
- (l3_len > frame_len - *offset))) {
- prs->flags.ip_err = 1;
- return 0;
- }
-
- if (chksums.chksum.ipv4) {
- prs->input_flags.l3_chksum_done = 1;
- if (chksum_finalize(chksum_partial(ipv4, ihl * 4, 0)) != 0xffff) {
- prs->flags.ip_err = 1;
- prs->flags.l3_chksum_err = 1;
- return 0;
- }
- }
-
- *offset += ihl * 4;
- *parseptr += ihl * 4;
-
- if (chksums.chksum.udp || chksums.chksum.tcp)
- *l4_part_sum = chksum_partial((const uint8_t *)&ipv4->src_addr,
- 2 * _ODP_IPV4ADDR_LEN, 0);
-
- if (odp_unlikely(ihl > _ODP_IPV4HDR_IHL_MIN))
- prs->input_flags.ipopt = 1;
-
- /* A packet is a fragment if:
- * "more fragments" flag is set (all fragments except the last)
- * OR
- * "fragment offset" field is nonzero (all fragments except the first)
- */
- if (odp_unlikely(_ODP_IPV4HDR_IS_FRAGMENT(frag_offset)))
- prs->input_flags.ipfrag = 1;
-
- /* Handle IPv4 broadcast / multicast */
- if (odp_unlikely(dstaddr == 0xffffffff))
- prs->input_flags.ip_bcast = 1;
-
- if (odp_unlikely((dstaddr >> 28) == 0xe))
- prs->input_flags.ip_mcast = 1;
-
- return ipv4->proto;
-}
-
-/*
- * Peeks 2 bytes beyond IPv6 base header without length check if there
- * are extension headers.
- */
-#define PARSE_IPV6_BYTES (sizeof(_odp_ipv6hdr_t) + 2)
-/**
- * Parser helper function for IPv6
- *
- * Requires at least PARSE_IPV6_BYTES bytes of contiguous packet data.
- */
-static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr,
- uint32_t *offset, uint32_t frame_len,
- uint32_t seg_len,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- const _odp_ipv6hdr_t *ipv6 = (const _odp_ipv6hdr_t *)*parseptr;
- const _odp_ipv6hdr_ext_t *ipv6ext;
- uint32_t dstaddr0 = odp_be_to_cpu_32(ipv6->dst_addr.u8[0]);
- uint32_t l3_len = odp_be_to_cpu_16(ipv6->payload_len) +
- _ODP_IPV6HDR_LEN;
-
- /* Basic sanity checks on IPv6 header */
- if (odp_unlikely((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 ||
- sizeof(*ipv6) > frame_len - *offset ||
- l3_len > frame_len - *offset)) {
- prs->flags.ip_err = 1;
- return 0;
- }
-
- /* IPv6 broadcast / multicast flags */
- prs->input_flags.ip_mcast = (dstaddr0 & 0xff000000) == 0xff000000;
- prs->input_flags.ip_bcast = 0;
-
- /* Skip past IPv6 header */
- *offset += sizeof(_odp_ipv6hdr_t);
- *parseptr += sizeof(_odp_ipv6hdr_t);
-
- if (chksums.chksum.udp || chksums.chksum.tcp)
- *l4_part_sum = chksum_partial((const uint8_t *)&ipv6->src_addr,
- 2 * _ODP_IPV6ADDR_LEN, 0);
-
- /* Skip past any IPv6 extension headers */
- if (ipv6->next_hdr == _ODP_IPPROTO_HOPOPTS ||
- ipv6->next_hdr == _ODP_IPPROTO_ROUTE) {
- prs->input_flags.ipopt = 1;
-
- do {
- ipv6ext = (const _odp_ipv6hdr_ext_t *)*parseptr;
- uint16_t extlen = 8 + ipv6ext->ext_len * 8;
-
- *offset += extlen;
- *parseptr += extlen;
- } while ((ipv6ext->next_hdr == _ODP_IPPROTO_HOPOPTS ||
- ipv6ext->next_hdr == _ODP_IPPROTO_ROUTE) &&
- *offset < seg_len);
-
- if (*offset >= prs->l3_offset +
- odp_be_to_cpu_16(ipv6->payload_len)) {
- prs->flags.ip_err = 1;
- return 0;
- }
-
- if (ipv6ext->next_hdr == _ODP_IPPROTO_FRAG)
- prs->input_flags.ipfrag = 1;
-
- return ipv6ext->next_hdr;
- }
-
- if (odp_unlikely(ipv6->next_hdr == _ODP_IPPROTO_FRAG)) {
- prs->input_flags.ipopt = 1;
- prs->input_flags.ipfrag = 1;
- }
-
- return ipv6->next_hdr;
-}
-
-#define PARSE_TCP_BYTES (sizeof(_odp_tcphdr_t))
-/**
- * Parser helper function for TCP
- *
- * Requires PARSE_TCP_BYTES bytes of contiguous packet data.
- */
-static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr,
- uint16_t tcp_len,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- const _odp_tcphdr_t *tcp = (const _odp_tcphdr_t *)*parseptr;
- uint32_t len = tcp->hl * 4;
-
- if (odp_unlikely(tcp->hl < sizeof(_odp_tcphdr_t) / sizeof(uint32_t)))
- prs->flags.tcp_err = 1;
-
- if (chksums.chksum.tcp &&
- !prs->input_flags.ipfrag) {
- *l4_part_sum += odp_cpu_to_be_16(tcp_len);
-#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
- *l4_part_sum += _ODP_IPPROTO_TCP;
-#else
- *l4_part_sum += _ODP_IPPROTO_TCP << 8;
-#endif
- }
-
- *parseptr += len;
-}
-
-/*
- * In the worst case we look at the UDP header and 4 bytes of the UDP
- * payload (the non-ESP marker to distinguish IKE packets from ESP packets).
- */
-#define PARSE_UDP_BYTES (sizeof(_odp_udphdr_t) + 4)
-/**
- * Parser helper function for UDP
- *
- * Requires PARSE_UDP_BYTES bytes of contiguous packet data.
- */
-static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr;
- uint32_t udplen = odp_be_to_cpu_16(udp->length);
- uint16_t ipsec_port = odp_cpu_to_be_16(_ODP_UDP_IPSEC_PORT);
-
- if (odp_unlikely(udplen < sizeof(_odp_udphdr_t))) {
- prs->flags.udp_err = 1;
- return;
- }
-
- if (chksums.chksum.udp &&
- !prs->input_flags.ipfrag) {
- if (udp->chksum == 0) {
- prs->input_flags.l4_chksum_done = 1;
- prs->flags.l4_chksum_err =
- (prs->input_flags.ipv4 != 1);
- } else {
- *l4_part_sum += udp->length;
-#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
- *l4_part_sum += _ODP_IPPROTO_UDP;
-#else
- *l4_part_sum += _ODP_IPPROTO_UDP << 8;
-#endif
- }
- prs->input_flags.udp_chksum_zero = (udp->chksum == 0);
- }
-
- if (odp_unlikely(ipsec_port == udp->dst_port && udplen > 4)) {
- uint32_t val;
-
- memcpy(&val, udp + 1, 4);
- if (val != 0) {
- prs->input_flags.ipsec = 1;
- prs->input_flags.ipsec_udp = 1;
- }
- }
-
- *parseptr += sizeof(_odp_udphdr_t);
-}
-
-#define PARSE_SCTP_BYTES (sizeof(_odp_sctphdr_t))
-/**
- * Parser helper function for SCTP
- *
- * Requires PARSE_SCTP_BYTES bytes of contiguous packet data.
- */
-static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr,
- uint16_t sctp_len,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- if (odp_unlikely(sctp_len < sizeof(_odp_sctphdr_t))) {
- prs->flags.sctp_err = 1;
- return;
- }
-
- if (chksums.chksum.sctp &&
- !prs->input_flags.ipfrag) {
- const _odp_sctphdr_t *sctp =
- (const _odp_sctphdr_t *)*parseptr;
- uint32_t crc = ~0;
- uint32_t zero = 0;
-
- crc = odp_hash_crc32c(sctp, sizeof(*sctp) - 4, crc);
- crc = odp_hash_crc32c(&zero, 4, crc);
- *l4_part_sum = crc;
- }
-
- *parseptr += sizeof(_odp_sctphdr_t);
-}
-
-#define MAX3(a, b, c) (MAX(MAX((a), (b)), (c)))
-#define PARSE_L3_L4_BYTES (MAX(PARSE_IPV4_BYTES, PARSE_IPV6_BYTES) + \
- MAX3(PARSE_TCP_BYTES, PARSE_UDP_BYTES, PARSE_SCTP_BYTES))
-/* Requires up to PARSE_L3_L4_BYTES bytes of contiguous packet data. */
-static inline
-int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr,
- uint32_t offset,
- uint32_t frame_len, uint32_t seg_len,
- int layer, uint16_t ethtype,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- uint8_t ip_proto;
-
- prs->l3_offset = offset;
-
- if (odp_unlikely(layer <= ODP_PROTO_LAYER_L2))
- return prs->flags.all.error != 0;
-
- /* Set l3 flag only for known ethtypes */
- prs->input_flags.l3 = 1;
-
- /* Parse Layer 3 headers */
- switch (ethtype) {
- case _ODP_ETHTYPE_IPV4:
- prs->input_flags.ipv4 = 1;
- ip_proto = parse_ipv4(prs, &parseptr, &offset, frame_len,
- chksums, l4_part_sum);
- prs->l4_offset = offset;
- break;
-
- case _ODP_ETHTYPE_IPV6:
- prs->input_flags.ipv6 = 1;
- ip_proto = parse_ipv6(prs, &parseptr, &offset, frame_len,
- seg_len, chksums, l4_part_sum);
- prs->l4_offset = offset;
- break;
-
- case _ODP_ETHTYPE_ARP:
- prs->input_flags.arp = 1;
- ip_proto = 255; /* Reserved invalid by IANA */
- break;
-
- default:
- prs->input_flags.l3 = 0;
- ip_proto = 255; /* Reserved invalid by IANA */
- }
-
- if (layer == ODP_PROTO_LAYER_L3)
- return prs->flags.all.error != 0;
-
- /* Set l4 flag only for known ip_proto */
- prs->input_flags.l4 = 1;
-
- /* Parse Layer 4 headers */
- switch (ip_proto) {
- case _ODP_IPPROTO_ICMPV4:
- /* Fall through */
-
- case _ODP_IPPROTO_ICMPV6:
- prs->input_flags.icmp = 1;
- break;
-
- case _ODP_IPPROTO_IPIP:
- /* Do nothing */
- break;
-
- case _ODP_IPPROTO_TCP:
- if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len))
- return -1;
- prs->input_flags.tcp = 1;
- parse_tcp(prs, &parseptr, frame_len - prs->l4_offset, chksums,
- l4_part_sum);
- break;
-
- case _ODP_IPPROTO_UDP:
- if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len))
- return -1;
- prs->input_flags.udp = 1;
- parse_udp(prs, &parseptr, chksums, l4_part_sum);
- break;
-
- case _ODP_IPPROTO_AH:
- prs->input_flags.ipsec = 1;
- prs->input_flags.ipsec_ah = 1;
- break;
-
- case _ODP_IPPROTO_ESP:
- prs->input_flags.ipsec = 1;
- prs->input_flags.ipsec_esp = 1;
- break;
-
- case _ODP_IPPROTO_SCTP:
- prs->input_flags.sctp = 1;
- parse_sctp(prs, &parseptr, frame_len - prs->l4_offset, chksums,
- l4_part_sum);
- break;
-
- case _ODP_IPPROTO_NO_NEXT:
- prs->input_flags.no_next_hdr = 1;
- break;
-
- default:
- prs->input_flags.l4 = 0;
- break;
- }
-
- return prs->flags.all.error != 0;
-}
-
-/**
- * Parse common packet headers up to given layer
- *
- * The function expects at least PACKET_PARSE_SEG_LEN bytes of data to be
- * available from the ptr. Also parse metadata must be already initialized.
- */
-int _odp_packet_parse_common(packet_parser_t *prs, const uint8_t *ptr,
- uint32_t frame_len, uint32_t seg_len,
- int layer, odp_proto_chksums_t chksums)
-{
- uint32_t offset;
- uint16_t ethtype;
- const uint8_t *parseptr;
- uint64_t l4_part_sum;
-
- parseptr = ptr;
- offset = 0;
-
- if (odp_unlikely(layer == ODP_PROTO_LAYER_NONE))
- return 0;
-
- /* Assume valid L2 header, no CRC/FCS check in SW */
- prs->l2_offset = offset;
-
- ethtype = parse_eth(prs, &parseptr, &offset, frame_len);
-
- return packet_parse_common_l3_l4(prs, parseptr, offset, frame_len,
- seg_len, layer, ethtype, chksums,
- &l4_part_sum);
-}
-
static inline int packet_ipv4_chksum(odp_packet_t pkt,
uint32_t offset,
_odp_ipv4hdr_t *ip,
@@ -1853,7 +1366,7 @@ static int _odp_packet_tcp_udp_chksum_insert(odp_packet_t pkt, uint16_t proto)
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
uint32_t zero = 0;
uint64_t sum;
- uint16_t l3_ver;
+ uint16_t l3_ver = 0; /* GCC */
uint16_t chksum;
uint32_t chksum_offset;
uint32_t frame_len = odp_packet_len(pkt);
@@ -1964,9 +1477,8 @@ int _odp_packet_sctp_chksum_insert(odp_packet_t pkt)
return odp_packet_copy_from_mem(pkt, pkt_hdr->p.l4_offset + 8, 4, &sum);
}
-static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
- odp_proto_chksums_t chksums,
- uint64_t l4_part_sum)
+int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, odp_proto_chksums_t chksums,
+ uint64_t l4_part_sum)
{
uint32_t frame_len = odp_packet_len(packet_handle(pkt_hdr));
@@ -2042,44 +1554,6 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
return pkt_hdr->p.flags.all.error != 0;
}
-/**
- * Simple packet parser
- */
-int _odp_packet_parse_layer(odp_packet_hdr_t *pkt_hdr,
- odp_proto_layer_t layer,
- odp_proto_chksums_t chksums)
-{
- odp_packet_t pkt = packet_handle(pkt_hdr);
- uint32_t seg_len = odp_packet_seg_len(pkt);
- uint32_t len = odp_packet_len(pkt);
- const uint8_t *base = odp_packet_data(pkt);
- uint32_t offset = 0;
- uint16_t ethtype;
- uint64_t l4_part_sum = 0;
- int rc;
-
- if (odp_unlikely(layer == ODP_PROTO_LAYER_NONE))
- return 0;
-
- /* Assume valid L2 header, no CRC/FCS check in SW */
- pkt_hdr->p.l2_offset = offset;
-
- ethtype = parse_eth(&pkt_hdr->p, &base, &offset, len);
-
- rc = packet_parse_common_l3_l4(&pkt_hdr->p, base, offset,
- len,
- seg_len, layer, ethtype, chksums,
- &l4_part_sum);
-
- if (rc != 0)
- return rc;
-
- if (layer >= ODP_PROTO_LAYER_L4)
- return packet_l4_chksum(pkt_hdr, chksums, l4_part_sum);
- else
- return 0;
-}
-
int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
const odp_packet_parse_param_t *param)
{
@@ -2094,6 +1568,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
uint64_t l4_part_sum = 0;
const uint32_t min_seglen = PARSE_ETH_BYTES + PARSE_L3_L4_BYTES;
uint8_t buf[min_seglen];
+ odp_pktin_config_opt_t opt;
if (proto == ODP_PROTO_NONE || layer == ODP_PROTO_LAYER_NONE)
return -1;
@@ -2124,7 +1599,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
/* Assume valid L2 header, no CRC/FCS check in SW */
pkt_hdr->p.l2_offset = offset;
- ethtype = parse_eth(&pkt_hdr->p, &data, &offset, packet_len);
+ ethtype = _odp_parse_eth(&pkt_hdr->p, &data, &offset, packet_len);
} else if (proto == ODP_PROTO_IPV4) {
ethtype = _ODP_ETHTYPE_IPV4;
} else if (proto == ODP_PROTO_IPV6) {
@@ -2133,17 +1608,18 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
ethtype = 0; /* Invalid */
}
- ret = packet_parse_common_l3_l4(&pkt_hdr->p, data, offset,
- packet_len, seg_len,
- layer, ethtype,
- param->chksums,
- &l4_part_sum);
+ opt.all_bits = 0;
+
+ ret = _odp_packet_parse_common_l3_l4(&pkt_hdr->p, data, offset,
+ packet_len, seg_len, layer,
+ ethtype, param->chksums,
+ &l4_part_sum, opt);
if (ret)
return -1;
if (layer >= ODP_PROTO_LAYER_L4) {
- ret = packet_l4_chksum(pkt_hdr, param->chksums, l4_part_sum);
+ ret = _odp_packet_l4_chksum(pkt_hdr, param->chksums, l4_part_sum);
if (ret)
return -1;
}
@@ -2390,14 +1866,17 @@ int odp_packet_payload_offset_set(odp_packet_t pkt, uint32_t offset)
void odp_packet_aging_tmo_set(odp_packet_t pkt, uint64_t tmo_ns)
{
- (void)pkt;
- (void)tmo_ns;
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+
+ pkt_hdr->p.flags.tx_aging = tmo_ns ? 1 : 0;
+ pkt_hdr->tx_aging_ns = tmo_ns;
}
uint64_t odp_packet_aging_tmo(odp_packet_t pkt)
{
- (void)pkt;
- return 0;
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+
+ return pkt_hdr->p.flags.tx_aging ? pkt_hdr->tx_aging_ns : 0;
}
int odp_packet_tx_compl_request(odp_packet_t pkt, const odp_packet_tx_compl_opt_t *opt)
diff --git a/platform/linux-dpdk/odp_packet_dpdk.c b/platform/linux-dpdk/odp_packet_dpdk.c
index a74772740..72d2eb890 100644
--- a/platform/linux-dpdk/odp_packet_dpdk.c
+++ b/platform/linux-dpdk/odp_packet_dpdk.c
@@ -1,46 +1,33 @@
/* Copyright (c) 2013-2018, Linaro Limited
- * Copyright (c) 2019-2021, Nokia
+ * Copyright (c) 2019-2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
#include <odp_posix_extensions.h>
-#include <stdio.h>
-#include <errno.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/ioctl.h>
-#include <sys/mman.h>
-#include <poll.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <string.h>
-#include <stdlib.h>
-#include <inttypes.h>
-#include <linux/ethtool.h>
-#include <linux/sockios.h>
-
-#include <odp_align_internal.h>
-#include <odp/api/cpu.h>
#include <odp/api/hints.h>
-#include <odp/api/system_info.h>
+#include <odp/api/packet.h>
+#include <odp/api/packet_io.h>
+#include <odp/api/pool.h>
+#include <odp/api/std_types.h>
+#include <odp/api/ticketlock.h>
+#include <odp/api/time.h>
+
+#include <odp/api/plat/packet_inlines.h>
+#include <odp/api/plat/time_inlines.h>
+
+#include <odp_classification_internal.h>
#include <odp_debug_internal.h>
#include <odp_errno_define.h>
-#include <odp_classification_internal.h>
-#include <odp_packet_io_internal.h>
+#include <odp_eventdev_internal.h>
#include <odp_libconfig_internal.h>
-#include <odp/api/plat/packet_inlines.h>
-#include <odp/api/time.h>
-#include <odp/api/plat/time_inlines.h>
#include <odp_packet_dpdk.h>
-#include <odp_eventdev_internal.h>
+#include <odp_packet_internal.h>
+#include <odp_packet_io_internal.h>
#include <protocols/eth.h>
-#include <net/if.h>
-#include <protocols/udp.h>
-
#include <rte_config.h>
#if defined(__clang__)
#undef RTE_TOOLCHAIN_GCC
@@ -50,6 +37,32 @@
#include <rte_ip_frag.h>
#include <rte_udp.h>
#include <rte_tcp.h>
+#include <rte_version.h>
+
+#include <linux/ethtool.h>
+#include <linux/sockios.h>
+
+#include <errno.h>
+#include <fcntl.h>
+#include <inttypes.h>
+#include <net/if.h>
+#include <poll.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0)
+ #define RTE_MBUF_F_TX_IPV4 PKT_TX_IPV4
+ #define RTE_MBUF_F_TX_IPV6 PKT_TX_IPV6
+ #define RTE_MBUF_F_TX_IP_CKSUM PKT_TX_IP_CKSUM
+ #define RTE_MBUF_F_TX_UDP_CKSUM PKT_TX_UDP_CKSUM
+ #define RTE_MBUF_F_TX_TCP_CKSUM PKT_TX_TCP_CKSUM
+#endif
/* DPDK poll mode drivers requiring minimum RX burst size DPDK_MIN_RX_BURST */
#define IXGBE_DRV_NAME "net_ixgbe"
@@ -293,7 +306,11 @@ static int dpdk_setup_eth_dev(pktio_entry_t *pktio_entry)
/* RX packet len same size as pool segment minus headroom and double
* VLAN tag
*/
+#if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0)
eth_conf.rxmode.max_rx_pkt_len =
+#else
+ eth_conf.rxmode.mtu =
+#endif
rte_pktmbuf_data_room_size(pool->rte_mempool) -
2 * 4 - RTE_PKTMBUF_HEADROOM;
@@ -508,16 +525,18 @@ static int dpdk_init_capability(pktio_entry_t *pktio_entry,
/* Check if setting MTU is supported */
ret = rte_eth_dev_set_mtu(pkt_dpdk->port_id, pkt_dpdk->mtu - _ODP_ETHHDR_LEN);
- if (ret == 0) {
+ /* From DPDK 21.11 onwards, calling rte_eth_dev_set_mtu() before device is configured with
+ * rte_eth_dev_configure() will result in failure. The least hacky (unfortunately still
+ * very hacky) way to continue checking the support is to take into account that the
+ * function will fail earlier with -ENOTSUP if MTU setting is not supported by device than
+ * if the device was not yet configured. */
+ if (ret != -ENOTSUP) {
capa->set_op.op.maxlen = 1;
capa->maxlen.equal = true;
capa->maxlen.min_input = DPDK_MTU_MIN;
capa->maxlen.max_input = pkt_dpdk->mtu_max;
capa->maxlen.min_output = DPDK_MTU_MIN;
capa->maxlen.max_output = pkt_dpdk->mtu_max;
- } else if (ret != -ENOTSUP) {
- ODP_ERR("Failed to set interface MTU: %d\n", ret);
- return -1;
}
ptype_cnt = rte_eth_dev_get_supported_ptypes(pkt_dpdk->port_id,
@@ -624,7 +643,7 @@ static int setup_pkt_dpdk(odp_pktio_t pktio ODP_UNUSED,
uint32_t mtu;
struct rte_eth_dev_info dev_info;
pkt_dpdk_t * const pkt_dpdk = pkt_priv(pktio_entry);
- int i;
+ int i, ret;
uint16_t port_id;
if (!rte_eth_dev_get_port_by_name(netdev, &port_id))
@@ -642,7 +661,11 @@ static int setup_pkt_dpdk(odp_pktio_t pktio ODP_UNUSED,
}
memset(&dev_info, 0, sizeof(struct rte_eth_dev_info));
- rte_eth_dev_info_get(pkt_dpdk->port_id, &dev_info);
+ ret = rte_eth_dev_info_get(pkt_dpdk->port_id, &dev_info);
+ if (ret) {
+ ODP_ERR("Failed to read device info: %d\n", ret);
+ return -1;
+ }
/* Initialize runtime options */
if (init_options(pktio_entry, &dev_info)) {
@@ -903,101 +926,90 @@ int _odp_input_pkts(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], int nu
{
pkt_dpdk_t * const pkt_dpdk = pkt_priv(pktio_entry);
uint16_t i;
+ uint16_t num_pkts = 0;
odp_pktin_config_opt_t pktin_cfg = pktio_entry->s.config.pktin;
- odp_proto_layer_t parse_layer = pktio_entry->s.config.parser.layer;
odp_pktio_t input = pktio_entry->s.handle;
odp_time_t ts_val;
odp_time_t *ts = NULL;
uint16_t num_prefetch = RTE_MIN(num, NUM_RX_PREFETCH);
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
for (i = 0; i < num_prefetch; i++)
prefetch_pkt(pkt_table[i]);
- if (pktio_entry->s.config.pktin.bit.ts_all ||
- pktio_entry->s.config.pktin.bit.ts_ptp) {
+ if (pktin_cfg.bit.ts_all || pktin_cfg.bit.ts_ptp) {
ts_val = odp_time_global();
ts = &ts_val;
}
for (i = 0; i < num; ++i) {
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt_table[i]);
- struct rte_mbuf *mbuf = pkt_to_mbuf(pkt_table[i]);
+ odp_packet_t pkt = pkt_table[i];
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+ struct rte_mbuf *mbuf = pkt_to_mbuf(pkt);
if (odp_likely(i + num_prefetch < num))
prefetch_pkt(pkt_table[i + num_prefetch]);
packet_init(pkt_hdr, input);
- if (!pktio_cls_enabled(pktio_entry) && parse_layer != ODP_PROTO_LAYER_NONE) {
- uint32_t ptypes = pkt_dpdk->supported_ptypes;
-
- if (_odp_dpdk_packet_parse_layer(pkt_hdr, mbuf, parse_layer,
- ptypes, pktin_cfg)) {
- odp_packet_free(pkt_table[i]);
+ if (layer != ODP_PROTO_LAYER_NONE) {
+ if (_odp_dpdk_packet_parse_common(&pkt_hdr->p,
+ rte_pktmbuf_mtod(mbuf, uint8_t *),
+ rte_pktmbuf_pkt_len(mbuf),
+ rte_pktmbuf_data_len(mbuf),
+ mbuf, layer,
+ pkt_dpdk->supported_ptypes, pktin_cfg)) {
+ odp_packet_free(pkt);
continue;
}
+ if (odp_unlikely(num_pkts != i))
+ pkt_table[num_pkts] = pkt;
}
packet_set_ts(pkt_hdr, ts);
odp_prefetch(rte_pktmbuf_mtod(mbuf, char *));
+ num_pkts++;
}
if (pktio_cls_enabled(pktio_entry)) {
int failed = 0, success = 0;
- for (i = 0; i < num; i++) {
- odp_packet_t new_pkt;
+ for (i = 0; i < num_pkts; i++) {
odp_packet_t pkt = pkt_table[i];
odp_pool_t new_pool;
uint8_t *data;
- odp_packet_hdr_t parsed_hdr;
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
- struct rte_mbuf *mbuf = pkt_to_mbuf(pkt);
- uint32_t pkt_len = odp_packet_len(pkt);
- uint32_t ptypes = pkt_dpdk->supported_ptypes;
data = odp_packet_data(pkt);
- packet_parse_reset(&parsed_hdr, 1);
- packet_set_len(&parsed_hdr, pkt_len);
-
- if (_odp_dpdk_packet_parse_common(&parsed_hdr.p, data, pkt_len, pkt_len,
- mbuf, ODP_PROTO_LAYER_ALL, ptypes,
- pktin_cfg)) {
- odp_packet_free(pkt);
- continue;
- }
- if (_odp_cls_classify_packet(pktio_entry, data, pkt_len,
- pkt_len, &new_pool, &parsed_hdr, 0)) {
- failed++;
+ if (_odp_cls_classify_packet(pktio_entry, data,
+ &new_pool, pkt_hdr)) {
odp_packet_free(pkt);
continue;
}
if (new_pool != odp_packet_pool(pkt)) {
- new_pkt = odp_packet_copy(pkt, new_pool);
+ odp_packet_t new_pkt = odp_packet_copy(pkt, new_pool);
odp_packet_free(pkt);
- if (new_pkt == ODP_PACKET_INVALID) {
+ if (odp_unlikely(new_pkt == ODP_PACKET_INVALID)) {
failed++;
continue;
}
pkt_table[i] = new_pkt;
pkt = new_pkt;
- pkt_hdr = packet_hdr(pkt);
}
- packet_set_ts(pkt_hdr, ts);
+
pktio_entry->s.stats.in_octets += odp_packet_len(pkt);
- _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr);
if (success != i)
pkt_table[success] = pkt;
++success;
}
pktio_entry->s.stats.in_errors += failed;
- pktio_entry->s.stats.in_ucast_pkts += num - failed;
- num = success;
+ pktio_entry->s.stats.in_ucast_pkts += num_pkts - failed;
+ num_pkts = success;
}
- return num;
+ return num_pkts;
}
static int recv_pkt_dpdk(pktio_entry_t *pktio_entry, int index,
@@ -1125,12 +1137,12 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg,
mbuf->l2_len = pkt_p->l3_offset - pkt_p->l2_offset;
if (l3_proto_v4)
- mbuf->ol_flags = PKT_TX_IPV4;
+ mbuf->ol_flags = RTE_MBUF_F_TX_IPV4;
else
- mbuf->ol_flags = PKT_TX_IPV6;
+ mbuf->ol_flags = RTE_MBUF_F_TX_IPV6;
if (ipv4_chksum_pkt) {
- mbuf->ol_flags |= PKT_TX_IP_CKSUM;
+ mbuf->ol_flags |= RTE_MBUF_F_TX_IP_CKSUM;
((struct rte_ipv4_hdr *)l3_hdr)->hdr_checksum = 0;
mbuf->l3_len = _ODP_IPV4HDR_IHL(*(uint8_t *)l3_hdr) * 4;
@@ -1144,12 +1156,12 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg,
l4_hdr = (void *)(mbuf_data + pkt_p->l4_offset);
if (udp_chksum_pkt) {
- mbuf->ol_flags |= PKT_TX_UDP_CKSUM;
+ mbuf->ol_flags |= RTE_MBUF_F_TX_UDP_CKSUM;
((struct rte_udp_hdr *)l4_hdr)->dgram_cksum =
phdr_csum(l3_proto_v4, l3_hdr, mbuf->ol_flags);
} else if (tcp_chksum_pkt) {
- mbuf->ol_flags |= PKT_TX_TCP_CKSUM;
+ mbuf->ol_flags |= RTE_MBUF_F_TX_TCP_CKSUM;
((struct rte_tcp_hdr *)l4_hdr)->cksum =
phdr_csum(l3_proto_v4, l3_hdr, mbuf->ol_flags);
diff --git a/platform/linux-dpdk/odp_packet_flags.c b/platform/linux-dpdk/odp_packet_flags.c
index 251136e0f..728df157a 100644
--- a/platform/linux-dpdk/odp_packet_flags.c
+++ b/platform/linux-dpdk/odp_packet_flags.c
@@ -8,137 +8,11 @@
#include <odp/api/packet_flags.h>
#include <odp_packet_internal.h>
-#define retflag(pkt, x) do { \
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); \
- return pkt_hdr->p.x; \
- } while (0)
-
#define setflag(pkt, x, v) do { \
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); \
pkt_hdr->p.x = (v) & 1; \
} while (0)
-int odp_packet_has_error(odp_packet_t pkt)
-{
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
-
- return pkt_hdr->p.flags.all.error != 0;
-}
-
-/* Get Input Flags */
-
-int odp_packet_has_l2_error(odp_packet_t pkt)
-{
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
- /* L2 parsing is always done by default and hence
- no additional check is required */
- return pkt_hdr->p.flags.snap_len_err;
-}
-
-int odp_packet_has_l3(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.l3);
-}
-
-int odp_packet_has_l3_error(odp_packet_t pkt)
-{
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
-
- return pkt_hdr->p.flags.ip_err;
-}
-
-int odp_packet_has_l4(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.l4);
-}
-
-int odp_packet_has_l4_error(odp_packet_t pkt)
-{
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
-
- return pkt_hdr->p.flags.tcp_err | pkt_hdr->p.flags.udp_err;
-}
-
-int odp_packet_has_eth_bcast(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.eth_bcast);
-}
-
-int odp_packet_has_eth_mcast(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.eth_mcast);
-}
-
-int odp_packet_has_vlan(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.vlan);
-}
-
-int odp_packet_has_vlan_qinq(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.vlan_qinq);
-}
-
-int odp_packet_has_arp(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.arp);
-}
-
-int odp_packet_has_ipv4(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ipv4);
-}
-
-int odp_packet_has_ipv6(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ipv6);
-}
-
-int odp_packet_has_ip_bcast(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ip_bcast);
-}
-
-int odp_packet_has_ip_mcast(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ip_mcast);
-}
-
-int odp_packet_has_ipfrag(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ipfrag);
-}
-
-int odp_packet_has_ipopt(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ipopt);
-}
-
-int odp_packet_has_udp(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.udp);
-}
-
-int odp_packet_has_tcp(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.tcp);
-}
-
-int odp_packet_has_sctp(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.sctp);
-}
-
-int odp_packet_has_icmp(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.icmp);
-}
-
-odp_packet_color_t odp_packet_color(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.color);
-}
-
void odp_packet_color_set(odp_packet_t pkt, odp_packet_color_t color)
{
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
@@ -146,23 +20,11 @@ void odp_packet_color_set(odp_packet_t pkt, odp_packet_color_t color)
pkt_hdr->p.input_flags.color = color;
}
-odp_bool_t odp_packet_drop_eligible(odp_packet_t pkt)
-{
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
-
- return !pkt_hdr->p.input_flags.nodrop;
-}
-
void odp_packet_drop_eligible_set(odp_packet_t pkt, odp_bool_t drop)
{
setflag(pkt, input_flags.nodrop, !drop);
}
-int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt)
-{
- retflag(pkt, flags.shaper_len_adj);
-}
-
void odp_packet_shaper_len_adjust_set(odp_packet_t pkt, int8_t adj)
{
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
diff --git a/platform/linux-dpdk/odp_time.c b/platform/linux-dpdk/odp_time.c
index a53240564..e7cd5d1bb 100644
--- a/platform/linux-dpdk/odp_time.c
+++ b/platform/linux-dpdk/odp_time.c
@@ -1,5 +1,5 @@
/* Copyright (c) 2013-2018, Linaro Limited
- * Copyright (c) 2021, Nokia
+ * Copyright (c) 2021-2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -174,15 +174,6 @@ static inline void time_wait_until(odp_time_t time)
} while (odp_time_cmp(time, cur) > 0);
}
-uint64_t odp_time_diff_ns(odp_time_t t2, odp_time_t t1)
-{
- odp_time_t time;
-
- time.u64 = t2.u64 - t1.u64;
-
- return odp_time_to_ns(time);
-}
-
odp_time_t odp_time_local_from_ns(uint64_t ns)
{
return time_from_ns(ns);
diff --git a/platform/linux-dpdk/odp_timer.c b/platform/linux-dpdk/odp_timer.c
index f2f28f1d6..01f23ee31 100644
--- a/platform/linux-dpdk/odp_timer.c
+++ b/platform/linux-dpdk/odp_timer.c
@@ -16,6 +16,9 @@
#include <odp/api/timer.h>
#include <odp/api/plat/queue_inlines.h>
+#include <odp/api/plat/timer_inlines.h>
+
+#include <odp/api/plat/timer_inline_types.h>
#include <odp_debug_internal.h>
#include <odp_init_internal.h>
@@ -172,6 +175,18 @@ static timer_global_t *timer_global;
/* Timer thread local data */
static __thread timer_local_t timer_local;
+#include <odp/visibility_begin.h>
+
+/* Fill in timeout header field offsets for inline functions */
+const _odp_timeout_inline_offset_t
+_odp_timeout_inline_offset ODP_ALIGNED_CACHE = {
+ .expiration = offsetof(odp_timeout_hdr_t, expiration),
+ .timer = offsetof(odp_timeout_hdr_t, timer),
+ .user_ptr = offsetof(odp_timeout_hdr_t, user_ptr)
+};
+
+#include <odp/visibility_end.h>
+
static void timer_cb(struct rte_timer *rte_timer, void *arg ODP_UNUSED)
{
timer_entry_t *timer = rte_timer->arg;
@@ -1204,16 +1219,6 @@ uint64_t odp_timer_to_u64(odp_timer_t timer_hdl)
return (uint64_t)(uintptr_t)timer_hdl;
}
-odp_timeout_t odp_timeout_from_event(odp_event_t ev)
-{
- return (odp_timeout_t)ev;
-}
-
-odp_event_t odp_timeout_to_event(odp_timeout_t tmo)
-{
- return (odp_event_t)tmo;
-}
-
uint64_t odp_timeout_to_u64(odp_timeout_t tmo)
{
return (uint64_t)(uintptr_t)tmo;
@@ -1231,25 +1236,6 @@ int odp_timeout_fresh(odp_timeout_t tmo)
return timeout_hdr->expiration == timer->tick;
}
-odp_timer_t odp_timeout_timer(odp_timeout_t tmo)
-{
- odp_timeout_hdr_t *timeout_hdr = timeout_to_hdr(tmo);
-
- return timeout_hdr->timer;
-}
-
-uint64_t odp_timeout_tick(odp_timeout_t tmo)
-{
- return timeout_to_hdr(tmo)->expiration;
-}
-
-void *odp_timeout_user_ptr(odp_timeout_t tmo)
-{
- odp_timeout_hdr_t *timeout_hdr = timeout_to_hdr(tmo);
-
- return (void *)(uintptr_t)timeout_hdr->user_ptr;
-}
-
odp_timeout_t odp_timeout_alloc(odp_pool_t pool_hdl)
{
odp_event_t event;
diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am
index 47e5cf759..d4e94f677 100644
--- a/platform/linux-generic/Makefile.am
+++ b/platform/linux-generic/Makefile.am
@@ -49,7 +49,9 @@ odpapiplatinclude_HEADERS = \
include/odp/api/plat/sync_inlines.h \
include/odp/api/plat/thread_inlines.h \
include/odp/api/plat/ticketlock_inlines.h \
- include/odp/api/plat/time_inlines.h
+ include/odp/api/plat/time_inlines.h \
+ include/odp/api/plat/timer_inlines.h \
+ include/odp/api/plat/timer_inline_types.h
odpapiabiarchinclude_HEADERS += \
include-abi/odp/api/abi/align.h \
@@ -65,6 +67,7 @@ odpapiabiarchinclude_HEADERS += \
include-abi/odp/api/abi/dma_types.h \
include-abi/odp/api/abi/errno.h \
include-abi/odp/api/abi/event.h \
+ include-abi/odp/api/abi/event_types.h \
include-abi/odp/api/abi/hash.h \
include-abi/odp/api/abi/init.h \
include-abi/odp/api/abi/ipsec.h \
@@ -95,6 +98,7 @@ odpapiabiarchinclude_HEADERS += \
include-abi/odp/api/abi/thrmask.h \
include-abi/odp/api/abi/ticketlock.h \
include-abi/odp/api/abi/time.h \
+ include-abi/odp/api/abi/timer.h \
include-abi/odp/api/abi/timer_types.h \
include-abi/odp/api/abi/traffic_mngr.h \
include-abi/odp/api/abi/version.h
@@ -126,6 +130,7 @@ noinst_HEADERS = \
include/odp_packet_dpdk.h \
include/odp_packet_internal.h \
include/odp_packet_io_internal.h \
+ include/odp_parse_internal.h \
include/odp_socket_common.h \
include/odp_packet_io_stats_common.h \
include/odp_packet_io_stats.h \
@@ -160,6 +165,7 @@ noinst_HEADERS = \
include/odp_timer_internal.h \
include/odp_timer_wheel_internal.h \
include/odp_traffic_mngr_internal.h \
+ include/odp_types_internal.h \
include/odp_event_vector_internal.h \
include/protocols/eth.h \
include/protocols/ip.h \
@@ -201,6 +207,7 @@ __LIB__libodp_linux_la_SOURCES = \
odp_packet_vector.c \
odp_packet_flags.c \
odp_packet_io.c \
+ odp_parse.c \
odp_pkt_queue.c \
odp_pool.c \
odp_queue_basic.c \
@@ -279,7 +286,8 @@ __LIB__libodp_linux_la_SOURCES += \
odp_sync_api.c \
odp_thread_api.c \
odp_ticketlock_api.c \
- odp_time_api.c
+ odp_time_api.c \
+ odp_timer_api.c
endif
if ARCH_IS_ARM
diff --git a/platform/linux-generic/arch/aarch64/odp_atomic.h b/platform/linux-generic/arch/aarch64/odp_atomic.h
index 445d91e28..d3b8ea4dc 100644
--- a/platform/linux-generic/arch/aarch64/odp_atomic.h
+++ b/platform/linux-generic/arch/aarch64/odp_atomic.h
@@ -12,6 +12,7 @@
#error This file should not be included directly, please include odp_cpu.h
#endif
+#include <odp_types_internal.h>
#include <limits.h>
#ifdef CONFIG_DMBSTR
@@ -36,22 +37,18 @@ do { \
#define LL_MO(mo) (HAS_ACQ((mo)) ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED)
#define SC_MO(mo) (HAS_RLS((mo)) ? __ATOMIC_RELEASE : __ATOMIC_RELAXED)
-/* Prevent warnings about ISO C not supporting __int128 */
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wpedantic"
-
#ifndef __ARM_FEATURE_QRDMX /* Feature only available in v8.1a and beyond */
static inline bool
-__lockfree_compare_exchange_16(register __int128 *var, __int128 *exp,
- register __int128 neu, bool weak, int mo_success,
+__lockfree_compare_exchange_16(register _odp_u128_t *var, _odp_u128_t *exp,
+ register _odp_u128_t neu, bool weak, int mo_success,
int mo_failure)
{
(void)weak; /* Always do strong CAS or we can't perform atomic read */
/* Ignore memory ordering for failure, memory order for
* success must be stronger or equal. */
(void)mo_failure;
- register __int128 old;
- register __int128 expected;
+ register _odp_u128_t old;
+ register _odp_u128_t expected;
int ll_mo = LL_MO(mo_success);
int sc_mo = SC_MO(mo_success);
@@ -66,10 +63,10 @@ __lockfree_compare_exchange_16(register __int128 *var, __int128 *exp,
return old == expected;
}
-static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu,
- int mo)
+static inline _odp_u128_t __lockfree_exchange_16(_odp_u128_t *var,
+ _odp_u128_t neu, int mo)
{
- register __int128 old;
+ register _odp_u128_t old;
int ll_mo = LL_MO(mo);
int sc_mo = SC_MO(mo);
@@ -81,10 +78,10 @@ static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu,
return old;
}
-static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask,
- int mo)
+static inline _odp_u128_t __lockfree_fetch_and_16(_odp_u128_t *var,
+ _odp_u128_t mask, int mo)
{
- register __int128 old;
+ register _odp_u128_t old;
int ll_mo = LL_MO(mo);
int sc_mo = SC_MO(mo);
@@ -96,10 +93,10 @@ static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask,
return old;
}
-static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask,
- int mo)
+static inline _odp_u128_t __lockfree_fetch_or_16(_odp_u128_t *var,
+ _odp_u128_t mask, int mo)
{
- register __int128 old;
+ register _odp_u128_t old;
int ll_mo = LL_MO(mo);
int sc_mo = SC_MO(mo);
@@ -113,8 +110,8 @@ static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask,
#else
-static inline __int128_t cas_u128(__int128_t *ptr, __int128_t old_val,
- __int128_t new_val, int mo)
+static inline _odp_u128_t cas_u128(_odp_u128_t *ptr, _odp_u128_t old_val,
+ _odp_u128_t new_val, int mo)
{
/* CASP instructions require that the first register number is paired */
register uint64_t old0 __asm__ ("x0");
@@ -151,18 +148,18 @@ static inline __int128_t cas_u128(__int128_t *ptr, __int128_t old_val,
abort();
}
- return ((__int128)old0) | (((__int128)old1) << 64);
+ return ((_odp_u128_t)old0) | (((_odp_u128_t)old1) << 64);
}
static inline bool
-__lockfree_compare_exchange_16(register __int128 *var, __int128 *exp,
- register __int128 neu, bool weak, int mo_success,
+__lockfree_compare_exchange_16(register _odp_u128_t *var, _odp_u128_t *exp,
+ register _odp_u128_t neu, bool weak, int mo_success,
int mo_failure)
{
(void)weak;
(void)mo_failure;
- __int128 old;
- __int128 expected;
+ _odp_u128_t old;
+ _odp_u128_t expected;
expected = *exp;
old = cas_u128(var, expected, neu, mo_success);
@@ -170,11 +167,11 @@ __lockfree_compare_exchange_16(register __int128 *var, __int128 *exp,
return old == expected;
}
-static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu,
- int mo)
+static inline _odp_u128_t __lockfree_exchange_16(_odp_u128_t *var,
+ _odp_u128_t neu, int mo)
{
- __int128 old;
- __int128 expected;
+ _odp_u128_t old;
+ _odp_u128_t expected;
do {
expected = *var;
@@ -183,11 +180,11 @@ static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu,
return old;
}
-static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask,
- int mo)
+static inline _odp_u128_t __lockfree_fetch_and_16(_odp_u128_t *var,
+ _odp_u128_t mask, int mo)
{
- __int128 old;
- __int128 expected;
+ _odp_u128_t old;
+ _odp_u128_t expected;
do {
expected = *var;
@@ -196,11 +193,11 @@ static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask,
return old;
}
-static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask,
- int mo)
+static inline _odp_u128_t __lockfree_fetch_or_16(_odp_u128_t *var,
+ _odp_u128_t mask, int mo)
{
- __int128 old;
- __int128 expected;
+ _odp_u128_t old;
+ _odp_u128_t expected;
do {
expected = *var;
@@ -211,9 +208,9 @@ static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask,
#endif /* __ARM_FEATURE_QRDMX */
-static inline __int128 __lockfree_load_16(__int128 *var, int mo)
+static inline _odp_u128_t __lockfree_load_16(_odp_u128_t *var, int mo)
{
- __int128 old = *var; /* Possibly torn read */
+ _odp_u128_t old = *var; /* Possibly torn read */
/* Do CAS to ensure atomicity
* Either CAS succeeds (writing back the same value)
@@ -223,19 +220,17 @@ static inline __int128 __lockfree_load_16(__int128 *var, int mo)
return old;
}
-__extension__ typedef unsigned __int128 _u128_t;
-
-static inline _u128_t lockfree_load_u128(_u128_t *atomic)
+static inline _odp_u128_t lockfree_load_u128(_odp_u128_t *atomic)
{
- return __lockfree_load_16((__int128 *)atomic, __ATOMIC_RELAXED);
+ return __lockfree_load_16((_odp_u128_t *)atomic, __ATOMIC_RELAXED);
}
-static inline int lockfree_cas_acq_rel_u128(_u128_t *atomic,
- _u128_t old_val,
- _u128_t new_val)
+static inline int lockfree_cas_acq_rel_u128(_odp_u128_t *atomic,
+ _odp_u128_t old_val,
+ _odp_u128_t new_val)
{
- return __lockfree_compare_exchange_16((__int128 *)atomic,
- (__int128 *)&old_val,
+ return __lockfree_compare_exchange_16((_odp_u128_t *)atomic,
+ (_odp_u128_t *)&old_val,
new_val,
0,
__ATOMIC_ACQ_REL,
@@ -249,7 +244,7 @@ static inline int lockfree_check_u128(void)
/** Atomic bit set operations with memory ordering */
#if defined(__SIZEOF_INT128__) && __SIZEOF_INT128__ == 16
-typedef __int128 bitset_t;
+typedef _odp_u128_t bitset_t;
#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT128__)
#elif __GCC_ATOMIC_LLONG_LOCK_FREE == 2 && \
@@ -292,11 +287,9 @@ static inline bitset_t bitset_mask(uint32_t bit)
if (bit < 64)
return 1ULL << bit;
else
- return (unsigned __int128)(1ULL << (bit - 64)) << 64;
+ return (_odp_u128_t)(1ULL << (bit - 64)) << 64;
}
-#pragma GCC diagnostic pop
-
#else
#error Unsupported size of bit sets (ATOM_BITSET_SIZE)
#endif
diff --git a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c
index d719c6cfa..4d46846ce 100644
--- a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c
+++ b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c
@@ -1,5 +1,6 @@
/* Copyright (c) 2014-2018, Linaro Limited
* Copyright (c) 2021, ARM Limited
+ * Copyright (c) 2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -37,9 +38,9 @@
/*
* ARM crypto library may read up to 15 bytes past the end of input
- * data and AAD.
+ * data and AAD and write up to 15 bytes past the end of output data.
*/
-#define OOB_READ_LEN 15
+#define OOB_WRITE_LEN 16 /* rounded up to 16 bytes for efficiency */
/*
* Data buffer size must be a multiple of 16, because the ARM crypto
@@ -80,9 +81,11 @@ static const odp_crypto_cipher_capability_t cipher_capa_aes_gcm[] = {
static const odp_crypto_auth_capability_t auth_capa_null[] = {
{.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0} } };
+#define AES_GCM_TAG_LEN 16
+
#ifdef __ARM_FEATURE_AES
static const odp_crypto_auth_capability_t auth_capa_aes_gcm[] = {
-{.digest_len = 16, .key_len = 0, .aad_len = {.min = 8, .max = 12, .inc = 4} } };
+{.digest_len = AES_GCM_TAG_LEN, .key_len = 0, .aad_len = {.min = 8, .max = 12, .inc = 4} } };
#endif
/** Forward declaration of session structure */
@@ -92,9 +95,9 @@ typedef struct odp_crypto_generic_session_t odp_crypto_generic_session_t;
* Algorithm handler function prototype
*/
typedef
-odp_crypto_alg_err_t (*crypto_func_t)(odp_packet_t pkt,
- const odp_crypto_packet_op_param_t *param,
- odp_crypto_generic_session_t *session);
+odp_bool_t (*crypto_func_t)(odp_packet_t pkt,
+ const odp_crypto_packet_op_param_t *param,
+ odp_crypto_generic_session_t *session);
/**
* Per crypto session data structure
@@ -105,15 +108,12 @@ struct odp_crypto_generic_session_t {
/* Session creation parameters */
odp_crypto_session_param_t p;
- odp_bool_t do_cipher_first;
-
struct {
#if ODP_DEPRECATED_API
/* Copy of session IV data */
uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH];
#endif
uint8_t key_data[ARM_CRYPTO_MAX_CIPHER_KEY_LENGTH];
- crypto_func_t func;
} cipher;
struct {
@@ -121,11 +121,10 @@ struct odp_crypto_generic_session_t {
#if ODP_DEPRECATED_API
uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH];
#endif
- crypto_func_t func;
} auth;
+ crypto_func_t func;
unsigned int idx;
-
armv8_cipher_constants_t cc;
};
@@ -141,7 +140,6 @@ static odp_crypto_global_t *global;
typedef struct crypto_local_t {
uint8_t buffer[ARM_CRYPTO_MAX_DATA_LENGTH];
- uint8_t digest[ARM_CRYPTO_MAX_DIGEST_LENGTH];
} crypto_local_t;
static __thread crypto_local_t local;
@@ -176,18 +174,60 @@ void free_session(odp_crypto_generic_session_t *session)
odp_spinlock_unlock(&global->lock);
}
-static odp_crypto_alg_err_t
+static
+odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt)
+{
+ odp_packet_hdr_t *hdr = packet_hdr(pkt);
+
+ return &hdr->crypto_op_result;
+}
+
+static inline void set_crypto_op_result(odp_packet_t pkt,
+ odp_crypto_alg_err_t cipher_err,
+ odp_crypto_alg_err_t auth_err)
+{
+ odp_crypto_packet_result_t *op_result;
+
+ op_result = get_op_result_from_packet(pkt);
+ op_result->cipher_status.alg_err = cipher_err;
+ op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
+ op_result->auth_status.alg_err = auth_err;
+ op_result->auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
+ op_result->ok = (cipher_err == ODP_CRYPTO_ALG_ERR_NONE &&
+ auth_err == ODP_CRYPTO_ALG_ERR_NONE);
+}
+
+static inline void set_crypto_op_result_ok(odp_packet_t pkt)
+{
+ set_crypto_op_result(pkt,
+ ODP_CRYPTO_ALG_ERR_NONE,
+ ODP_CRYPTO_ALG_ERR_NONE);
+}
+
+static odp_bool_t
null_crypto_routine(odp_packet_t pkt ODP_UNUSED,
const odp_crypto_packet_op_param_t *param ODP_UNUSED,
odp_crypto_generic_session_t *session ODP_UNUSED)
{
- return ODP_CRYPTO_ALG_ERR_NONE;
+ set_crypto_op_result_ok(pkt);
+ return true;
+}
+
+static inline void copy_aad(uint8_t *dst, uint8_t *src, uint32_t len)
+{
+ ODP_ASSERT(len == 8 || len == 12);
+
+ /* Use constant length memcpy for better optimization result */
+ if (len == 8)
+ memcpy(dst, src, 8);
+ else
+ memcpy(dst, src, 12);
}
static
-odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt,
- const odp_crypto_packet_op_param_t *param,
- odp_crypto_generic_session_t *session)
+odp_bool_t aes_gcm_encrypt(odp_packet_t pkt,
+ const odp_crypto_packet_op_param_t *param,
+ odp_crypto_generic_session_t *session)
{
armv8_cipher_state_t cs = {
.counter = {
@@ -200,12 +240,14 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt,
uint64_t aad_bit_length = session->p.auth_aad_len * 8;
uint32_t in_pos = param->cipher_range.offset;
uint32_t in_len = param->cipher_range.length;
- int ret = 0;
+ odp_bool_t continuous_data;
+ uint16_t saved_tail[OOB_WRITE_LEN];
+ uint8_t tag[AES_GCM_TAG_LEN];
+ int rc;
/* Fail early if cipher_range is too large */
- if (in_len > ARM_CRYPTO_MAX_DATA_LENGTH) {
+ if (odp_unlikely(in_len > ARM_CRYPTO_MAX_DATA_LENGTH)) {
ODP_DBG("ARM Crypto: Packet size too large for requested operation\n");
- ret = -1;
goto err;
}
@@ -223,9 +265,9 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt,
cs.constants = &session->cc;
- if (armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs) != 0) {
+ rc = armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs);
+ if (odp_unlikely(rc)) {
ODP_DBG("ARM Crypto: Failure while setting nonce\n");
- ret = -1;
goto err;
}
@@ -233,46 +275,60 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt,
* read it in 16 byte chunks. */
uint8_t aad[ARM_CRYPTO_MAX_AAD_LENGTH];
- memcpy(aad, param->aad_ptr, session->p.auth_aad_len);
+ copy_aad(aad, param->aad_ptr, session->p.auth_aad_len);
uint32_t seg_len = 0;
uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL);
if (odp_unlikely(odp_packet_is_segmented(pkt)) ||
- odp_unlikely(odp_packet_tailroom(pkt) < OOB_READ_LEN)) {
- /* Packet is segmented or it may not be safe to read beyond
- * the end of packet data. Copy the cipher range to a
+ odp_unlikely(odp_packet_tailroom(pkt) < OOB_WRITE_LEN)) {
+ /* Packet is segmented or it may not be safe to read and write
+ * beyond the end of packet data. Copy the cipher range to a
* contiguous buffer. */
odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer);
data = local.buffer;
+ continuous_data = false;
+ } else {
+ /* Save data that might get overwritten */
+ memcpy(saved_tail, data + in_len, OOB_WRITE_LEN);
+ continuous_data = true;
}
- if (armv8_enc_aes_gcm_from_state(&cs,
- aad, aad_bit_length,
- data, plaintext_bit_length,
- local.buffer,
- local.digest) != 0) {
+ rc = armv8_enc_aes_gcm_from_state(&cs,
+ aad, aad_bit_length,
+ data, plaintext_bit_length,
+ data,
+ tag);
+ if (odp_unlikely(rc)) {
ODP_DBG("ARM Crypto: AES GCM Encoding failed\n");
- ret = -1;
goto err;
}
- odp_packet_copy_from_mem(pkt, param->cipher_range.offset,
- param->cipher_range.length, local.buffer);
+ if (odp_likely(continuous_data)) {
+ memcpy(data + in_len, saved_tail, OOB_WRITE_LEN);
+ memcpy(data - in_pos + param->hash_result_offset,
+ tag, AES_GCM_TAG_LEN);
+ } else {
+ odp_packet_copy_from_mem(pkt, in_pos, in_len, data);
+ odp_packet_copy_from_mem(pkt, param->hash_result_offset,
+ AES_GCM_TAG_LEN, tag);
+ }
- odp_packet_copy_from_mem(pkt, param->hash_result_offset,
- session->p.auth_digest_len, local.digest);
+ set_crypto_op_result_ok(pkt);
+ return true;
err:
- return ret < 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE :
- ODP_CRYPTO_ALG_ERR_NONE;
+ set_crypto_op_result(pkt,
+ ODP_CRYPTO_ALG_ERR_DATA_SIZE,
+ ODP_CRYPTO_ALG_ERR_NONE);
+ return false;
}
static
-odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt,
- const odp_crypto_packet_op_param_t *param,
- odp_crypto_generic_session_t *session)
+odp_bool_t aes_gcm_decrypt(odp_packet_t pkt,
+ const odp_crypto_packet_op_param_t *param,
+ odp_crypto_generic_session_t *session)
{
armv8_cipher_state_t cs = {
.counter = {
@@ -280,18 +336,19 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt,
}
};
uint8_t *iv_ptr;
- uint8_t tag[16];
+ uint8_t tag[AES_GCM_TAG_LEN];
uint64_t iv_bit_length = session->p.cipher_iv_len * 8;
uint64_t plaintext_bit_length = param->cipher_range.length * 8;
uint64_t aad_bit_length = session->p.auth_aad_len * 8;
uint32_t in_pos = param->cipher_range.offset;
uint32_t in_len = param->cipher_range.length;
- int ret = 0;
+ odp_bool_t continuous_data;
+ uint16_t saved_tail[OOB_WRITE_LEN];
+ int rc;
/* Fail early if cipher_range is too large */
- if (in_len > ARM_CRYPTO_MAX_DATA_LENGTH) {
+ if (odp_unlikely(in_len > ARM_CRYPTO_MAX_DATA_LENGTH)) {
ODP_DBG("ARM Crypto: Packet size too large for requested operation\n");
- ret = -1;
goto err;
}
@@ -309,51 +366,63 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt,
cs.constants = &session->cc;
- if (armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs) != 0) {
+ rc = armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs);
+ if (odp_unlikely(rc)) {
ODP_DBG("ARM Crypto: Failure while setting nonce\n");
- ret = -1;
goto err;
}
- /* Copy current hash bytes to a buffer and clear it */
- odp_packet_copy_to_mem(pkt, param->hash_result_offset,
- session->p.auth_digest_len, tag);
-
/* Copy AAD in a stack to make sure that the ARM crypto library can
* read it in 16 byte chunks. */
uint8_t aad[ARM_CRYPTO_MAX_AAD_LENGTH];
- memcpy(aad, param->aad_ptr, session->p.auth_aad_len);
+ copy_aad(aad, param->aad_ptr, session->p.auth_aad_len);
uint32_t seg_len = 0;
uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL);
if (odp_unlikely(odp_packet_is_segmented(pkt)) ||
- odp_unlikely(odp_packet_tailroom(pkt) < OOB_READ_LEN)) {
- /* Packet is segmented or it may not be safe to read beyond
- * the end of packet data. Copy the cipher range to a
+ odp_unlikely(odp_packet_tailroom(pkt) < OOB_WRITE_LEN)) {
+ /* Packet is segmented or it may not be safe to read and write
+ * beyond the end of packet data. Copy the cipher range to a
* contiguous buffer. */
odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer);
-
data = local.buffer;
+ /* Copy tag from the packet to a buffer */
+ odp_packet_copy_to_mem(pkt, param->hash_result_offset,
+ AES_GCM_TAG_LEN, tag);
+ continuous_data = false;
+ } else {
+ /* Save data that might get overwritten */
+ memcpy(saved_tail, data + in_len, OOB_WRITE_LEN);
+ /* Copy tag from the packet to a buffer */
+ memcpy(tag, data - in_pos + param->hash_result_offset, AES_GCM_TAG_LEN);
+ continuous_data = true;
}
- if (armv8_dec_aes_gcm_from_state(&cs,
- aad, aad_bit_length,
- data, plaintext_bit_length,
- tag,
- local.buffer) != 0) {
+ rc = armv8_dec_aes_gcm_from_state(&cs,
+ aad, aad_bit_length,
+ data, plaintext_bit_length,
+ tag,
+ data);
+ if (odp_unlikely(rc)) {
ODP_DBG("ARM Crypto: AES GCM Decoding failed\n");
- ret = -1;
goto err;
}
- odp_packet_copy_from_mem(pkt, param->cipher_range.offset,
- param->cipher_range.length, local.buffer);
+ if (odp_likely(continuous_data))
+ memcpy(data + in_len, saved_tail, OOB_WRITE_LEN);
+ else
+ odp_packet_copy_from_mem(pkt, in_pos, in_len, data);
+
+ set_crypto_op_result_ok(pkt);
+ return true;
err:
- return ret < 0 ? ODP_CRYPTO_ALG_ERR_ICV_CHECK :
- ODP_CRYPTO_ALG_ERR_NONE;
+ set_crypto_op_result(pkt,
+ ODP_CRYPTO_ALG_ERR_NONE,
+ ODP_CRYPTO_ALG_ERR_ICV_CHECK);
+ return false;
}
static int process_aes_gcm_param(odp_crypto_generic_session_t *session)
@@ -376,9 +445,9 @@ static int process_aes_gcm_param(odp_crypto_generic_session_t *session)
/* Set function */
if (ODP_CRYPTO_OP_ENCODE == session->p.op)
- session->cipher.func = aes_gcm_encrypt;
+ session->func = aes_gcm_encrypt;
else
- session->cipher.func = aes_gcm_decrypt;
+ session->func = aes_gcm_decrypt;
return 0;
}
@@ -520,16 +589,10 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param,
session->p.auth_iv.length);
#endif
- /* Derive order */
- if (ODP_CRYPTO_OP_ENCODE == param->op)
- session->do_cipher_first = param->auth_cipher_text;
- else
- session->do_cipher_first = !param->auth_cipher_text;
-
/* Process based on cipher */
switch (param->cipher_alg) {
case ODP_CIPHER_ALG_NULL:
- session->cipher.func = null_crypto_routine;
+ session->func = null_crypto_routine;
rc = 0;
break;
case ODP_CIPHER_ALG_AES_GCM:
@@ -584,14 +647,15 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param,
/* Process based on auth */
switch (param->auth_alg) {
case ODP_AUTH_ALG_NULL:
- session->auth.func = null_crypto_routine;
- rc = 0;
+ if (param->cipher_alg == ODP_CIPHER_ALG_NULL)
+ rc = 0;
+ else
+ rc = -1;
break;
case ODP_AUTH_ALG_AES_GCM:
/* AES-GCM requires to do both auth and
* cipher at the same time */
if (param->cipher_alg == ODP_CIPHER_ALG_AES_GCM) {
- session->auth.func = null_crypto_routine;
rc = 0;
} else {
rc = -1;
@@ -630,9 +694,7 @@ int odp_crypto_session_destroy(odp_crypto_session_t session)
return 0;
}
-/*
- * Shim function around packet operation, can be used by other implementations.
- */
+#if ODP_DEPRECATED_API
int
odp_crypto_operation(odp_crypto_op_param_t *param,
odp_bool_t *posted,
@@ -687,6 +749,7 @@ odp_crypto_operation(odp_crypto_op_param_t *param,
return 0;
}
+#endif
int _odp_crypto_init_global(void)
{
@@ -767,6 +830,7 @@ int _odp_crypto_term_local(void)
return 0;
}
+#if ODP_DEPRECATED_API
odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev)
{
/* This check not mandated by the API specification */
@@ -803,6 +867,7 @@ uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl)
{
return _odp_pri(hdl);
}
+#endif /* ODP_DEPRECATED_API */
void odp_crypto_session_param_init(odp_crypto_session_param_t *param)
{
@@ -828,14 +893,6 @@ odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt)
return odp_packet_to_event(pkt);
}
-static
-odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt)
-{
- odp_packet_hdr_t *hdr = packet_hdr(pkt);
-
- return &hdr->crypto_op_result;
-}
-
int odp_crypto_result(odp_crypto_packet_result_t *result,
odp_packet_t packet)
{
@@ -856,18 +913,16 @@ int crypto_int(odp_packet_t pkt_in,
odp_packet_t *pkt_out,
const odp_crypto_packet_op_param_t *param)
{
- odp_crypto_alg_err_t rc_cipher = ODP_CRYPTO_ALG_ERR_NONE;
- odp_crypto_alg_err_t rc_auth = ODP_CRYPTO_ALG_ERR_NONE;
odp_crypto_generic_session_t *session;
odp_bool_t allocated = false;
odp_packet_t out_pkt = *pkt_out;
- odp_crypto_packet_result_t *op_result;
odp_packet_hdr_t *pkt_hdr;
+ odp_bool_t ok;
session = (odp_crypto_generic_session_t *)(intptr_t)param->session;
/* Resolve output buffer */
- if (ODP_PACKET_INVALID == out_pkt &&
+ if (odp_unlikely(ODP_PACKET_INVALID == out_pkt) &&
ODP_POOL_INVALID != session->p.output_pool) {
out_pkt = odp_packet_alloc(session->p.output_pool,
odp_packet_len(pkt_in));
@@ -879,7 +934,7 @@ int crypto_int(odp_packet_t pkt_in,
return -1;
}
- if (pkt_in != out_pkt) {
+ if (odp_unlikely(pkt_in != out_pkt)) {
int ret;
int md_copy;
@@ -903,28 +958,12 @@ int crypto_int(odp_packet_t pkt_in,
pkt_in = ODP_PACKET_INVALID;
}
- /* Invoke the functions */
- if (session->do_cipher_first) {
- rc_cipher = session->cipher.func(out_pkt, param, session);
- rc_auth = session->auth.func(out_pkt, param, session);
- } else {
- rc_auth = session->auth.func(out_pkt, param, session);
- rc_cipher = session->cipher.func(out_pkt, param, session);
- }
+ /* Invoke the crypto function */
+ ok = session->func(out_pkt, param, session);
- /* Fill in result */
packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO);
- op_result = get_op_result_from_packet(out_pkt);
- op_result->cipher_status.alg_err = rc_cipher;
- op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
- op_result->auth_status.alg_err = rc_auth;
- op_result->auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
- op_result->ok =
- (rc_cipher == ODP_CRYPTO_ALG_ERR_NONE) &&
- (rc_auth == ODP_CRYPTO_ALG_ERR_NONE);
-
pkt_hdr = packet_hdr(out_pkt);
- pkt_hdr->p.flags.crypto_err = !op_result->ok;
+ pkt_hdr->p.flags.crypto_err = !ok;
/* Synchronous, simply return results */
*pkt_out = out_pkt;
diff --git a/platform/linux-generic/arch/aarch64/odp_llsc.h b/platform/linux-generic/arch/aarch64/odp_llsc.h
index 2561451ca..1c2610511 100644
--- a/platform/linux-generic/arch/aarch64/odp_llsc.h
+++ b/platform/linux-generic/arch/aarch64/odp_llsc.h
@@ -13,6 +13,8 @@
#error This file should not be included directly, please include odp_cpu.h
#endif
+#include <odp_types_internal.h>
+
static inline uint16_t ll8(uint8_t *var, int mm)
{
uint16_t old;
@@ -115,11 +117,11 @@ static inline uint32_t sc(uint64_t *var, uint64_t neu, int mm)
#define sc64(a, b, c) sc((a), (b), (c))
union i128 {
- __extension__ __int128 i128;
+ _odp_u128_t i128;
int64_t i64[2];
};
-__extension__ static inline __int128 lld(__int128 *var, int mm)
+static inline _odp_u128_t lld(_odp_u128_t *var, int mm)
{
union i128 old;
@@ -139,7 +141,7 @@ __extension__ static inline __int128 lld(__int128 *var, int mm)
}
/* Return 0 on success, 1 on failure */
-__extension__ static inline uint32_t scd(__int128 *var, __int128 neu, int mm)
+static inline uint32_t scd(_odp_u128_t *var, _odp_u128_t neu, int mm)
{
uint32_t ret;
diff --git a/platform/linux-generic/arch/arm/odp_atomic.h b/platform/linux-generic/arch/arm/odp_atomic.h
index bcc89ff73..e400f52d4 100644
--- a/platform/linux-generic/arch/arm/odp_atomic.h
+++ b/platform/linux-generic/arch/arm/odp_atomic.h
@@ -12,6 +12,7 @@
#error This file should not be included directly, please include odp_cpu.h
#endif
+#include <odp_types_internal.h>
#include <limits.h>
#ifdef CONFIG_DMBSTR
@@ -70,7 +71,7 @@ static inline bitset_t bitset_mask(uint32_t bit)
if (bit < 64)
return 1ULL << bit;
else
- return (unsigned __int128)(1ULL << (bit - 64)) << 64;
+ return (_odp_u128_t)(1ULL << (bit - 64)) << 64;
}
#else
diff --git a/platform/linux-generic/arch/default/odp_atomic.h b/platform/linux-generic/arch/default/odp_atomic.h
index 7650d2b52..4cfc6b4bd 100644
--- a/platform/linux-generic/arch/default/odp_atomic.h
+++ b/platform/linux-generic/arch/default/odp_atomic.h
@@ -7,18 +7,18 @@
#ifndef ODP_DEFAULT_ATOMIC_H_
#define ODP_DEFAULT_ATOMIC_H_
-#ifdef __SIZEOF_INT128__
+#include <odp_types_internal.h>
-__extension__ typedef unsigned __int128 _u128_t;
+#ifdef __SIZEOF_INT128__
-static inline _u128_t lockfree_load_u128(_u128_t *atomic)
+static inline _odp_u128_t lockfree_load_u128(_odp_u128_t *atomic)
{
return __atomic_load_n(atomic, __ATOMIC_RELAXED);
}
-static inline int lockfree_cas_acq_rel_u128(_u128_t *atomic,
- _u128_t old_val,
- _u128_t new_val)
+static inline int lockfree_cas_acq_rel_u128(_odp_u128_t *atomic,
+ _odp_u128_t old_val,
+ _odp_u128_t new_val)
{
return __atomic_compare_exchange_n(atomic, &old_val, new_val,
0 /* strong */,
@@ -76,7 +76,7 @@ static inline bitset_t bitset_mask(uint32_t bit)
if (bit < 64)
return 1ULL << bit;
else
- return (unsigned __int128)(1ULL << (bit - 64)) << 64;
+ return (_odp_u128_t)(1ULL << (bit - 64)) << 64;
}
#else
diff --git a/platform/linux-generic/include-abi/odp/api/abi/event.h b/platform/linux-generic/include-abi/odp/api/abi/event.h
index 6530ac2e4..e059f318c 100644
--- a/platform/linux-generic/include-abi/odp/api/abi/event.h
+++ b/platform/linux-generic/include-abi/odp/api/abi/event.h
@@ -1,4 +1,5 @@
/* Copyright (c) 2015-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -18,42 +19,9 @@
extern "C" {
#endif
-#include <odp/api/plat/strong_types.h>
-
-/** @ingroup odp_event
- * @{
- */
-
-typedef ODP_HANDLE_T(odp_event_t);
-
-#define ODP_EVENT_INVALID _odp_cast_scalar(odp_event_t, 0)
-
-typedef enum odp_event_type_t {
- ODP_EVENT_BUFFER = 1,
- ODP_EVENT_PACKET = 2,
- ODP_EVENT_TIMEOUT = 3,
- ODP_EVENT_CRYPTO_COMPL = 4,
- ODP_EVENT_IPSEC_STATUS = 5,
- ODP_EVENT_PACKET_VECTOR = 6,
- ODP_EVENT_PACKET_TX_COMPL = 7,
- ODP_EVENT_DMA_COMPL = 8,
-} odp_event_type_t;
-
-typedef enum odp_event_subtype_t {
- ODP_EVENT_NO_SUBTYPE = 0,
- ODP_EVENT_PACKET_BASIC = 1,
- ODP_EVENT_PACKET_CRYPTO = 2,
- ODP_EVENT_PACKET_IPSEC = 3,
- ODP_EVENT_PACKET_COMP = 4
-} odp_event_subtype_t;
-
/* Inlined functions for non-ABI compat mode */
#include <odp/api/plat/event_inlines.h>
-/**
- * @}
- */
-
#ifdef __cplusplus
}
#endif
diff --git a/platform/linux-generic/include-abi/odp/api/abi/event_types.h b/platform/linux-generic/include-abi/odp/api/abi/event_types.h
new file mode 100644
index 000000000..c39322330
--- /dev/null
+++ b/platform/linux-generic/include-abi/odp/api/abi/event_types.h
@@ -0,0 +1,61 @@
+/* Copyright (c) 2015-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODP event type definitions
+ */
+
+#ifndef ODP_API_ABI_EVENT_TYPES_H_
+#define ODP_API_ABI_EVENT_TYPES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp/api/plat/strong_types.h>
+#include <odp/api/deprecated.h>
+
+/** @ingroup odp_event
+ * @{
+ */
+
+typedef ODP_HANDLE_T(odp_event_t);
+
+#define ODP_EVENT_INVALID _odp_cast_scalar(odp_event_t, 0)
+
+typedef enum odp_event_type_t {
+ ODP_EVENT_BUFFER = 1,
+ ODP_EVENT_PACKET = 2,
+ ODP_EVENT_TIMEOUT = 3,
+#if ODP_DEPRECATED_API
+ ODP_EVENT_CRYPTO_COMPL = 4,
+#endif
+ ODP_EVENT_IPSEC_STATUS = 5,
+ ODP_EVENT_PACKET_VECTOR = 6,
+ ODP_EVENT_PACKET_TX_COMPL = 7,
+ ODP_EVENT_DMA_COMPL = 8,
+} odp_event_type_t;
+
+typedef enum odp_event_subtype_t {
+ ODP_EVENT_NO_SUBTYPE = 0,
+ ODP_EVENT_PACKET_BASIC = 1,
+ ODP_EVENT_PACKET_CRYPTO = 2,
+ ODP_EVENT_PACKET_IPSEC = 3,
+ ODP_EVENT_PACKET_COMP = 4
+} odp_event_subtype_t;
+
+/**
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/include-abi/odp/api/abi/timer.h b/platform/linux-generic/include-abi/odp/api/abi/timer.h
new file mode 100644
index 000000000..0a3b3a9cc
--- /dev/null
+++ b/platform/linux-generic/include-abi/odp/api/abi/timer.h
@@ -0,0 +1,8 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/* Inlined functions for non-ABI compat mode */
+#include <odp/api/plat/timer_inlines.h>
diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h
index 632ba0e69..9e0ae3f50 100644
--- a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h
@@ -8,7 +8,7 @@
#define ODP_PLAT_BUFFER_INLINES_H_
#include <odp/api/abi/buffer.h>
-#include <odp/api/abi/event.h>
+#include <odp/api/abi/event_types.h>
#include <odp/api/plat/buffer_inline_types.h>
diff --git a/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h
index 6eb34a39b..04ad9e968 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h
@@ -1,4 +1,5 @@
/* Copyright (c) 2017-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -30,11 +31,32 @@ static inline uint64_t _odp_packet_input_flags(odp_packet_t pkt)
/* Inline functions by default */
#define _ODP_INLINE static inline
#define odp_packet_has_l2 __odp_packet_has_l2
+ #define odp_packet_has_l3 __odp_packet_has_l3
+ #define odp_packet_has_l4 __odp_packet_has_l4
#define odp_packet_has_eth __odp_packet_has_eth
#define odp_packet_has_jumbo __odp_packet_has_jumbo
#define odp_packet_has_flow_hash __odp_packet_has_flow_hash
#define odp_packet_has_ts __odp_packet_has_ts
#define odp_packet_has_ipsec __odp_packet_has_ipsec
+ #define odp_packet_has_eth_bcast __odp_packet_has_eth_bcast
+ #define odp_packet_has_eth_mcast __odp_packet_has_eth_mcast
+ #define odp_packet_has_vlan __odp_packet_has_vlan
+ #define odp_packet_has_vlan_qinq __odp_packet_has_vlan_qinq
+ #define odp_packet_has_arp __odp_packet_has_arp
+ #define odp_packet_has_ipv4 __odp_packet_has_ipv4
+ #define odp_packet_has_ipv6 __odp_packet_has_ipv6
+ #define odp_packet_has_ip_bcast __odp_packet_has_ip_bcast
+ #define odp_packet_has_ip_mcast __odp_packet_has_ip_mcast
+ #define odp_packet_has_ipfrag __odp_packet_has_ipfrag
+ #define odp_packet_has_ipopt __odp_packet_has_ipopt
+ #define odp_packet_has_udp __odp_packet_has_udp
+ #define odp_packet_has_tcp __odp_packet_has_tcp
+ #define odp_packet_has_sctp __odp_packet_has_sctp
+ #define odp_packet_has_icmp __odp_packet_has_icmp
+ #define odp_packet_has_error __odp_packet_has_error
+ #define odp_packet_has_l2_error __odp_packet_has_l2_error
+ #define odp_packet_has_l3_error __odp_packet_has_l3_error
+ #define odp_packet_has_l4_error __odp_packet_has_l4_error
#else
#undef _ODP_INLINE
#define _ODP_INLINE
@@ -48,6 +70,22 @@ _ODP_INLINE int odp_packet_has_l2(odp_packet_t pkt)
return flags.l2;
}
+_ODP_INLINE int odp_packet_has_l3(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.l3;
+}
+
+_ODP_INLINE int odp_packet_has_l4(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.l4;
+}
+
_ODP_INLINE int odp_packet_has_eth(odp_packet_t pkt)
{
_odp_packet_input_flags_t flags;
@@ -88,6 +126,163 @@ _ODP_INLINE int odp_packet_has_ipsec(odp_packet_t pkt)
return flags.ipsec;
}
+_ODP_INLINE int odp_packet_has_eth_bcast(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.eth_bcast;
+}
+
+_ODP_INLINE int odp_packet_has_eth_mcast(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.eth_mcast;
+}
+
+_ODP_INLINE int odp_packet_has_vlan(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.vlan;
+}
+
+_ODP_INLINE int odp_packet_has_vlan_qinq(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.vlan_qinq;
+}
+
+_ODP_INLINE int odp_packet_has_arp(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.arp;
+}
+
+_ODP_INLINE int odp_packet_has_ipv4(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ipv4;
+}
+
+_ODP_INLINE int odp_packet_has_ipv6(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ipv6;
+}
+
+_ODP_INLINE int odp_packet_has_ip_bcast(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ip_bcast;
+}
+
+_ODP_INLINE int odp_packet_has_ip_mcast(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ip_mcast;
+}
+
+_ODP_INLINE int odp_packet_has_ipfrag(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ipfrag;
+}
+
+_ODP_INLINE int odp_packet_has_ipopt(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.ipopt;
+}
+
+_ODP_INLINE int odp_packet_has_udp(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.udp;
+}
+
+_ODP_INLINE int odp_packet_has_tcp(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.tcp;
+}
+
+_ODP_INLINE int odp_packet_has_sctp(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.sctp;
+}
+
+_ODP_INLINE int odp_packet_has_icmp(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t flags;
+
+ flags.all = _odp_packet_input_flags(pkt);
+ return flags.icmp;
+}
+
+_ODP_INLINE int odp_packet_has_error(odp_packet_t pkt)
+{
+ _odp_packet_flags_t flags;
+
+ flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags);
+ return flags.all.error != 0;
+}
+
+_ODP_INLINE int odp_packet_has_l2_error(odp_packet_t pkt)
+{
+ _odp_packet_flags_t flags;
+
+ flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags);
+
+ /* L2 parsing is always done by default and hence
+ no additional check is required. */
+ return flags.snap_len_err;
+}
+
+_ODP_INLINE int odp_packet_has_l3_error(odp_packet_t pkt)
+{
+ _odp_packet_flags_t flags;
+
+ flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags);
+
+ return flags.ip_err;
+}
+
+_ODP_INLINE int odp_packet_has_l4_error(odp_packet_t pkt)
+{
+ _odp_packet_flags_t flags;
+
+ flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags);
+
+ return flags.tcp_err | flags.udp_err;
+}
+
/** @endcond */
#endif
diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h
index 66451ce49..126404f2d 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h
@@ -115,7 +115,7 @@ typedef union {
uint32_t all_flags;
struct {
- uint32_t reserved1: 7;
+ uint32_t reserved1: 6;
/*
* Init flags
@@ -133,6 +133,7 @@ typedef union {
uint32_t l4_chksum: 1; /* L4 chksum override */
uint32_t ts_set: 1; /* Set Tx timestamp */
uint32_t tx_compl: 1; /* Tx completion event requested */
+ uint32_t tx_aging: 1; /* Packet aging at Tx requested */
uint32_t shaper_len_adj: 8; /* Adjustment for traffic mgr */
/*
@@ -150,8 +151,8 @@ typedef union {
/* Flag groups */
struct {
- uint32_t reserved2: 7;
- uint32_t other: 17; /* All other flags */
+ uint32_t reserved2: 6;
+ uint32_t other: 18; /* All other flags */
uint32_t error: 8; /* All error flags */
} all;
diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h
index fb63f207b..007c19876 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h
@@ -20,7 +20,7 @@
#include <odp/api/time.h>
#include <odp/api/abi/buffer.h>
-#include <odp/api/abi/event.h>
+#include <odp/api/abi/event_types.h>
#include <odp/api/abi/packet.h>
#include <odp/api/abi/packet_io.h>
@@ -73,6 +73,9 @@
#define odp_packet_subtype __odp_packet_subtype
#define odp_packet_tx_compl_from_event __odp_packet_tx_compl_from_event
#define odp_packet_tx_compl_to_event __odp_packet_tx_compl_to_event
+ #define odp_packet_color __odp_packet_color
+ #define odp_packet_drop_eligible __odp_packet_drop_eligible
+ #define odp_packet_shaper_len_adjust __odp_packet_shaper_len_adjust
#else
#undef _ODP_INLINE
#define _ODP_INLINE
@@ -372,6 +375,33 @@ _ODP_INLINE odp_event_t odp_packet_tx_compl_to_event(odp_packet_tx_compl_t tx_co
return (odp_event_t)(uintptr_t)tx_compl;
}
+_ODP_INLINE odp_packet_color_t odp_packet_color(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t input_flags;
+
+ input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags);
+
+ return (odp_packet_color_t)input_flags.color;
+}
+
+_ODP_INLINE odp_bool_t odp_packet_drop_eligible(odp_packet_t pkt)
+{
+ _odp_packet_input_flags_t input_flags;
+
+ input_flags.all = _odp_pkt_get(pkt, uint64_t, input_flags);
+
+ return !input_flags.nodrop;
+}
+
+_ODP_INLINE int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt)
+{
+ _odp_packet_flags_t flags;
+
+ flags.all_flags = _odp_pkt_get(pkt, uint32_t, flags);
+
+ return (int8_t)flags.shaper_len_adj;
+}
+
/** @endcond */
#endif
diff --git a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h
index 3586aa6d5..54a009a36 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_vector_inlines.h
@@ -14,7 +14,7 @@
#ifndef _ODP_PLAT_PACKET_VECTOR_INLINES_H_
#define _ODP_PLAT_PACKET_VECTOR_INLINES_H_
-#include <odp/api/abi/event.h>
+#include <odp/api/abi/event_types.h>
#include <odp/api/abi/packet_types.h>
#include <odp/api/abi/pool_types.h>
diff --git a/platform/linux-generic/include/odp/api/plat/queue_inline_types.h b/platform/linux-generic/include/odp/api/plat/queue_inline_types.h
index 5ce767a16..4eade3dea 100644
--- a/platform/linux-generic/include/odp/api/plat/queue_inline_types.h
+++ b/platform/linux-generic/include/odp/api/plat/queue_inline_types.h
@@ -12,6 +12,7 @@ extern "C" {
#endif
#include <stdint.h>
+#include <odp/api/event_types.h>
#include <odp/api/queue_types.h>
/** @cond _ODP_HIDE_FROM_DOXYGEN_ */
diff --git a/platform/linux-generic/include/odp/api/plat/time_inlines.h b/platform/linux-generic/include/odp/api/plat/time_inlines.h
index bb2913532..2ffb94c66 100644
--- a/platform/linux-generic/include/odp/api/plat/time_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/time_inlines.h
@@ -1,5 +1,5 @@
/* Copyright (c) 2018, Linaro Limited
- * Copyright (c) 2020-2021, Nokia
+ * Copyright (c) 2020-2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -99,6 +99,7 @@ static inline uint64_t _odp_time_convert_to_ns(odp_time_t time)
#define odp_time_cmp __odp_time_cmp
#define odp_time_diff __odp_time_diff
+ #define odp_time_diff_ns __odp_time_diff_ns
#define odp_time_sum __odp_time_sum
#else
@@ -170,6 +171,15 @@ _ODP_INLINE odp_time_t odp_time_diff(odp_time_t t2, odp_time_t t1)
return time;
}
+_ODP_INLINE uint64_t odp_time_diff_ns(odp_time_t t2, odp_time_t t1)
+{
+ odp_time_t time;
+
+ time.u64 = t2.u64 - t1.u64;
+
+ return odp_time_to_ns(time);
+}
+
_ODP_INLINE odp_time_t odp_time_sum(odp_time_t t1, odp_time_t t2)
{
odp_time_t time;
diff --git a/platform/linux-generic/include/odp/api/plat/timer_inline_types.h b/platform/linux-generic/include/odp/api/plat/timer_inline_types.h
new file mode 100644
index 000000000..e3397c4df
--- /dev/null
+++ b/platform/linux-generic/include/odp/api/plat/timer_inline_types.h
@@ -0,0 +1,37 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_PLAT_TIMER_INLINE_TYPES_H_
+#define ODP_PLAT_TIMER_INLINE_TYPES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+/** @cond _ODP_HIDE_FROM_DOXYGEN_ */
+
+/* Timeout header field accessor */
+#define _odp_timeout_hdr_field(hdr, cast, field) \
+ (*(cast *)(uintptr_t)((uint8_t *)hdr + \
+ _odp_timeout_inline_offset.field))
+
+/* Timeout header field offsets for inline functions */
+typedef struct _odp_timeout_inline_offset_t {
+ uint16_t expiration;
+ uint16_t timer;
+ uint16_t user_ptr;
+
+} _odp_timeout_inline_offset_t;
+
+/** @endcond */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/include/odp/api/plat/timer_inlines.h b/platform/linux-generic/include/odp/api/plat/timer_inlines.h
new file mode 100644
index 000000000..406aefdf6
--- /dev/null
+++ b/platform/linux-generic/include/odp/api/plat/timer_inlines.h
@@ -0,0 +1,78 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_PLAT_TIMER_INLINES_H_
+#define ODP_PLAT_TIMER_INLINES_H_
+
+#include <odp/api/event.h>
+#include <odp/api/timer_types.h>
+
+#include <odp/api/plat/timer_inline_types.h>
+
+#include <stdint.h>
+
+/** @cond _ODP_HIDE_FROM_DOXYGEN_ */
+
+extern const _odp_timeout_inline_offset_t _odp_timeout_inline_offset;
+
+#ifndef _ODP_NO_INLINE
+ /* Inline functions by default */
+ #define _ODP_INLINE static inline
+ #define odp_timeout_timer __odp_timeout_timer
+ #define odp_timeout_tick __odp_timeout_tick
+ #define odp_timeout_user_ptr __odp_timeout_user_ptr
+ #define odp_timer_tick_to_ns __odp_timer_tick_to_ns
+ #define odp_timer_ns_to_tick __odp_timer_ns_to_tick
+ #define odp_timeout_from_event __odp_timeout_from_event
+ #define odp_timeout_to_event __odp_timeout_to_event
+#else
+ #define _ODP_INLINE
+#endif
+
+_ODP_INLINE odp_timer_t odp_timeout_timer(odp_timeout_t tmo)
+{
+ return _odp_timeout_hdr_field(tmo, odp_timer_t, timer);
+}
+
+_ODP_INLINE uint64_t odp_timeout_tick(odp_timeout_t tmo)
+{
+ return _odp_timeout_hdr_field(tmo, uint64_t, expiration);
+}
+
+_ODP_INLINE void *odp_timeout_user_ptr(odp_timeout_t tmo)
+{
+ return _odp_timeout_hdr_field(tmo, void *, user_ptr);
+}
+
+_ODP_INLINE uint64_t odp_timer_tick_to_ns(odp_timer_pool_t tp, uint64_t ticks)
+{
+ (void)tp;
+
+ /* Timer ticks in API are nsec */
+ return ticks;
+}
+
+_ODP_INLINE uint64_t odp_timer_ns_to_tick(odp_timer_pool_t tp, uint64_t ns)
+{
+ (void)tp;
+
+ /* Timer ticks in API are nsec */
+ return ns;
+}
+
+_ODP_INLINE odp_timeout_t odp_timeout_from_event(odp_event_t ev)
+{
+ return (odp_timeout_t)ev;
+}
+
+_ODP_INLINE odp_event_t odp_timeout_to_event(odp_timeout_t tmo)
+{
+ return (odp_event_t)tmo;
+}
+
+/** @endcond */
+
+#endif
diff --git a/platform/linux-generic/include/odp_atomic_internal.h b/platform/linux-generic/include/odp_atomic_internal.h
index d98e0e6be..da79b3723 100644
--- a/platform/linux-generic/include/odp_atomic_internal.h
+++ b/platform/linux-generic/include/odp_atomic_internal.h
@@ -20,6 +20,7 @@
#include <odp/api/align.h>
#include <odp/api/hints.h>
#include <odp/api/atomic.h>
+#include <odp_types_internal.h>
#include <stdbool.h>
#ifdef __cplusplus
@@ -137,12 +138,10 @@ static inline void _odp_atomic_flag_clear(_odp_atomic_flag_t *flag)
#endif
#ifdef ODP_ATOMIC_U128
-/** An unsigned 128-bit (16-byte) scalar type */
-__extension__ typedef __int128 _uint128_t;
/** Atomic 128-bit type */
typedef struct ODP_ALIGNED(16) {
- _uint128_t v; /**< Actual storage for the atomic variable */
+ _odp_u128_t v; /**< Actual storage for the atomic variable */
} _odp_atomic_u128_t;
/**
@@ -154,8 +153,8 @@ typedef struct ODP_ALIGNED(16) {
* @param mmodel Memory model associated with the exchange operation
*/
static inline void _odp_atomic_u128_xchg_mm(_odp_atomic_u128_t *ptr,
- _uint128_t *val,
- _uint128_t *old,
+ _odp_u128_t *val,
+ _odp_u128_t *old,
_odp_memmodel_t mm)
{
__atomic_exchange(&ptr->v, val, old, mm);
@@ -177,8 +176,8 @@ static inline void _odp_atomic_u128_xchg_mm(_odp_atomic_u128_t *ptr,
* @retval 0 exchange failed and '*exp' updated with current value
*/
static inline int _odp_atomic_u128_cmp_xchg_mm(_odp_atomic_u128_t *ptr,
- _uint128_t *exp,
- _uint128_t *val,
+ _odp_u128_t *exp,
+ _odp_u128_t *val,
_odp_memmodel_t succ,
_odp_memmodel_t fail)
{
diff --git a/platform/linux-generic/include/odp_classification_datamodel.h b/platform/linux-generic/include/odp_classification_datamodel.h
index cc0e7f081..6b50fef68 100644
--- a/platform/linux-generic/include/odp_classification_datamodel.h
+++ b/platform/linux-generic/include/odp_classification_datamodel.h
@@ -129,6 +129,7 @@ typedef struct pmr_term_value {
Class Of Service
*/
struct cos_s {
+ odp_cos_action_t action; /* Action */
odp_queue_t queue; /* Associated Queue */
odp_pool_t pool; /* Associated Buffer pool */
odp_pktin_vector_config_t vector; /* Packet vector config */
@@ -145,10 +146,11 @@ struct cos_s {
odp_queue_param_t queue_param;
char name[ODP_COS_NAME_LEN]; /* name */
uint8_t index;
+ odp_bool_t stats_enable;
struct {
odp_atomic_u64_t discards;
odp_atomic_u64_t packets;
- } stats[CLS_COS_QUEUE_MAX];
+ } stats, queue_stats[CLS_COS_QUEUE_MAX];
};
typedef union cos_u {
diff --git a/platform/linux-generic/include/odp_classification_internal.h b/platform/linux-generic/include/odp_classification_internal.h
index bc0a12f8f..c20d24c58 100644
--- a/platform/linux-generic/include/odp_classification_internal.h
+++ b/platform/linux-generic/include/odp_classification_internal.h
@@ -67,9 +67,9 @@ static inline void _odp_cos_queue_stats_add(cos_t *cos, odp_queue_t queue,
}
if (packets)
- odp_atomic_add_u64(&cos->s.stats[queue_idx].packets, packets);
+ odp_atomic_add_u64(&cos->s.queue_stats[queue_idx].packets, packets);
if (discards)
- odp_atomic_add_u64(&cos->s.stats[queue_idx].discards, discards);
+ odp_atomic_add_u64(&cos->s.queue_stats[queue_idx].discards, discards);
}
/** Classification Internal function **/
@@ -84,8 +84,7 @@ This function calls Classifier module internal functions for a given packet and
selects destination queue and packet pool based on selected PMR and CoS.
**/
int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base,
- uint16_t pkt_len, uint32_t seg_len, odp_pool_t *pool,
- odp_packet_hdr_t *pkt_hdr, odp_bool_t parse);
+ odp_pool_t *pool, odp_packet_hdr_t *pkt_hdr);
/**
Packet IO classifier init
diff --git a/platform/linux-generic/include/odp_llqueue.h b/platform/linux-generic/include/odp_llqueue.h
index e9cf9945e..68325624a 100644
--- a/platform/linux-generic/include/odp_llqueue.h
+++ b/platform/linux-generic/include/odp_llqueue.h
@@ -15,6 +15,7 @@
#include <odp_config_internal.h>
#include <odp_debug_internal.h>
+#include <odp_types_internal.h>
#include <odp_cpu.h>
#include <stdint.h>
@@ -49,7 +50,7 @@ static odp_bool_t llq_on_queue(struct llnode *node);
typedef uint64_t dintptr_t;
#endif
#if __SIZEOF_PTRDIFF_T__ == 8
-__extension__ typedef __int128 dintptr_t;
+typedef _odp_u128_t dintptr_t;
#endif
struct llnode {
diff --git a/platform/linux-generic/include/odp_macros_internal.h b/platform/linux-generic/include/odp_macros_internal.h
index 997e0fd5b..b8be7f938 100644
--- a/platform/linux-generic/include/odp_macros_internal.h
+++ b/platform/linux-generic/include/odp_macros_internal.h
@@ -35,6 +35,8 @@ extern "C" {
tmp_a > tmp_b ? tmp_a : tmp_b; \
})
+#define MAX3(a, b, c) (MAX(MAX((a), (b)), (c)))
+
#define odp_container_of(pointer, type, member) \
((type *)(void *)(((char *)pointer) - offsetof(type, member)))
diff --git a/platform/linux-generic/include/odp_packet_dpdk.h b/platform/linux-generic/include/odp_packet_dpdk.h
index 3add83f19..b326000e6 100644
--- a/platform/linux-generic/include/odp_packet_dpdk.h
+++ b/platform/linux-generic/include/odp_packet_dpdk.h
@@ -46,18 +46,4 @@ int _odp_dpdk_packet_parse_common(packet_parser_t *pkt_hdr,
uint32_t supported_ptypes,
odp_pktin_config_opt_t pktin_cfg);
-static inline int _odp_dpdk_packet_parse_layer(odp_packet_hdr_t *pkt_hdr,
- struct rte_mbuf *mbuf,
- odp_proto_layer_t layer,
- uint32_t supported_ptypes,
- odp_pktin_config_opt_t pktin_cfg)
-{
- uint32_t seg_len = pkt_hdr->seg_len;
- void *base = pkt_hdr->seg_data;
-
- return _odp_dpdk_packet_parse_common(&pkt_hdr->p, base,
- pkt_hdr->frame_len, seg_len, mbuf,
- layer, supported_ptypes,
- pktin_cfg);
-}
#endif
diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h
index 5a29210dd..a8f58146a 100644
--- a/platform/linux-generic/include/odp_packet_internal.h
+++ b/platform/linux-generic/include/odp_packet_internal.h
@@ -38,15 +38,23 @@ extern "C" {
#include <stdint.h>
#include <string.h>
-/** Minimum segment length expected by _odp_packet_parse_common() */
-#define PACKET_PARSE_SEG_LEN 96
-
ODP_STATIC_ASSERT(sizeof(_odp_packet_input_flags_t) == sizeof(uint64_t),
"INPUT_FLAGS_SIZE_ERROR");
ODP_STATIC_ASSERT(sizeof(_odp_packet_flags_t) == sizeof(uint32_t),
"PACKET_FLAGS_SIZE_ERROR");
+/* Packet extra data length */
+#define PKT_EXTRA_LEN 128
+
+/* Packet extra data types */
+#define PKT_EXTRA_TYPE_DPDK 1
+
+/* Maximum number of segments per packet */
+#define PKT_MAX_SEGS 255
+
+ODP_STATIC_ASSERT(PKT_MAX_SEGS < UINT16_MAX, "PACKET_MAX_SEGS_ERROR");
+
/**
* Packet parser metadata
*/
@@ -67,17 +75,6 @@ typedef struct {
uint16_t l4_offset;
} packet_parser_t;
-/* Packet extra data length */
-#define PKT_EXTRA_LEN 128
-
-/* Packet extra data types */
-#define PKT_EXTRA_TYPE_DPDK 1
-
-/* Maximum number of segments per packet */
-#define PKT_MAX_SEGS 255
-
-ODP_STATIC_ASSERT(PKT_MAX_SEGS < UINT16_MAX, "PACKET_MAX_SEGS_ERROR");
-
/**
* Internal Packet header
*
@@ -146,6 +143,10 @@ typedef struct ODP_ALIGNED_CACHE odp_packet_hdr_t {
/* Max payload size in a LSO segment */
uint16_t lso_max_payload;
+ /* Packet aging drop timeout before enqueue. Once enqueued holds the maximum age (time of
+ * request + requested drop timeout). */
+ uint64_t tx_aging_ns;
+
/* LSO profile index */
uint8_t lso_profile_idx;
@@ -194,6 +195,11 @@ static inline odp_packet_t packet_from_event_hdr(_odp_event_hdr_t *event_hdr)
return (odp_packet_t)(uintptr_t)event_hdr;
}
+static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr)
+{
+ return pkt_hdr->seg_len;
+}
+
static inline odp_packet_hdr_t *packet_last_seg(odp_packet_hdr_t *hdr)
{
while (hdr->seg_next != NULL)
@@ -400,11 +406,6 @@ static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len)
int _odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len,
odp_packet_t pkt[], int max_num);
-/* Perform packet parse up to a given protocol layer */
-int _odp_packet_parse_layer(odp_packet_hdr_t *pkt_hdr,
- odp_proto_layer_t layer,
- odp_proto_chksums_t chksums);
-
/* Reset parser metadata for a new parse */
static inline void packet_parse_reset(odp_packet_hdr_t *pkt_hdr, int all)
{
@@ -454,12 +455,6 @@ static inline void packet_set_ts(odp_packet_hdr_t *pkt_hdr, odp_time_t *ts)
}
}
-int _odp_packet_parse_common(packet_parser_t *pkt_hdr, const uint8_t *ptr,
- uint32_t pkt_len, uint32_t seg_len, int layer,
- odp_proto_chksums_t chksums);
-
-int _odp_cls_parse(odp_packet_hdr_t *pkt_hdr, const uint8_t *parseptr);
-
int _odp_packet_set_data(odp_packet_t pkt, uint32_t offset,
uint8_t c, uint32_t len);
@@ -471,6 +466,9 @@ int _odp_packet_tcp_chksum_insert(odp_packet_t pkt);
int _odp_packet_udp_chksum_insert(odp_packet_t pkt);
int _odp_packet_sctp_chksum_insert(odp_packet_t pkt);
+int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
+ odp_proto_chksums_t chksums, uint64_t l4_part_sum);
+
#ifdef __cplusplus
}
#endif
diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h
index 3e9f94100..aed6de412 100644
--- a/platform/linux-generic/include/odp_packet_io_internal.h
+++ b/platform/linux-generic/include/odp_packet_io_internal.h
@@ -78,6 +78,7 @@ struct pktio_entry {
/* These two locks together lock the whole pktio device */
odp_ticketlock_t rxl; /**< RX ticketlock */
odp_ticketlock_t txl; /**< TX ticketlock */
+ odp_proto_layer_t parse_layer;
uint16_t pktin_frame_offset;
struct {
@@ -93,6 +94,8 @@ struct pktio_entry {
uint8_t tx_ts : 1;
/* Tx completion events */
uint8_t tx_compl : 1;
+ /* Packet aging */
+ uint8_t tx_aging : 1;
};
};
} enabled;
@@ -144,8 +147,8 @@ struct pktio_entry {
/* Storage for queue handles
* Multi-queue support is pktio driver specific */
- unsigned num_in_queue;
- unsigned num_out_queue;
+ uint32_t num_in_queue;
+ uint32_t num_out_queue;
struct {
odp_queue_t queue;
@@ -227,8 +230,8 @@ typedef struct pktio_if_ops {
int num);
int (*recv_tmo)(pktio_entry_t *entry, int index, odp_packet_t packets[],
int num, uint64_t wait_usecs);
- int (*recv_mq_tmo)(pktio_entry_t *entry[], int index[], int num_q,
- odp_packet_t packets[], int num, unsigned *from,
+ int (*recv_mq_tmo)(pktio_entry_t *entry[], int index[], uint32_t num_q,
+ odp_packet_t packets[], int num, uint32_t *from,
uint64_t wait_usecs);
int (*fd_set)(pktio_entry_t *entry, int index, fd_set *readfds);
int (*send)(pktio_entry_t *entry, int index,
@@ -291,6 +294,11 @@ static inline int _odp_pktio_tx_compl_enabled(const pktio_entry_t *entry)
return entry->s.enabled.tx_compl;
}
+static inline int _odp_pktio_tx_aging_enabled(pktio_entry_t *entry)
+{
+ return entry->s.enabled.tx_aging;
+}
+
static inline void _odp_pktio_tx_ts_set(pktio_entry_t *entry)
{
odp_time_t ts_val = odp_time_global();
@@ -326,7 +334,7 @@ extern const pktio_if_ops_t * const _odp_pktio_if_ops[];
* @return <0 on failure
*/
int _odp_sock_recv_mq_tmo_try_int_driven(const struct odp_pktin_queue_t queues[],
- unsigned int num_q, unsigned int *from,
+ uint32_t num_q, uint32_t *from,
odp_packet_t packets[], int num,
uint64_t usecs,
int *trial_successful);
diff --git a/platform/linux-generic/include/odp_parse_internal.h b/platform/linux-generic/include/odp_parse_internal.h
new file mode 100644
index 000000000..8aa5e118b
--- /dev/null
+++ b/platform/linux-generic/include/odp_parse_internal.h
@@ -0,0 +1,112 @@
+/* Copyright (c) 2013-2018, Linaro Limited
+ * Copyright (c) 2019-2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_PARSE_INTERNAL_H_
+#define ODP_PARSE_INTERNAL_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp_packet_internal.h>
+#include <odp_macros_internal.h>
+#include <protocols/eth.h>
+#include <protocols/ip.h>
+#include <protocols/sctp.h>
+#include <protocols/tcp.h>
+#include <protocols/udp.h>
+#include <odp/api/plat/packet_inline_types.h>
+#include <odp/api/hints.h>
+#include <odp/api/packet_io.h>
+#include <odp/api/packet_types.h>
+#include <stdint.h>
+
+/*
+ * In the worst case we look at the Ethernet header, 8 bytes of LLC/SNAP
+ * header and two VLAN tags in the same packet.
+ */
+#define PARSE_ETH_BYTES (sizeof(_odp_ethhdr_t) + 8 + 2 * sizeof(_odp_vlanhdr_t))
+#define PARSE_IPV4_BYTES (0xfU * 4) /* max IPv4 header length with options */
+/*
+ * Peek 2 bytes beyond IPv6 base header without length check if there are
+ * extension headers.
+ */
+#define PARSE_IPV6_BYTES (sizeof(_odp_ipv6hdr_t) + 2)
+#define PARSE_TCP_BYTES (sizeof(_odp_tcphdr_t))
+/*
+ * In the worst case we look at the UDP header and 4 bytes of the UDP
+ * payload (the non-ESP marker to distinguish IKE packets from ESP packets).
+ */
+#define PARSE_UDP_BYTES (sizeof(_odp_udphdr_t) + 4)
+#define PARSE_SCTP_BYTES (sizeof(_odp_sctphdr_t))
+
+/* _odp_packet_parse_common_l3_l4() requires up to this many bytes. */
+#define PARSE_L3_L4_BYTES (MAX(PARSE_IPV4_BYTES, PARSE_IPV6_BYTES) + \
+ MAX3(PARSE_TCP_BYTES, PARSE_UDP_BYTES, PARSE_SCTP_BYTES))
+
+/* _odp_packet_parse_common() requires up to this many bytes. */
+#define PARSE_BYTES (PARSE_ETH_BYTES + PARSE_L3_L4_BYTES)
+
+uint16_t _odp_parse_eth(packet_parser_t *prs, const uint8_t **parseptr,
+ uint32_t *offset, uint32_t frame_len);
+
+/*
+ * Parse common L3 and L4 packet headers up to given layer
+ *
+ * See _odp_packet_parse_common(). Requires up to PARSE_L3_L4_BYTES bytes of
+ * contiguous packet data.
+ */
+int _odp_packet_parse_common_l3_l4(packet_parser_t *prs,
+ const uint8_t *parseptr, uint32_t offset,
+ uint32_t frame_len, uint32_t seg_len,
+ int layer, uint16_t ethtype,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum,
+ odp_pktin_config_opt_t opt);
+
+/**
+ * Parse common packet headers up to given layer
+ *
+ * Requires up to PARSE_BYTES bytes of contiguous packet data. Also parse
+ * metadata must be already initialized.
+ *
+ * Returns 0 on success, 1 on packet errors, and -1 if the packet should be
+ * dropped.
+ */
+static inline int _odp_packet_parse_common(packet_parser_t *prs,
+ const uint8_t *ptr,
+ uint32_t frame_len, uint32_t seg_len,
+ int layer,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum,
+ odp_pktin_config_opt_t opt)
+{
+ uint32_t offset;
+ uint16_t ethtype;
+ const uint8_t *parseptr;
+
+ parseptr = ptr;
+ offset = 0;
+
+ if (odp_unlikely(layer == ODP_PROTO_LAYER_NONE))
+ return 0;
+
+ /* Assume valid L2 header, no CRC/FCS check in SW */
+ prs->l2_offset = offset;
+
+ ethtype = _odp_parse_eth(prs, &parseptr, &offset, frame_len);
+
+ return _odp_packet_parse_common_l3_l4(prs, parseptr, offset, frame_len,
+ seg_len, layer, ethtype, chksums,
+ l4_part_sum, opt);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h
index ba4ad53e8..cec9c8bb8 100644
--- a/platform/linux-generic/include/odp_schedule_if.h
+++ b/platform/linux-generic/include/odp_schedule_if.h
@@ -89,8 +89,6 @@ extern const schedule_fn_t *_odp_sched_fn;
/* Interface for the scheduler */
int _odp_sched_cb_pktin_poll(int pktio_index, int pktin_index,
_odp_event_hdr_t *hdr_tbl[], int num);
-/* evts must have at least QUEUE_MULTI_MAX elements */
-int _odp_sched_cb_pktin_poll_one(int pktio_index, int rx_queue, odp_event_t evts[]);
void _odp_sched_cb_pktio_stop_finalize(int pktio_index);
/* API functions */
diff --git a/platform/linux-generic/include/odp_types_internal.h b/platform/linux-generic/include/odp_types_internal.h
new file mode 100644
index 000000000..a97ac9cd4
--- /dev/null
+++ b/platform/linux-generic/include/odp_types_internal.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_TYPES_INTERNAL_H_
+#define ODP_TYPES_INTERNAL_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef __SIZEOF_INT128__
+
+__extension__ typedef unsigned __int128 _odp_u128_t;
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c
index 3b50232d6..2fdecbc51 100644
--- a/platform/linux-generic/odp_classification.c
+++ b/platform/linux-generic/odp_classification.c
@@ -150,7 +150,7 @@ void odp_cls_pmr_param_init(odp_pmr_param_t *param)
int odp_cls_capability(odp_cls_capability_t *capability)
{
- unsigned int count = 0;
+ uint32_t count = 0;
memset(capability, 0, sizeof(odp_cls_capability_t));
@@ -161,6 +161,7 @@ int odp_cls_capability(odp_cls_capability_t *capability)
capability->max_pmr_terms = CLS_PMR_MAX_ENTRY;
capability->available_pmr_terms = count;
capability->max_cos = CLS_COS_MAX_ENTRY;
+ capability->max_cos_stats = capability->max_cos;
capability->pmr_range_supported = false;
capability->supported_terms.all_bits = 0;
capability->supported_terms.bit.len = 1;
@@ -186,6 +187,8 @@ int odp_cls_capability(odp_cls_capability_t *capability)
capability->threshold_bp.all_bits = 0;
capability->max_hash_queues = CLS_COS_QUEUE_MAX;
capability->max_mark = MAX_MARK;
+ capability->stats.cos.counter.discards = 1;
+ capability->stats.cos.counter.packets = 1;
capability->stats.queue.counter.discards = 1;
capability->stats.queue.counter.packets = 1;
@@ -220,21 +223,29 @@ static inline void _cls_queue_unwind(uint32_t tbl_index, uint32_t j)
odp_queue_destroy(queue_grp_tbl->s.queue[tbl_index + --j]);
}
-odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param)
+odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param_in)
{
uint32_t i, j;
odp_queue_t queue;
odp_cls_drop_t drop_policy;
cos_t *cos;
uint32_t tbl_index;
+ odp_cls_cos_param_t param = *param_in;
+
+ if (param.action == ODP_COS_ACTION_DROP) {
+ param.num_queue = 1;
+ param.queue = ODP_QUEUE_INVALID;
+ param.pool = ODP_POOL_INVALID;
+ param.vector.enable = false;
+ }
/* num_queue should not be zero */
- if (param->num_queue > CLS_COS_QUEUE_MAX || param->num_queue < 1)
+ if (param.num_queue > CLS_COS_QUEUE_MAX || param.num_queue < 1)
return ODP_COS_INVALID;
/* Validate packet vector parameters */
- if (param->vector.enable) {
- odp_pool_t pool = param->vector.pool;
+ if (param.vector.enable) {
+ odp_pool_t pool = param.vector.pool;
odp_pool_info_t pool_info;
if (pool == ODP_POOL_INVALID || odp_pool_info(pool, &pool_info)) {
@@ -245,17 +256,17 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param)
ODP_ERR("wrong pool type\n");
return ODP_COS_INVALID;
}
- if (param->vector.max_size == 0) {
+ if (param.vector.max_size == 0) {
ODP_ERR("vector.max_size is zero\n");
return ODP_COS_INVALID;
}
- if (param->vector.max_size > pool_info.params.vector.max_size) {
+ if (param.vector.max_size > pool_info.params.vector.max_size) {
ODP_ERR("vector.max_size larger than pool max vector size\n");
return ODP_COS_INVALID;
}
}
- drop_policy = param->drop_policy;
+ drop_policy = param.drop_policy;
for (i = 0; i < CLS_COS_MAX_ENTRY; i++) {
cos = &cos_tbl->cos_entry[i];
@@ -274,16 +285,16 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param)
cos->s.linked_cos[j] = NULL;
}
- cos->s.num_queue = param->num_queue;
+ cos->s.num_queue = param.num_queue;
- if (param->num_queue > 1) {
+ if (param.num_queue > 1) {
odp_queue_param_init(&cos->s.queue_param);
cos->s.queue_group = true;
cos->s.queue = ODP_QUEUE_INVALID;
_odp_cls_update_hash_proto(cos,
- param->hash_proto);
+ param.hash_proto);
tbl_index = i * CLS_COS_QUEUE_MAX;
- for (j = 0; j < param->num_queue; j++) {
+ for (j = 0; j < param.num_queue; j++) {
queue = odp_queue_create(NULL, &cos->s.queue_param);
if (queue == ODP_QUEUE_INVALID) {
/* unwind the queues */
@@ -296,21 +307,27 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param)
}
} else {
- cos->s.queue = param->queue;
+ cos->s.queue = param.queue;
}
+
+ odp_atomic_init_u64(&cos->s.stats.discards, 0);
+ odp_atomic_init_u64(&cos->s.stats.packets, 0);
+
/* Initialize statistics counters */
for (j = 0; j < cos->s.num_queue; j++) {
- odp_atomic_init_u64(&cos->s.stats[j].discards, 0);
- odp_atomic_init_u64(&cos->s.stats[j].packets, 0);
+ odp_atomic_init_u64(&cos->s.queue_stats[j].discards, 0);
+ odp_atomic_init_u64(&cos->s.queue_stats[j].packets, 0);
}
- cos->s.pool = param->pool;
+ cos->s.action = param.action;
+ cos->s.pool = param.pool;
cos->s.headroom = 0;
cos->s.valid = 1;
cos->s.drop_policy = drop_policy;
odp_atomic_init_u32(&cos->s.num_rule, 0);
cos->s.index = i;
- cos->s.vector = param->vector;
+ cos->s.vector = param.vector;
+ cos->s.stats_enable = param.stats_enable;
UNLOCK(&cos->s.lock);
return _odp_cos_from_ndx(i);
}
@@ -1461,6 +1478,9 @@ static cos_t *match_pmr_cos(cos_t *cos, const uint8_t *pkt_addr, pmr_t *pmr,
/* PMR matched */
pmr_debug_print(pmr, cos);
+ if (cos->s.stats_enable)
+ odp_atomic_inc_u64(&cos->s.stats.packets);
+
hdr->p.input_flags.cls_mark = 0;
if (pmr->s.mark) {
hdr->p.input_flags.cls_mark = 1;
@@ -1537,8 +1557,11 @@ static inline cos_t *cls_select_cos(pktio_entry_t *entry,
default_cos = cls->default_cos;
/* Return error cos for error packet */
- if (pkt_hdr->p.flags.all.error)
- return cls->error_cos;
+ if (pkt_hdr->p.flags.all.error) {
+ cos = cls->error_cos;
+ goto done;
+ }
+
/* Calls all the PMRs attached at the PKTIO level*/
for (i = 0; i < odp_atomic_load_u32(&default_cos->s.num_rule); i++) {
pmr = default_cos->s.pmr[i];
@@ -1551,11 +1574,17 @@ static inline cos_t *cls_select_cos(pktio_entry_t *entry,
cos = match_qos_cos(entry, pkt_addr, pkt_hdr);
if (cos) {
ODP_DBG_RAW(CLS_DBG, " QoS matched -> cos: %s(%u)\n", cos->s.name, cos->s.index);
- return cos;
+ goto done;
}
ODP_DBG_RAW(CLS_DBG, " No match -> default cos\n");
- return cls->default_cos;
+ cos = cls->default_cos;
+
+done:
+ if (cos && cos->s.stats_enable)
+ odp_atomic_inc_u64(&cos->s.stats.packets);
+
+ return cos;
}
static uint32_t packet_rss_hash(odp_packet_hdr_t *pkt_hdr,
@@ -1575,12 +1604,12 @@ static uint32_t packet_rss_hash(odp_packet_hdr_t *pkt_hdr,
* @retval 0 on success
* @retval -EFAULT Bug
* @retval -EINVAL Config error
+ * @retval -ENOENT Drop action
*
* @note *base is not released
*/
int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base,
- uint16_t pkt_len, uint32_t seg_len, odp_pool_t *pool,
- odp_packet_hdr_t *pkt_hdr, odp_bool_t parse)
+ odp_pool_t *pool, odp_packet_hdr_t *pkt_hdr)
{
cos_t *cos;
uint32_t tbl_index;
@@ -1588,24 +1617,19 @@ int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base,
ODP_DBG_LVL(CLS_DBG, "Classify packet from %s\n", entry->s.full_name);
- if (parse) {
- packet_parse_reset(pkt_hdr, 1);
- packet_set_len(pkt_hdr, pkt_len);
-
- _odp_packet_parse_common(&pkt_hdr->p, base, pkt_len, seg_len,
- ODP_PROTO_LAYER_ALL,
- entry->s.in_chksums);
- }
cos = cls_select_cos(entry, base, pkt_hdr);
if (cos == NULL)
return -EINVAL;
+ if (cos->s.action == ODP_COS_ACTION_DROP)
+ return -ENOENT;
+
if (cos->s.queue == ODP_QUEUE_INVALID && cos->s.num_queue == 1)
- return -EFAULT;
+ goto error;
if (cos->s.pool == ODP_POOL_INVALID)
- return -EFAULT;
+ goto error;
*pool = cos->s.pool;
pkt_hdr->p.input_flags.dst_queue = 1;
@@ -1623,6 +1647,10 @@ int _odp_cls_classify_packet(pktio_entry_t *entry, const uint8_t *base,
cos->s.num_queue);
pkt_hdr->dst_queue = queue_grp_tbl->s.queue[tbl_index];
return 0;
+
+error:
+ odp_atomic_inc_u64(&cos->s.stats.discards);
+ return -EFAULT;
}
static uint32_t packet_rss_hash(odp_packet_hdr_t *pkt_hdr,
@@ -1780,6 +1808,27 @@ uint64_t odp_pmr_to_u64(odp_pmr_t hdl)
return _odp_pri(hdl);
}
+int odp_cls_cos_stats(odp_cos_t hdl, odp_cls_cos_stats_t *stats)
+{
+ cos_t *cos = get_cos_entry(hdl);
+
+ if (odp_unlikely(cos == NULL)) {
+ ODP_ERR("Invalid odp_cos_t handle\n");
+ return -1;
+ }
+
+ if (odp_unlikely(stats == NULL)) {
+ ODP_ERR("Output structure NULL\n");
+ return -1;
+ }
+
+ memset(stats, 0, sizeof(*stats));
+ stats->discards = odp_atomic_load_u64(&cos->s.stats.discards);
+ stats->packets = odp_atomic_load_u64(&cos->s.stats.packets);
+
+ return 0;
+}
+
int odp_cls_queue_stats(odp_cos_t hdl, odp_queue_t queue,
odp_cls_queue_stats_t *stats)
{
@@ -1803,8 +1852,8 @@ int odp_cls_queue_stats(odp_cos_t hdl, odp_queue_t queue,
}
memset(stats, 0, sizeof(odp_cls_queue_stats_t));
- stats->discards = odp_atomic_load_u64(&cos->s.stats[queue_idx].discards);
- stats->packets = odp_atomic_load_u64(&cos->s.stats[queue_idx].packets);
+ stats->discards = odp_atomic_load_u64(&cos->s.queue_stats[queue_idx].discards);
+ stats->packets = odp_atomic_load_u64(&cos->s.queue_stats[queue_idx].packets);
return 0;
}
diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c
index bc14bad9a..f276d4659 100644
--- a/platform/linux-generic/odp_crypto_null.c
+++ b/platform/linux-generic/odp_crypto_null.c
@@ -269,9 +269,7 @@ int odp_crypto_session_destroy(odp_crypto_session_t session)
return 0;
}
-/*
- * Shim function around packet operation, can be used by other implementations.
- */
+#if ODP_DEPRECATED_API
int
odp_crypto_operation(odp_crypto_op_param_t *param,
odp_bool_t *posted,
@@ -326,6 +324,7 @@ odp_crypto_operation(odp_crypto_op_param_t *param,
return 0;
}
+#endif
int
_odp_crypto_init_global(void)
@@ -402,6 +401,7 @@ int _odp_crypto_term_local(void)
return 0;
}
+#if ODP_DEPRECATED_API
odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev)
{
/* This check not mandated by the API specification */
@@ -438,6 +438,7 @@ uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl)
{
return _odp_pri(hdl);
}
+#endif
void odp_crypto_session_param_init(odp_crypto_session_param_t *param)
{
diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c
index 13101fbca..6ff0ac041 100644
--- a/platform/linux-generic/odp_crypto_openssl.c
+++ b/platform/linux-generic/odp_crypto_openssl.c
@@ -31,6 +31,7 @@
#include <openssl/hmac.h>
#include <openssl/cmac.h>
#include <openssl/evp.h>
+#include <openssl/opensslv.h>
#if (OPENSSL_VERSION_NUMBER >= 0x10100000L) && !defined(OPENSSL_NO_POLY1305)
#define _ODP_HAVE_CHACHA20_POLY1305 1
@@ -38,6 +39,11 @@
#define _ODP_HAVE_CHACHA20_POLY1305 0
#endif
+/* Ignore warnings about APIs deprecated in OpenSSL 3.0 */
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#endif
+
#define MAX_SESSIONS 4000
#define AES_BLOCK_SIZE 16
#define AES_KEY_LENGTH 16
@@ -2495,6 +2501,7 @@ int odp_crypto_session_destroy(odp_crypto_session_t session)
/*
* Shim function around packet operation, can be used by other implementations.
*/
+#if ODP_DEPRECATED_API
int
odp_crypto_operation(odp_crypto_op_param_t *param,
odp_bool_t *posted,
@@ -2549,6 +2556,7 @@ odp_crypto_operation(odp_crypto_op_param_t *param,
return 0;
}
+#endif
#if OPENSSL_VERSION_NUMBER < 0x10100000L
static void ODP_UNUSED openssl_thread_id(CRYPTO_THREADID ODP_UNUSED *id)
@@ -2705,6 +2713,7 @@ int _odp_crypto_term_local(void)
return 0;
}
+#if ODP_DEPRECATED_API
odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev)
{
/* This check not mandated by the API specification */
@@ -2741,6 +2750,7 @@ uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl)
{
return _odp_pri(hdl);
}
+#endif /* ODP_DEPRECATED_API */
void odp_crypto_session_param_init(odp_crypto_session_param_t *param)
{
diff --git a/platform/linux-generic/odp_dma.c b/platform/linux-generic/odp_dma.c
index 87e3b5b73..4c129c616 100644
--- a/platform/linux-generic/odp_dma.c
+++ b/platform/linux-generic/odp_dma.c
@@ -10,6 +10,7 @@
#include <odp/api/align.h>
#include <odp/api/buffer.h>
#include <odp/api/stash.h>
+#include <odp/api/packet.h>
#include <odp/api/pool.h>
#include <odp/api/queue.h>
diff --git a/platform/linux-generic/odp_event.c b/platform/linux-generic/odp_event.c
index 95be9dec2..90c3e3a3c 100644
--- a/platform/linux-generic/odp_event.c
+++ b/platform/linux-generic/odp_event.c
@@ -24,6 +24,7 @@
#include <odp/api/plat/event_inlines.h>
#include <odp/api/plat/packet_inlines.h>
#include <odp/api/plat/packet_vector_inlines.h>
+#include <odp/api/plat/timer_inlines.h>
odp_event_subtype_t odp_event_subtype(odp_event_t event)
{
@@ -70,9 +71,11 @@ void odp_event_free(odp_event_t event)
case ODP_EVENT_TIMEOUT:
odp_timeout_free(odp_timeout_from_event(event));
break;
+#if ODP_DEPRECATED_API
case ODP_EVENT_CRYPTO_COMPL:
odp_crypto_compl_free(odp_crypto_compl_from_event(event));
break;
+#endif
case ODP_EVENT_IPSEC_STATUS:
_odp_ipsec_status_free(_odp_ipsec_status_from_event(event));
break;
@@ -120,8 +123,10 @@ int odp_event_is_valid(odp_event_t event)
/* Fall through */
case ODP_EVENT_TIMEOUT:
/* Fall through */
+#if ODP_DEPRECATED_API
case ODP_EVENT_CRYPTO_COMPL:
/* Fall through */
+#endif
case ODP_EVENT_IPSEC_STATUS:
/* Fall through */
case ODP_EVENT_PACKET_VECTOR:
diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c
index 4dec8f440..29f0e4bcd 100644
--- a/platform/linux-generic/odp_ipsec.c
+++ b/platform/linux-generic/odp_ipsec.c
@@ -1803,6 +1803,7 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt,
state.out_tunnel.ip_tos = 0;
state.out_tunnel.ip_df = 0;
state.out_tunnel.ip_flabel = 0;
+ state.ip_next_hdr = _ODP_IPPROTO_NO_NEXT;
rc = 0;
} else {
rc = -1;
@@ -2269,12 +2270,6 @@ int odp_ipsec_out_inline(const odp_packet_t pkt_in[], int num_in,
ptr) < 0)
status.error.alg = 1;
- packet_subtype_set(pkt, ODP_EVENT_PACKET_IPSEC);
- result = ipsec_pkt_result(pkt);
- memset(result, 0, sizeof(*result));
- result->sa = ipsec_sa->ipsec_sa_hdl;
- result->status = status;
-
if (!status.error.all) {
odp_pktout_queue_t pkqueue;
diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c
index c47e4912e..4e4057ede 100644
--- a/platform/linux-generic/odp_ipsec_sad.c
+++ b/platform/linux-generic/odp_ipsec_sad.c
@@ -863,7 +863,10 @@ ipsec_sa_t *_odp_ipsec_sa_lookup(const ipsec_sa_lookup_t *lookup)
lookup->proto == ipsec_sa->proto &&
lookup->spi == ipsec_sa->spi &&
lookup->ver == ipsec_sa->in.lookup_ver &&
- !memcmp(lookup->dst_addr, &ipsec_sa->in.lookup_dst_ipv4,
+ !memcmp(lookup->dst_addr,
+ lookup->ver == ODP_IPSEC_IPV4 ?
+ (void *)&ipsec_sa->in.lookup_dst_ipv4 :
+ (void *)&ipsec_sa->in.lookup_dst_ipv6,
lookup->ver == ODP_IPSEC_IPV4 ?
_ODP_IPV4ADDR_LEN :
_ODP_IPV6ADDR_LEN)) {
diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c
index b91f8ca70..b0d0fc03c 100644
--- a/platform/linux-generic/odp_packet.c
+++ b/platform/linux-generic/odp_packet.c
@@ -16,6 +16,7 @@
#include <odp/api/proto_stats.h>
#include <odp/api/timer.h>
+#include <odp_parse_internal.h>
#include <odp_chksum_internal.h>
#include <odp_debug_internal.h>
#include <odp_errno_define.h>
@@ -131,11 +132,6 @@ static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr,
return pkt_hdr->seg_len;
}
-static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr)
-{
- return pkt_hdr->seg_len;
-}
-
static inline void *packet_tail(odp_packet_hdr_t *pkt_hdr)
{
odp_packet_hdr_t *last_seg = packet_last_seg(pkt_hdr);
@@ -1771,500 +1767,6 @@ static uint32_t packet_sum_crc32c(odp_packet_hdr_t *pkt_hdr,
return sum;
}
-/*
- * In the worst case we look at the Ethernet header, 8 bytes of LLC/SNAP
- * header and two VLAN tags in the same packet.
- */
-#define PARSE_ETH_BYTES (sizeof(_odp_ethhdr_t) + 8 + 2 * sizeof(_odp_vlanhdr_t))
-/** Parser helper function for Ethernet packets
- *
- * Requires up to PARSE_ETH_BYTES bytes of contiguous packet data.
- */
-static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr,
- uint32_t *offset, uint32_t frame_len)
-{
- uint16_t ethtype;
- const _odp_ethhdr_t *eth;
- uint16_t macaddr0, macaddr2, macaddr4;
- const _odp_vlanhdr_t *vlan;
- _odp_packet_input_flags_t input_flags;
-
- input_flags.all = 0;
- input_flags.l2 = 1;
- input_flags.eth = 1;
-
- eth = (const _odp_ethhdr_t *)*parseptr;
-
- /* Detect jumbo frames */
- if (odp_unlikely(frame_len - *offset > _ODP_ETH_LEN_MAX))
- input_flags.jumbo = 1;
-
- /* Handle Ethernet broadcast/multicast addresses */
- macaddr0 = odp_be_to_cpu_16(*((const uint16_t *)(const void *)eth));
- if (odp_unlikely((macaddr0 & 0x0100) == 0x0100))
- input_flags.eth_mcast = 1;
-
- if (odp_unlikely(macaddr0 == 0xffff)) {
- macaddr2 =
- odp_be_to_cpu_16(*((const uint16_t *)
- (const void *)eth + 1));
- macaddr4 =
- odp_be_to_cpu_16(*((const uint16_t *)
- (const void *)eth + 2));
-
- if ((macaddr2 == 0xffff) && (macaddr4 == 0xffff))
- input_flags.eth_bcast = 1;
- }
-
- /* Get Ethertype */
- ethtype = odp_be_to_cpu_16(eth->type);
- *offset += sizeof(*eth);
- *parseptr += sizeof(*eth);
-
- /* Check for SNAP vs. DIX */
- if (odp_unlikely(ethtype < _ODP_ETH_LEN_MAX)) {
- input_flags.snap = 1;
- if (ethtype > frame_len - *offset) {
- prs->flags.snap_len_err = 1;
- ethtype = 0;
- goto error;
- }
- ethtype = odp_be_to_cpu_16(*((const uint16_t *)(uintptr_t)
- (*parseptr + 6)));
- *offset += 8;
- *parseptr += 8;
- }
-
- /* Parse the VLAN header(s), if present */
- if (odp_unlikely(ethtype == _ODP_ETHTYPE_VLAN_OUTER)) {
- input_flags.vlan_qinq = 1;
- input_flags.vlan = 1;
-
- vlan = (const _odp_vlanhdr_t *)*parseptr;
- ethtype = odp_be_to_cpu_16(vlan->type);
- *offset += sizeof(_odp_vlanhdr_t);
- *parseptr += sizeof(_odp_vlanhdr_t);
- }
-
- if (ethtype == _ODP_ETHTYPE_VLAN) {
- input_flags.vlan = 1;
- vlan = (const _odp_vlanhdr_t *)*parseptr;
- ethtype = odp_be_to_cpu_16(vlan->type);
- *offset += sizeof(_odp_vlanhdr_t);
- *parseptr += sizeof(_odp_vlanhdr_t);
- }
-
- /*
- * The packet was too short for what we parsed. We just give up
- * entirely without trying to parse what fits in the packet.
- */
- if (odp_unlikely(*offset > frame_len)) {
- input_flags.all = 0;
- input_flags.l2 = 1;
- ethtype = 0;
- }
-
-error:
- prs->input_flags.all |= input_flags.all;
-
- return ethtype;
-}
-
-#define PARSE_IPV4_BYTES (0xfU * 4) /* max IPv4 header length with options */
-/**
- * Parser helper function for IPv4
- *
- * Requires up to PARSE_IPV4_BYTES bytes of contiguous packet data.
- */
-static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr,
- uint32_t *offset, uint32_t frame_len,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- const _odp_ipv4hdr_t *ipv4 = (const _odp_ipv4hdr_t *)*parseptr;
- uint32_t dstaddr = odp_be_to_cpu_32(ipv4->dst_addr);
- uint32_t l3_len = odp_be_to_cpu_16(ipv4->tot_len);
- uint16_t frag_offset = odp_be_to_cpu_16(ipv4->frag_offset);
- uint8_t ver = _ODP_IPV4HDR_VER(ipv4->ver_ihl);
- uint8_t ihl = _ODP_IPV4HDR_IHL(ipv4->ver_ihl);
-
- if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN ||
- ver != 4 ||
- sizeof(*ipv4) > frame_len - *offset ||
- (l3_len > frame_len - *offset))) {
- prs->flags.ip_err = 1;
- return 0;
- }
-
- if (chksums.chksum.ipv4) {
- prs->input_flags.l3_chksum_done = 1;
- if (chksum_finalize(chksum_partial(ipv4, ihl * 4, 0)) != 0xffff) {
- prs->flags.ip_err = 1;
- prs->flags.l3_chksum_err = 1;
- return 0;
- }
- }
-
- *offset += ihl * 4;
- *parseptr += ihl * 4;
-
- if (chksums.chksum.udp || chksums.chksum.tcp)
- *l4_part_sum = chksum_partial((const uint8_t *)&ipv4->src_addr,
- 2 * _ODP_IPV4ADDR_LEN, 0);
-
- if (odp_unlikely(ihl > _ODP_IPV4HDR_IHL_MIN))
- prs->input_flags.ipopt = 1;
-
- /* A packet is a fragment if:
- * "more fragments" flag is set (all fragments except the last)
- * OR
- * "fragment offset" field is nonzero (all fragments except the first)
- */
- if (odp_unlikely(_ODP_IPV4HDR_IS_FRAGMENT(frag_offset)))
- prs->input_flags.ipfrag = 1;
-
- /* Handle IPv4 broadcast / multicast */
- if (odp_unlikely(dstaddr == 0xffffffff))
- prs->input_flags.ip_bcast = 1;
-
- if (odp_unlikely((dstaddr >> 28) == 0xe))
- prs->input_flags.ip_mcast = 1;
-
- return ipv4->proto;
-}
-
-/*
- * Peeks 2 bytes beyond IPv6 base header without length check if there
- * are extension headers.
- */
-#define PARSE_IPV6_BYTES (sizeof(_odp_ipv6hdr_t) + 2)
-/**
- * Parser helper function for IPv6
- *
- * Requires at least PARSE_IPV6_BYTES bytes of contiguous packet data.
- */
-static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr,
- uint32_t *offset, uint32_t frame_len,
- uint32_t seg_len,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- const _odp_ipv6hdr_t *ipv6 = (const _odp_ipv6hdr_t *)*parseptr;
- const _odp_ipv6hdr_ext_t *ipv6ext;
- uint32_t dstaddr0 = odp_be_to_cpu_32(ipv6->dst_addr.u8[0]);
- uint32_t l3_len = odp_be_to_cpu_16(ipv6->payload_len) +
- _ODP_IPV6HDR_LEN;
-
- /* Basic sanity checks on IPv6 header */
- if (odp_unlikely((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 ||
- sizeof(*ipv6) > frame_len - *offset ||
- l3_len > frame_len - *offset)) {
- prs->flags.ip_err = 1;
- return 0;
- }
-
- /* IPv6 broadcast / multicast flags */
- prs->input_flags.ip_mcast = (dstaddr0 & 0xff000000) == 0xff000000;
- prs->input_flags.ip_bcast = 0;
-
- /* Skip past IPv6 header */
- *offset += sizeof(_odp_ipv6hdr_t);
- *parseptr += sizeof(_odp_ipv6hdr_t);
-
- if (chksums.chksum.udp || chksums.chksum.tcp)
- *l4_part_sum = chksum_partial((const uint8_t *)&ipv6->src_addr,
- 2 * _ODP_IPV6ADDR_LEN, 0);
-
- /* Skip past any IPv6 extension headers */
- if (ipv6->next_hdr == _ODP_IPPROTO_HOPOPTS ||
- ipv6->next_hdr == _ODP_IPPROTO_ROUTE) {
- prs->input_flags.ipopt = 1;
-
- do {
- ipv6ext = (const _odp_ipv6hdr_ext_t *)*parseptr;
- uint16_t extlen = 8 + ipv6ext->ext_len * 8;
-
- *offset += extlen;
- *parseptr += extlen;
- } while ((ipv6ext->next_hdr == _ODP_IPPROTO_HOPOPTS ||
- ipv6ext->next_hdr == _ODP_IPPROTO_ROUTE) &&
- *offset < seg_len);
-
- if (*offset >= prs->l3_offset +
- odp_be_to_cpu_16(ipv6->payload_len)) {
- prs->flags.ip_err = 1;
- return 0;
- }
-
- if (ipv6ext->next_hdr == _ODP_IPPROTO_FRAG)
- prs->input_flags.ipfrag = 1;
-
- return ipv6ext->next_hdr;
- }
-
- if (odp_unlikely(ipv6->next_hdr == _ODP_IPPROTO_FRAG)) {
- prs->input_flags.ipopt = 1;
- prs->input_flags.ipfrag = 1;
- }
-
- return ipv6->next_hdr;
-}
-
-#define PARSE_TCP_BYTES (sizeof(_odp_tcphdr_t))
-/**
- * Parser helper function for TCP
- *
- * Requires PARSE_TCP_BYTES bytes of contiguous packet data.
- */
-static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr,
- uint16_t tcp_len,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- const _odp_tcphdr_t *tcp = (const _odp_tcphdr_t *)*parseptr;
- uint32_t len = tcp->hl * 4;
-
- if (odp_unlikely(tcp->hl < sizeof(_odp_tcphdr_t) / sizeof(uint32_t)))
- prs->flags.tcp_err = 1;
-
- if (chksums.chksum.tcp &&
- !prs->input_flags.ipfrag) {
- *l4_part_sum += odp_cpu_to_be_16(tcp_len);
-#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
- *l4_part_sum += _ODP_IPPROTO_TCP;
-#else
- *l4_part_sum += _ODP_IPPROTO_TCP << 8;
-#endif
- }
-
- *parseptr += len;
-}
-
-/*
- * In the worst case we look at the UDP header and 4 bytes of the UDP
- * payload (the non-ESP marker to distinguish IKE packets from ESP packets).
- */
-#define PARSE_UDP_BYTES (sizeof(_odp_udphdr_t) + 4)
-/**
- * Parser helper function for UDP
- *
- * Requires PARSE_UDP_BYTES bytes of contiguous packet data.
- */
-static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr;
- uint32_t udplen = odp_be_to_cpu_16(udp->length);
- uint16_t ipsec_port = odp_cpu_to_be_16(_ODP_UDP_IPSEC_PORT);
-
- if (odp_unlikely(udplen < sizeof(_odp_udphdr_t))) {
- prs->flags.udp_err = 1;
- return;
- }
-
- if (chksums.chksum.udp &&
- !prs->input_flags.ipfrag) {
- if (udp->chksum == 0) {
- prs->input_flags.l4_chksum_done = 1;
- prs->flags.l4_chksum_err =
- (prs->input_flags.ipv4 != 1);
- } else {
- *l4_part_sum += udp->length;
-#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
- *l4_part_sum += _ODP_IPPROTO_UDP;
-#else
- *l4_part_sum += _ODP_IPPROTO_UDP << 8;
-#endif
- }
- prs->input_flags.udp_chksum_zero = (udp->chksum == 0);
- }
-
- if (odp_unlikely(ipsec_port == udp->dst_port && udplen > 4)) {
- uint32_t val;
-
- memcpy(&val, udp + 1, 4);
- if (val != 0) {
- prs->input_flags.ipsec = 1;
- prs->input_flags.ipsec_udp = 1;
- }
- }
-
- *parseptr += sizeof(_odp_udphdr_t);
-}
-
-#define PARSE_SCTP_BYTES (sizeof(_odp_sctphdr_t))
-/**
- * Parser helper function for SCTP
- *
- * Requires PARSE_SCTP_BYTES bytes of contiguous packet data.
- */
-static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr,
- uint16_t sctp_len,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- if (odp_unlikely(sctp_len < sizeof(_odp_sctphdr_t))) {
- prs->flags.sctp_err = 1;
- return;
- }
-
- if (chksums.chksum.sctp &&
- !prs->input_flags.ipfrag) {
- const _odp_sctphdr_t *sctp =
- (const _odp_sctphdr_t *)*parseptr;
- uint32_t crc = ~0;
- uint32_t zero = 0;
-
- crc = odp_hash_crc32c(sctp, sizeof(*sctp) - 4, crc);
- crc = odp_hash_crc32c(&zero, 4, crc);
- *l4_part_sum = crc;
- }
-
- *parseptr += sizeof(_odp_sctphdr_t);
-}
-
-#define MAX3(a, b, c) (MAX(MAX((a), (b)), (c)))
-#define PARSE_L3_L4_BYTES (MAX(PARSE_IPV4_BYTES, PARSE_IPV6_BYTES) + \
- MAX3(PARSE_TCP_BYTES, PARSE_UDP_BYTES, PARSE_SCTP_BYTES))
-/* Requires up to PARSE_L3_L4_BYTES bytes of contiguous packet data. */
-static inline
-int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr,
- uint32_t offset,
- uint32_t frame_len, uint32_t seg_len,
- int layer, uint16_t ethtype,
- odp_proto_chksums_t chksums,
- uint64_t *l4_part_sum)
-{
- uint8_t ip_proto;
-
- prs->l3_offset = offset;
-
- if (odp_unlikely(layer <= ODP_PROTO_LAYER_L2))
- return prs->flags.all.error != 0;
-
- /* Set l3 flag only for known ethtypes */
- prs->input_flags.l3 = 1;
-
- /* Parse Layer 3 headers */
- switch (ethtype) {
- case _ODP_ETHTYPE_IPV4:
- prs->input_flags.ipv4 = 1;
- ip_proto = parse_ipv4(prs, &parseptr, &offset, frame_len,
- chksums, l4_part_sum);
- prs->l4_offset = offset;
- break;
-
- case _ODP_ETHTYPE_IPV6:
- prs->input_flags.ipv6 = 1;
- ip_proto = parse_ipv6(prs, &parseptr, &offset, frame_len,
- seg_len, chksums, l4_part_sum);
- prs->l4_offset = offset;
- break;
-
- case _ODP_ETHTYPE_ARP:
- prs->input_flags.arp = 1;
- ip_proto = 255; /* Reserved invalid by IANA */
- break;
-
- default:
- prs->input_flags.l3 = 0;
- ip_proto = 255; /* Reserved invalid by IANA */
- }
-
- if (layer == ODP_PROTO_LAYER_L3)
- return prs->flags.all.error != 0;
-
- /* Set l4 flag only for known ip_proto */
- prs->input_flags.l4 = 1;
-
- /* Parse Layer 4 headers */
- switch (ip_proto) {
- case _ODP_IPPROTO_ICMPV4:
- /* Fall through */
-
- case _ODP_IPPROTO_ICMPV6:
- prs->input_flags.icmp = 1;
- break;
-
- case _ODP_IPPROTO_IPIP:
- /* Do nothing */
- break;
-
- case _ODP_IPPROTO_TCP:
- if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len))
- return -1;
- prs->input_flags.tcp = 1;
- parse_tcp(prs, &parseptr, frame_len - prs->l4_offset, chksums,
- l4_part_sum);
- break;
-
- case _ODP_IPPROTO_UDP:
- if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len))
- return -1;
- prs->input_flags.udp = 1;
- parse_udp(prs, &parseptr, chksums, l4_part_sum);
- break;
-
- case _ODP_IPPROTO_AH:
- prs->input_flags.ipsec = 1;
- prs->input_flags.ipsec_ah = 1;
- break;
-
- case _ODP_IPPROTO_ESP:
- prs->input_flags.ipsec = 1;
- prs->input_flags.ipsec_esp = 1;
- break;
-
- case _ODP_IPPROTO_SCTP:
- prs->input_flags.sctp = 1;
- parse_sctp(prs, &parseptr, frame_len - prs->l4_offset, chksums,
- l4_part_sum);
- break;
-
- case _ODP_IPPROTO_NO_NEXT:
- prs->input_flags.no_next_hdr = 1;
- break;
-
- default:
- prs->input_flags.l4 = 0;
- break;
- }
-
- return prs->flags.all.error != 0;
-}
-
-/**
- * Parse common packet headers up to given layer
- *
- * The function expects at least PACKET_PARSE_SEG_LEN bytes of data to be
- * available from the ptr. Also parse metadata must be already initialized.
- */
-int _odp_packet_parse_common(packet_parser_t *prs, const uint8_t *ptr,
- uint32_t frame_len, uint32_t seg_len,
- int layer, odp_proto_chksums_t chksums)
-{
- uint32_t offset;
- uint16_t ethtype;
- const uint8_t *parseptr;
- uint64_t l4_part_sum;
-
- parseptr = ptr;
- offset = 0;
-
- if (odp_unlikely(layer == ODP_PROTO_LAYER_NONE))
- return 0;
-
- /* Assume valid L2 header, no CRC/FCS check in SW */
- prs->l2_offset = offset;
-
- ethtype = parse_eth(prs, &parseptr, &offset, frame_len);
-
- return packet_parse_common_l3_l4(prs, parseptr, offset, frame_len,
- seg_len, layer, ethtype, chksums,
- &l4_part_sum);
-}
-
static inline int packet_ipv4_chksum(odp_packet_t pkt,
uint32_t offset,
_odp_ipv4hdr_t *ip,
@@ -2442,9 +1944,8 @@ int _odp_packet_sctp_chksum_insert(odp_packet_t pkt)
return odp_packet_copy_from_mem(pkt, pkt_hdr->p.l4_offset + 8, 4, &sum);
}
-static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
- odp_proto_chksums_t chksums,
- uint64_t l4_part_sum)
+int _odp_packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
+ odp_proto_chksums_t chksums, uint64_t l4_part_sum)
{
/* UDP chksum == 0 case is covered in parse_udp() */
if (chksums.chksum.udp &&
@@ -2518,42 +2019,6 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
return pkt_hdr->p.flags.all.error != 0;
}
-/**
- * Simple packet parser
- */
-int _odp_packet_parse_layer(odp_packet_hdr_t *pkt_hdr,
- odp_proto_layer_t layer,
- odp_proto_chksums_t chksums)
-{
- uint32_t seg_len = packet_first_seg_len(pkt_hdr);
- const uint8_t *base = packet_data(pkt_hdr);
- uint32_t offset = 0;
- uint16_t ethtype;
- uint64_t l4_part_sum = 0;
- int rc;
-
- if (odp_unlikely(layer == ODP_PROTO_LAYER_NONE))
- return 0;
-
- /* Assume valid L2 header, no CRC/FCS check in SW */
- pkt_hdr->p.l2_offset = offset;
-
- ethtype = parse_eth(&pkt_hdr->p, &base, &offset, pkt_hdr->frame_len);
-
- rc = packet_parse_common_l3_l4(&pkt_hdr->p, base, offset,
- pkt_hdr->frame_len,
- seg_len, layer, ethtype, chksums,
- &l4_part_sum);
-
- if (rc != 0)
- return rc;
-
- if (layer >= ODP_PROTO_LAYER_L4)
- return packet_l4_chksum(pkt_hdr, chksums, l4_part_sum);
- else
- return 0;
-}
-
int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
const odp_packet_parse_param_t *param)
{
@@ -2568,6 +2033,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
uint64_t l4_part_sum = 0;
const uint32_t min_seglen = PARSE_ETH_BYTES + PARSE_L3_L4_BYTES;
uint8_t buf[min_seglen];
+ odp_pktin_config_opt_t opt;
if (proto == ODP_PROTO_NONE || layer == ODP_PROTO_LAYER_NONE)
return -1;
@@ -2598,7 +2064,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
/* Assume valid L2 header, no CRC/FCS check in SW */
pkt_hdr->p.l2_offset = offset;
- ethtype = parse_eth(&pkt_hdr->p, &data, &offset, packet_len);
+ ethtype = _odp_parse_eth(&pkt_hdr->p, &data, &offset, packet_len);
} else if (proto == ODP_PROTO_IPV4) {
ethtype = _ODP_ETHTYPE_IPV4;
} else if (proto == ODP_PROTO_IPV6) {
@@ -2607,17 +2073,18 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
ethtype = 0; /* Invalid */
}
- ret = packet_parse_common_l3_l4(&pkt_hdr->p, data, offset,
- packet_len, seg_len,
- layer, ethtype,
- param->chksums,
- &l4_part_sum);
+ opt.all_bits = 0;
+
+ ret = _odp_packet_parse_common_l3_l4(&pkt_hdr->p, data, offset,
+ packet_len, seg_len, layer,
+ ethtype, param->chksums,
+ &l4_part_sum, opt);
if (ret)
return -1;
if (layer >= ODP_PROTO_LAYER_L4) {
- ret = packet_l4_chksum(pkt_hdr, param->chksums, l4_part_sum);
+ ret = _odp_packet_l4_chksum(pkt_hdr, param->chksums, l4_part_sum);
if (ret)
return -1;
}
@@ -2879,14 +2346,17 @@ int odp_packet_payload_offset_set(odp_packet_t pkt, uint32_t offset)
void odp_packet_aging_tmo_set(odp_packet_t pkt, uint64_t tmo_ns)
{
- (void)pkt;
- (void)tmo_ns;
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+
+ pkt_hdr->p.flags.tx_aging = tmo_ns ? 1 : 0;
+ pkt_hdr->tx_aging_ns = tmo_ns;
}
uint64_t odp_packet_aging_tmo(odp_packet_t pkt)
{
- (void)pkt;
- return 0;
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+
+ return pkt_hdr->p.flags.tx_aging ? pkt_hdr->tx_aging_ns : 0;
}
int odp_packet_tx_compl_request(odp_packet_t pkt, const odp_packet_tx_compl_opt_t *opt)
diff --git a/platform/linux-generic/odp_packet_flags.c b/platform/linux-generic/odp_packet_flags.c
index bb5c26792..ef542a2cd 100644
--- a/platform/linux-generic/odp_packet_flags.c
+++ b/platform/linux-generic/odp_packet_flags.c
@@ -8,137 +8,11 @@
#include <odp/api/packet_flags.h>
#include <odp_packet_internal.h>
-#define retflag(pkt, x) do { \
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); \
- return pkt_hdr->p.x; \
- } while (0)
-
#define setflag(pkt, x, v) do { \
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); \
pkt_hdr->p.x = (v) & 1; \
} while (0)
-int odp_packet_has_error(odp_packet_t pkt)
-{
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
-
- return pkt_hdr->p.flags.all.error != 0;
-}
-
-/* Get Input Flags */
-
-int odp_packet_has_l2_error(odp_packet_t pkt)
-{
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
- /* L2 parsing is always done by default and hence
- no additional check is required */
- return pkt_hdr->p.flags.snap_len_err;
-}
-
-int odp_packet_has_l3(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.l3);
-}
-
-int odp_packet_has_l3_error(odp_packet_t pkt)
-{
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
-
- return pkt_hdr->p.flags.ip_err;
-}
-
-int odp_packet_has_l4(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.l4);
-}
-
-int odp_packet_has_l4_error(odp_packet_t pkt)
-{
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
-
- return pkt_hdr->p.flags.tcp_err | pkt_hdr->p.flags.udp_err;
-}
-
-int odp_packet_has_eth_bcast(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.eth_bcast);
-}
-
-int odp_packet_has_eth_mcast(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.eth_mcast);
-}
-
-int odp_packet_has_vlan(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.vlan);
-}
-
-int odp_packet_has_vlan_qinq(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.vlan_qinq);
-}
-
-int odp_packet_has_arp(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.arp);
-}
-
-int odp_packet_has_ipv4(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ipv4);
-}
-
-int odp_packet_has_ipv6(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ipv6);
-}
-
-int odp_packet_has_ip_bcast(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ip_bcast);
-}
-
-int odp_packet_has_ip_mcast(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ip_mcast);
-}
-
-int odp_packet_has_ipfrag(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ipfrag);
-}
-
-int odp_packet_has_ipopt(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.ipopt);
-}
-
-int odp_packet_has_udp(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.udp);
-}
-
-int odp_packet_has_tcp(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.tcp);
-}
-
-int odp_packet_has_sctp(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.sctp);
-}
-
-int odp_packet_has_icmp(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.icmp);
-}
-
-odp_packet_color_t odp_packet_color(odp_packet_t pkt)
-{
- retflag(pkt, input_flags.color);
-}
-
void odp_packet_color_set(odp_packet_t pkt, odp_packet_color_t color)
{
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
@@ -146,23 +20,11 @@ void odp_packet_color_set(odp_packet_t pkt, odp_packet_color_t color)
pkt_hdr->p.input_flags.color = color;
}
-odp_bool_t odp_packet_drop_eligible(odp_packet_t pkt)
-{
- odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
-
- return !pkt_hdr->p.input_flags.nodrop;
-}
-
void odp_packet_drop_eligible_set(odp_packet_t pkt, odp_bool_t drop)
{
setflag(pkt, input_flags.nodrop, !drop);
}
-int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt)
-{
- retflag(pkt, flags.shaper_len_adj);
-}
-
void odp_packet_shaper_len_adjust_set(odp_packet_t pkt, int8_t adj)
{
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c
index 899f10425..7c68b10c5 100644
--- a/platform/linux-generic/odp_packet_io.c
+++ b/platform/linux-generic/odp_packet_io.c
@@ -53,6 +53,10 @@
/* Max wait time supported to avoid potential overflow */
#define MAX_WAIT_TIME (UINT64_MAX / 1024)
+/* One hour maximum aging timeout, no real limitations imposed by the implementation other than
+ * integer width, so just use some value. */
+#define MAX_TX_AGING_TMO_NS 3600000000000ULL
+
typedef struct {
const void *user_ptr;
odp_queue_t queue;
@@ -624,6 +628,8 @@ int odp_pktio_config(odp_pktio_t hdl, const odp_pktio_config_t *config)
return -1;
}
+ entry->s.enabled.tx_aging = config->pktout.bit.aging_ena;
+
if (entry->s.ops->config)
res = entry->s.ops->config(entry, config);
@@ -650,6 +656,9 @@ int odp_pktio_start(odp_pktio_t hdl)
ODP_ERR("Already started\n");
return -1;
}
+ entry->s.parse_layer = pktio_cls_enabled(entry) ?
+ ODP_PROTO_LAYER_ALL :
+ entry->s.config.parser.layer;
if (entry->s.ops->start)
res = entry->s.ops->start(entry);
if (!res)
@@ -660,8 +669,8 @@ int odp_pktio_start(odp_pktio_t hdl)
mode = entry->s.param.in_mode;
if (mode == ODP_PKTIN_MODE_SCHED) {
- unsigned int i;
- unsigned int num = entry->s.num_in_queue;
+ uint32_t i;
+ uint32_t num = entry->s.num_in_queue;
int index[num];
odp_queue_t odpq[num];
@@ -1182,95 +1191,6 @@ static int pktin_deq_multi(odp_queue_t queue, _odp_event_hdr_t *event_hdr[],
return nbr;
}
-int _odp_sched_cb_pktin_poll_one(int pktio_index,
- int rx_queue,
- odp_event_t evt_tbl[])
-{
- int num_rx, num_pkts, i;
- pktio_entry_t *entry = pktio_entry_by_index(pktio_index);
- odp_packet_t pkt;
- odp_packet_hdr_t *pkt_hdr;
- odp_pool_t pool = ODP_POOL_INVALID;
- odp_packet_t packets[QUEUE_MULTI_MAX];
- odp_queue_t queue;
- odp_bool_t vector_enabled = entry->s.in_queue[rx_queue].vector.enable;
- uint32_t num = QUEUE_MULTI_MAX;
- cos_t *cos_hdr = NULL;
-
- if (odp_unlikely(entry->s.state != PKTIO_STATE_STARTED)) {
- if (entry->s.state < PKTIO_STATE_ACTIVE ||
- entry->s.state == PKTIO_STATE_STOP_PENDING)
- return -1;
-
- ODP_DBG("interface not started\n");
- return 0;
- }
-
- if (vector_enabled) {
- /* Make sure all packets will fit into a single packet vector */
- if (entry->s.in_queue[rx_queue].vector.max_size < num)
- num = entry->s.in_queue[rx_queue].vector.max_size;
- pool = entry->s.in_queue[rx_queue].vector.pool;
- }
-
- ODP_ASSERT((unsigned int)rx_queue < entry->s.num_in_queue);
- num_pkts = entry->s.ops->recv(entry, rx_queue, packets, num);
-
- num_rx = 0;
- for (i = 0; i < num_pkts; i++) {
- pkt = packets[i];
- pkt_hdr = packet_hdr(pkt);
- if (odp_unlikely(pkt_hdr->p.input_flags.dst_queue)) {
- odp_event_t event = odp_packet_to_event(pkt);
- uint16_t cos_idx = pkt_hdr->cos;
-
- queue = pkt_hdr->dst_queue;
-
- if (cos_idx != CLS_COS_IDX_NONE) {
- /* Packets from classifier */
- cos_hdr = _odp_cos_entry_from_idx(cos_idx);
-
- if (cos_hdr->s.vector.enable) {
- packet_vector_enq_cos(queue, &event, 1, cos_hdr);
- continue;
- }
- } else if (vector_enabled) {
- /* Packets from inline IPsec */
- packet_vector_enq(queue, &event, 1, pool);
- continue;
- }
-
- if (odp_unlikely(odp_queue_enq(queue, event))) {
- /* Queue full? */
- odp_packet_free(pkt);
- if (cos_idx != CLS_COS_IDX_NONE)
- _odp_cos_queue_stats_add(cos_hdr, queue, 0, 1);
- else
- odp_atomic_inc_u64(&entry->s.stats_extra.in_discards);
- } else {
- if (cos_idx != CLS_COS_IDX_NONE)
- _odp_cos_queue_stats_add(cos_hdr, queue, 1, 0);
- }
- } else {
- evt_tbl[num_rx++] = odp_packet_to_event(pkt);
- }
- }
-
- /* Create packet vector */
- if (vector_enabled && num_rx > 0) {
- odp_packet_vector_t pktv = packet_vector_create((odp_packet_t *)evt_tbl,
- num_rx, pool);
-
- if (odp_unlikely(pktv == ODP_PACKET_VECTOR_INVALID))
- return 0;
-
- evt_tbl[0] = odp_packet_vector_to_event(pktv);
- return 1;
- }
-
- return num_rx;
-}
-
int _odp_sched_cb_pktin_poll(int pktio_index, int pktin_index,
_odp_event_hdr_t *hdr_tbl[], int num)
{
@@ -1886,6 +1806,9 @@ int odp_pktio_capability(odp_pktio_t pktio, odp_pktio_capability_t *capa)
capa->tx_compl.queue_type_sched = 1;
capa->tx_compl.queue_type_plain = 1;
capa->tx_compl.mode_all = 1;
+
+ capa->config.pktout.bit.aging_ena = 1;
+ capa->max_tx_aging_tmo_ns = MAX_TX_AGING_TMO_NS;
}
/* Packet vector generation is common for all pktio types */
@@ -2246,8 +2169,7 @@ int odp_pktin_queue_config(odp_pktio_t pktio,
pktio_entry_t *entry;
odp_pktin_mode_t mode;
odp_pktio_capability_t capa;
- unsigned int num_queues;
- unsigned int i;
+ uint32_t num_queues, i;
int rc;
odp_queue_t queue;
odp_pktin_queue_param_t default_param;
@@ -2410,7 +2332,7 @@ int _odp_pktio_pktout_tm_config(odp_pktio_t pktio_hdl,
bool pktio_started = false;
odp_pktout_mode_t mode;
pktio_entry_t *entry;
- unsigned int i;
+ uint32_t i;
int rc = 0;
odp_pktout_queue_param_init(&param);
@@ -2480,8 +2402,7 @@ int odp_pktout_queue_config(odp_pktio_t pktio,
pktio_entry_t *entry;
odp_pktout_mode_t mode;
odp_pktio_capability_t capa;
- unsigned int num_queues;
- unsigned int i;
+ uint32_t num_queues, i;
int rc;
odp_pktout_queue_param_t default_param;
@@ -2847,18 +2768,17 @@ int odp_pktin_recv_tmo(odp_pktin_queue_t queue, odp_packet_t packets[], int num,
}
}
-int odp_pktin_recv_mq_tmo(const odp_pktin_queue_t queues[], unsigned int num_q,
- unsigned int *from, odp_packet_t packets[], int num,
- uint64_t wait)
+int odp_pktin_recv_mq_tmo(const odp_pktin_queue_t queues[], uint32_t num_q, uint32_t *from,
+ odp_packet_t packets[], int num, uint64_t wait)
{
- unsigned int i;
+ uint32_t i;
int ret;
odp_time_t t1, t2;
struct timespec ts;
int started = 0;
uint64_t sleep_round = 0;
int trial_successful = 0;
- unsigned int lfrom = 0;
+ uint32_t lfrom = 0;
for (i = 0; i < num_q; i++) {
ret = odp_pktin_recv(queues[i], packets, num);
diff --git a/platform/linux-generic/odp_parse.c b/platform/linux-generic/odp_parse.c
new file mode 100644
index 000000000..2342a7e49
--- /dev/null
+++ b/platform/linux-generic/odp_parse.c
@@ -0,0 +1,471 @@
+/* Copyright (c) 2013-2018, Linaro Limited
+ * Copyright (c) 2019-2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp_parse_internal.h>
+#include <odp_chksum_internal.h>
+#include <protocols/eth.h>
+#include <protocols/ip.h>
+#include <protocols/sctp.h>
+#include <protocols/tcp.h>
+#include <protocols/udp.h>
+#include <odp/api/hash.h>
+#include <odp/api/packet_io.h>
+#include <odp/api/packet_types.h>
+#include <stdint.h>
+#include <string.h>
+
+/** Parser helper function for Ethernet packets
+ *
+ * Requires up to PARSE_ETH_BYTES bytes of contiguous packet data.
+ */
+uint16_t _odp_parse_eth(packet_parser_t *prs, const uint8_t **parseptr,
+ uint32_t *offset, uint32_t frame_len)
+{
+ uint16_t ethtype;
+ const _odp_ethhdr_t *eth;
+ uint16_t macaddr0, macaddr2, macaddr4;
+ const _odp_vlanhdr_t *vlan;
+ _odp_packet_input_flags_t input_flags;
+
+ input_flags.all = 0;
+ input_flags.l2 = 1;
+ input_flags.eth = 1;
+
+ eth = (const _odp_ethhdr_t *)*parseptr;
+
+ /* Detect jumbo frames */
+ if (odp_unlikely(frame_len - *offset > _ODP_ETH_LEN_MAX))
+ input_flags.jumbo = 1;
+
+ /* Handle Ethernet broadcast/multicast addresses */
+ macaddr0 = odp_be_to_cpu_16(*((const uint16_t *)(const void *)eth));
+ if (odp_unlikely((macaddr0 & 0x0100) == 0x0100))
+ input_flags.eth_mcast = 1;
+
+ if (odp_unlikely(macaddr0 == 0xffff)) {
+ macaddr2 =
+ odp_be_to_cpu_16(*((const uint16_t *)
+ (const void *)eth + 1));
+ macaddr4 =
+ odp_be_to_cpu_16(*((const uint16_t *)
+ (const void *)eth + 2));
+
+ if ((macaddr2 == 0xffff) && (macaddr4 == 0xffff))
+ input_flags.eth_bcast = 1;
+ }
+
+ /* Get Ethertype */
+ ethtype = odp_be_to_cpu_16(eth->type);
+ *offset += sizeof(*eth);
+ *parseptr += sizeof(*eth);
+
+ /* Check for SNAP vs. DIX */
+ if (odp_unlikely(ethtype < _ODP_ETH_LEN_MAX)) {
+ input_flags.snap = 1;
+ if (ethtype > frame_len - *offset) {
+ prs->flags.snap_len_err = 1;
+ ethtype = 0;
+ goto error;
+ }
+ ethtype = odp_be_to_cpu_16(*((const uint16_t *)(uintptr_t)
+ (*parseptr + 6)));
+ *offset += 8;
+ *parseptr += 8;
+ }
+
+ /* Parse the VLAN header(s), if present */
+ if (odp_unlikely(ethtype == _ODP_ETHTYPE_VLAN_OUTER)) {
+ input_flags.vlan_qinq = 1;
+ input_flags.vlan = 1;
+
+ vlan = (const _odp_vlanhdr_t *)*parseptr;
+ ethtype = odp_be_to_cpu_16(vlan->type);
+ *offset += sizeof(_odp_vlanhdr_t);
+ *parseptr += sizeof(_odp_vlanhdr_t);
+ }
+
+ if (ethtype == _ODP_ETHTYPE_VLAN) {
+ input_flags.vlan = 1;
+ vlan = (const _odp_vlanhdr_t *)*parseptr;
+ ethtype = odp_be_to_cpu_16(vlan->type);
+ *offset += sizeof(_odp_vlanhdr_t);
+ *parseptr += sizeof(_odp_vlanhdr_t);
+ }
+
+ /*
+ * The packet was too short for what we parsed. We just give up
+ * entirely without trying to parse what fits in the packet.
+ */
+ if (odp_unlikely(*offset > frame_len)) {
+ input_flags.all = 0;
+ input_flags.l2 = 1;
+ ethtype = 0;
+ }
+
+error:
+ prs->input_flags.all |= input_flags.all;
+
+ return ethtype;
+}
+
+/**
+ * Parser helper function for IPv4
+ *
+ * Requires up to PARSE_IPV4_BYTES bytes of contiguous packet data.
+ */
+static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr,
+ uint32_t *offset, uint32_t frame_len,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum)
+{
+ const _odp_ipv4hdr_t *ipv4 = (const _odp_ipv4hdr_t *)*parseptr;
+ uint32_t dstaddr = odp_be_to_cpu_32(ipv4->dst_addr);
+ uint32_t l3_len = odp_be_to_cpu_16(ipv4->tot_len);
+ uint16_t frag_offset = odp_be_to_cpu_16(ipv4->frag_offset);
+ uint8_t ver = _ODP_IPV4HDR_VER(ipv4->ver_ihl);
+ uint8_t ihl = _ODP_IPV4HDR_IHL(ipv4->ver_ihl);
+
+ if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN ||
+ ver != 4 ||
+ sizeof(*ipv4) > frame_len - *offset ||
+ (l3_len > frame_len - *offset))) {
+ prs->flags.ip_err = 1;
+ return 0;
+ }
+
+ if (chksums.chksum.ipv4) {
+ prs->input_flags.l3_chksum_done = 1;
+ if (chksum_finalize(chksum_partial(ipv4, ihl * 4, 0)) != 0xffff) {
+ prs->flags.ip_err = 1;
+ prs->flags.l3_chksum_err = 1;
+ return 0;
+ }
+ }
+
+ *offset += ihl * 4;
+ *parseptr += ihl * 4;
+
+ if (chksums.chksum.udp || chksums.chksum.tcp)
+ *l4_part_sum = chksum_partial((const uint8_t *)&ipv4->src_addr,
+ 2 * _ODP_IPV4ADDR_LEN, 0);
+
+ if (odp_unlikely(ihl > _ODP_IPV4HDR_IHL_MIN))
+ prs->input_flags.ipopt = 1;
+
+ /* A packet is a fragment if:
+ * "more fragments" flag is set (all fragments except the last)
+ * OR
+ * "fragment offset" field is nonzero (all fragments except the first)
+ */
+ if (odp_unlikely(_ODP_IPV4HDR_IS_FRAGMENT(frag_offset)))
+ prs->input_flags.ipfrag = 1;
+
+ /* Handle IPv4 broadcast / multicast */
+ if (odp_unlikely(dstaddr == 0xffffffff))
+ prs->input_flags.ip_bcast = 1;
+
+ if (odp_unlikely((dstaddr >> 28) == 0xe))
+ prs->input_flags.ip_mcast = 1;
+
+ return ipv4->proto;
+}
+
+/**
+ * Parser helper function for IPv6
+ *
+ * Requires at least PARSE_IPV6_BYTES bytes of contiguous packet data.
+ */
+static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr,
+ uint32_t *offset, uint32_t frame_len,
+ uint32_t seg_len,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum)
+{
+ const _odp_ipv6hdr_t *ipv6 = (const _odp_ipv6hdr_t *)*parseptr;
+ const _odp_ipv6hdr_ext_t *ipv6ext;
+ uint32_t dstaddr0 = odp_be_to_cpu_32(ipv6->dst_addr.u8[0]);
+ uint32_t l3_len = odp_be_to_cpu_16(ipv6->payload_len) +
+ _ODP_IPV6HDR_LEN;
+
+ /* Basic sanity checks on IPv6 header */
+ if (odp_unlikely((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 ||
+ sizeof(*ipv6) > frame_len - *offset ||
+ l3_len > frame_len - *offset)) {
+ prs->flags.ip_err = 1;
+ return 0;
+ }
+
+ /* IPv6 broadcast / multicast flags */
+ prs->input_flags.ip_mcast = (dstaddr0 & 0xff000000) == 0xff000000;
+ prs->input_flags.ip_bcast = 0;
+
+ /* Skip past IPv6 header */
+ *offset += sizeof(_odp_ipv6hdr_t);
+ *parseptr += sizeof(_odp_ipv6hdr_t);
+
+ if (chksums.chksum.udp || chksums.chksum.tcp)
+ *l4_part_sum = chksum_partial((const uint8_t *)&ipv6->src_addr,
+ 2 * _ODP_IPV6ADDR_LEN, 0);
+
+ /* Skip past any IPv6 extension headers */
+ if (ipv6->next_hdr == _ODP_IPPROTO_HOPOPTS ||
+ ipv6->next_hdr == _ODP_IPPROTO_ROUTE) {
+ prs->input_flags.ipopt = 1;
+
+ do {
+ ipv6ext = (const _odp_ipv6hdr_ext_t *)*parseptr;
+ uint16_t extlen = 8 + ipv6ext->ext_len * 8;
+
+ *offset += extlen;
+ *parseptr += extlen;
+ } while ((ipv6ext->next_hdr == _ODP_IPPROTO_HOPOPTS ||
+ ipv6ext->next_hdr == _ODP_IPPROTO_ROUTE) &&
+ *offset < seg_len);
+
+ if (*offset >= prs->l3_offset +
+ odp_be_to_cpu_16(ipv6->payload_len)) {
+ prs->flags.ip_err = 1;
+ return 0;
+ }
+
+ if (ipv6ext->next_hdr == _ODP_IPPROTO_FRAG)
+ prs->input_flags.ipfrag = 1;
+
+ return ipv6ext->next_hdr;
+ }
+
+ if (odp_unlikely(ipv6->next_hdr == _ODP_IPPROTO_FRAG)) {
+ prs->input_flags.ipopt = 1;
+ prs->input_flags.ipfrag = 1;
+ }
+
+ return ipv6->next_hdr;
+}
+
+/**
+ * Parser helper function for TCP
+ *
+ * Requires PARSE_TCP_BYTES bytes of contiguous packet data.
+ */
+static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr,
+ uint16_t tcp_len,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum)
+{
+ const _odp_tcphdr_t *tcp = (const _odp_tcphdr_t *)*parseptr;
+ uint32_t len = tcp->hl * 4;
+
+ if (odp_unlikely(tcp->hl < sizeof(_odp_tcphdr_t) / sizeof(uint32_t)))
+ prs->flags.tcp_err = 1;
+
+ if (chksums.chksum.tcp &&
+ !prs->input_flags.ipfrag) {
+ *l4_part_sum += odp_cpu_to_be_16(tcp_len);
+#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
+ *l4_part_sum += _ODP_IPPROTO_TCP;
+#else
+ *l4_part_sum += _ODP_IPPROTO_TCP << 8;
+#endif
+ }
+
+ *parseptr += len;
+}
+
+/**
+ * Parser helper function for UDP
+ *
+ * Requires PARSE_UDP_BYTES bytes of contiguous packet data.
+ */
+static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum)
+{
+ const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr;
+ uint32_t udplen = odp_be_to_cpu_16(udp->length);
+ uint16_t ipsec_port = odp_cpu_to_be_16(_ODP_UDP_IPSEC_PORT);
+
+ if (odp_unlikely(udplen < sizeof(_odp_udphdr_t))) {
+ prs->flags.udp_err = 1;
+ return;
+ }
+
+ if (chksums.chksum.udp &&
+ !prs->input_flags.ipfrag) {
+ if (udp->chksum == 0) {
+ prs->input_flags.l4_chksum_done = 1;
+ prs->flags.l4_chksum_err =
+ (prs->input_flags.ipv4 != 1);
+ } else {
+ *l4_part_sum += udp->length;
+#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
+ *l4_part_sum += _ODP_IPPROTO_UDP;
+#else
+ *l4_part_sum += _ODP_IPPROTO_UDP << 8;
+#endif
+ }
+ prs->input_flags.udp_chksum_zero = (udp->chksum == 0);
+ }
+
+ if (odp_unlikely(ipsec_port == udp->dst_port && udplen > 4)) {
+ uint32_t val;
+
+ memcpy(&val, udp + 1, 4);
+ if (val != 0) {
+ prs->input_flags.ipsec = 1;
+ prs->input_flags.ipsec_udp = 1;
+ }
+ }
+
+ *parseptr += sizeof(_odp_udphdr_t);
+}
+
+/**
+ * Parser helper function for SCTP
+ *
+ * Requires PARSE_SCTP_BYTES bytes of contiguous packet data.
+ */
+static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr,
+ uint16_t sctp_len,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum)
+{
+ if (odp_unlikely(sctp_len < sizeof(_odp_sctphdr_t))) {
+ prs->flags.sctp_err = 1;
+ return;
+ }
+
+ if (chksums.chksum.sctp &&
+ !prs->input_flags.ipfrag) {
+ const _odp_sctphdr_t *sctp =
+ (const _odp_sctphdr_t *)*parseptr;
+ uint32_t crc = ~0;
+ uint32_t zero = 0;
+
+ crc = odp_hash_crc32c(sctp, sizeof(*sctp) - 4, crc);
+ crc = odp_hash_crc32c(&zero, 4, crc);
+ *l4_part_sum = crc;
+ }
+
+ *parseptr += sizeof(_odp_sctphdr_t);
+}
+
+/* Requires up to PARSE_L3_L4_BYTES bytes of contiguous packet data. */
+int _odp_packet_parse_common_l3_l4(packet_parser_t *prs,
+ const uint8_t *parseptr, uint32_t offset,
+ uint32_t frame_len, uint32_t seg_len,
+ int layer, uint16_t ethtype,
+ odp_proto_chksums_t chksums,
+ uint64_t *l4_part_sum,
+ odp_pktin_config_opt_t opt)
+{
+ uint8_t ip_proto;
+
+ prs->l3_offset = offset;
+
+ if (odp_unlikely(layer <= ODP_PROTO_LAYER_L2))
+ return prs->flags.all.error != 0;
+
+ /* Set l3 flag only for known ethtypes */
+ prs->input_flags.l3 = 1;
+
+ /* Parse Layer 3 headers */
+ switch (ethtype) {
+ case _ODP_ETHTYPE_IPV4:
+ prs->input_flags.ipv4 = 1;
+ ip_proto = parse_ipv4(prs, &parseptr, &offset, frame_len,
+ chksums, l4_part_sum);
+ prs->l4_offset = offset;
+ if (prs->flags.ip_err && opt.bit.drop_ipv4_err)
+ return -1; /* drop */
+ break;
+
+ case _ODP_ETHTYPE_IPV6:
+ prs->input_flags.ipv6 = 1;
+ ip_proto = parse_ipv6(prs, &parseptr, &offset, frame_len,
+ seg_len, chksums, l4_part_sum);
+ prs->l4_offset = offset;
+ if (prs->flags.ip_err && opt.bit.drop_ipv6_err)
+ return -1; /* drop */
+ break;
+
+ case _ODP_ETHTYPE_ARP:
+ prs->input_flags.arp = 1;
+ ip_proto = 255; /* Reserved invalid by IANA */
+ break;
+
+ default:
+ prs->input_flags.l3 = 0;
+ ip_proto = 255; /* Reserved invalid by IANA */
+ }
+
+ if (layer == ODP_PROTO_LAYER_L3)
+ return prs->flags.all.error != 0;
+
+ /* Set l4 flag only for known ip_proto */
+ prs->input_flags.l4 = 1;
+
+ /* Parse Layer 4 headers */
+ switch (ip_proto) {
+ case _ODP_IPPROTO_ICMPV4:
+ /* Fall through */
+
+ case _ODP_IPPROTO_ICMPV6:
+ prs->input_flags.icmp = 1;
+ break;
+
+ case _ODP_IPPROTO_IPIP:
+ /* Do nothing */
+ break;
+
+ case _ODP_IPPROTO_TCP:
+ if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len))
+ return -1;
+ prs->input_flags.tcp = 1;
+ parse_tcp(prs, &parseptr, frame_len - prs->l4_offset, chksums,
+ l4_part_sum);
+ if (prs->flags.tcp_err && opt.bit.drop_tcp_err)
+ return -1; /* drop */
+ break;
+
+ case _ODP_IPPROTO_UDP:
+ if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len))
+ return -1;
+ prs->input_flags.udp = 1;
+ parse_udp(prs, &parseptr, chksums, l4_part_sum);
+ if (prs->flags.udp_err && opt.bit.drop_udp_err)
+ return -1; /* drop */
+ break;
+
+ case _ODP_IPPROTO_AH:
+ prs->input_flags.ipsec = 1;
+ prs->input_flags.ipsec_ah = 1;
+ break;
+
+ case _ODP_IPPROTO_ESP:
+ prs->input_flags.ipsec = 1;
+ prs->input_flags.ipsec_esp = 1;
+ break;
+
+ case _ODP_IPPROTO_SCTP:
+ prs->input_flags.sctp = 1;
+ parse_sctp(prs, &parseptr, frame_len - prs->l4_offset, chksums,
+ l4_part_sum);
+ if (prs->flags.sctp_err && opt.bit.drop_sctp_err)
+ return -1; /* drop */
+ break;
+
+ case _ODP_IPPROTO_NO_NEXT:
+ prs->input_flags.no_next_hdr = 1;
+ break;
+
+ default:
+ prs->input_flags.l4 = 0;
+ break;
+ }
+
+ return prs->flags.all.error != 0;
+}
diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c
index cca2d55f0..defdeb4fb 100644
--- a/platform/linux-generic/odp_pool.c
+++ b/platform/linux-generic/odp_pool.c
@@ -494,12 +494,7 @@ static void init_buffers(pool_t *pool)
for (i = 0; i < pool->num + skipped_blocks ; i++) {
int skip = 0;
-
- addr = &pool->base_addr[(i * pool->block_size) +
- pool->block_offset];
- event_hdr = addr;
- buf_hdr = addr;
- pkt_hdr = addr;
+ addr = &pool->base_addr[i * pool->block_size];
/* Skip packet buffers which cross huge page boundaries. Some
* NICs cannot handle buffers which cross page boundaries. */
@@ -517,6 +512,12 @@ static void init_buffers(pool_t *pool)
skip = 1;
}
}
+
+ addr = (uint8_t *)addr + pool->block_offset;
+ event_hdr = addr;
+ buf_hdr = addr;
+ pkt_hdr = addr;
+
if (pool->uarea_size)
uarea = &pool->uarea_base_addr[(i - skipped_blocks) *
pool->uarea_size];
diff --git a/platform/linux-generic/odp_queue_lf.c b/platform/linux-generic/odp_queue_lf.c
index f5c3830e8..58959df31 100644
--- a/platform/linux-generic/odp_queue_lf.c
+++ b/platform/linux-generic/odp_queue_lf.c
@@ -13,6 +13,7 @@
#include <odp_debug_internal.h>
#include <odp_event_internal.h>
#include <odp_queue_basic_internal.h>
+#include <odp_types_internal.h>
#include <string.h>
#include <stdio.h>
@@ -24,9 +25,7 @@
#ifdef __SIZEOF_INT128__
-__extension__ typedef unsigned __int128 u128_t;
-
-static inline void lockfree_zero_u128(u128_t *atomic)
+static inline void lockfree_zero_u128(_odp_u128_t *atomic)
{
__atomic_store_n(atomic, 0, __ATOMIC_RELAXED);
}
@@ -40,21 +39,21 @@ static inline void lockfree_zero_u128(u128_t *atomic)
* So, these are never actually used. */
typedef struct ODP_ALIGNED(16) {
uint64_t u64[2];
-} u128_t;
+} _odp_u128_t;
-static inline u128_t lockfree_load_u128(u128_t *atomic)
+static inline _odp_u128_t lockfree_load_u128(_odp_u128_t *atomic)
{
return *atomic;
}
-static inline void lockfree_zero_u128(u128_t *atomic)
+static inline void lockfree_zero_u128(_odp_u128_t *atomic)
{
atomic->u64[0] = 0;
atomic->u64[1] = 0;
}
-static inline int lockfree_cas_acq_rel_u128(u128_t *atomic, u128_t old_val,
- u128_t new_val)
+static inline int lockfree_cas_acq_rel_u128(_odp_u128_t *atomic, _odp_u128_t old_val,
+ _odp_u128_t new_val)
{
if (atomic->u64[0] == old_val.u64[0] &&
atomic->u64[1] == old_val.u64[1]) {
@@ -75,7 +74,7 @@ static inline int lockfree_check_u128(void)
/* Node in lock-free ring */
typedef union {
- u128_t u128;
+ _odp_u128_t u128;
struct {
/* Data with lowest counter value is the head. Empty node has
diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c
index 5343c2834..7482d776d 100644
--- a/platform/linux-generic/odp_schedule_scalable.c
+++ b/platform/linux-generic/odp_schedule_scalable.c
@@ -817,23 +817,19 @@ events_dequeued:
}
/* Ingress queue empty => poll pktio RX queue */
- odp_event_t rx_evts[QUEUE_MULTI_MAX];
- int num_rx = _odp_sched_cb_pktin_poll_one(elem->pktio_idx,
- elem->rx_queue,
- rx_evts);
+ _odp_event_hdr_t *rx_evts[QUEUE_MULTI_MAX];
+ int num_rx = _odp_sched_cb_pktin_poll(elem->pktio_idx, elem->rx_queue,
+ rx_evts, QUEUE_MULTI_MAX);
+
if (odp_likely(num_rx > 0)) {
num = num_rx < num_evts ? num_rx : num_evts;
for (i = 0; i < num; i++) {
/* Return events directly to caller */
- ev[i] = rx_evts[i];
+ ev[i] = _odp_event_from_hdr(rx_evts[i]);
}
if (num_rx > num) {
/* Events remain, enqueue them */
- _odp_event_hdr_t *events[QUEUE_MULTI_MAX];
-
- for (i = num; i < num_rx; i++)
- events[i] = _odp_event_hdr(rx_evts[i]);
- i = _odp_queue_enq_sp(elem, &events[num], num_rx - num);
+ i = _odp_queue_enq_sp(elem, &rx_evts[num], num_rx - num);
/* Enqueue must succeed as the queue was empty */
ODP_ASSERT(i == num_rx - num);
}
diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c
index f5aba1a88..558c3f4fe 100644
--- a/platform/linux-generic/odp_schedule_sp.c
+++ b/platform/linux-generic/odp_schedule_sp.c
@@ -13,6 +13,7 @@
#pragma GCC diagnostic ignored "-Wzero-length-bounds"
#endif
+#include <odp/api/packet.h>
#include <odp/api/ticketlock.h>
#include <odp/api/thread.h>
#include <odp/api/plat/thread_inlines.h>
diff --git a/platform/linux-generic/odp_time.c b/platform/linux-generic/odp_time.c
index d908f02cb..b591d712a 100644
--- a/platform/linux-generic/odp_time.c
+++ b/platform/linux-generic/odp_time.c
@@ -145,15 +145,6 @@ static inline void time_wait_until(odp_time_t time)
} while (odp_time_cmp(time, cur) > 0);
}
-uint64_t odp_time_diff_ns(odp_time_t t2, odp_time_t t1)
-{
- odp_time_t time;
-
- time.u64 = t2.u64 - t1.u64;
-
- return odp_time_to_ns(time);
-}
-
odp_time_t odp_time_local_from_ns(uint64_t ns)
{
return time_from_ns(ns);
diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c
index d1d4a5bc8..1c54ab740 100644
--- a/platform/linux-generic/odp_timer.c
+++ b/platform/linux-generic/odp_timer.c
@@ -13,49 +13,52 @@
*/
#include <odp_posix_extensions.h>
-#include <errno.h>
-#include <stdlib.h>
-#include <time.h>
-#include <signal.h>
-#include <pthread.h>
-#include <unistd.h>
-#include <sys/syscall.h>
-#include <inttypes.h>
-#include <string.h>
-
-#include <odp/api/std.h>
#include <odp/api/align.h>
-#include <odp_align_internal.h>
#include <odp/api/atomic.h>
-#include <odp/api/plat/atomic_inlines.h>
-#include <odp_atomic_internal.h>
-#include <odp/api/buffer.h>
#include <odp/api/cpu.h>
-#include <odp/api/pool.h>
-#include <odp_pool_internal.h>
#include <odp/api/debug.h>
-#include <odp_debug_internal.h>
#include <odp/api/event.h>
#include <odp/api/hints.h>
-#include <odp_init_internal.h>
-#include <odp_errno_define.h>
+#include <odp/api/pool.h>
#include <odp/api/queue.h>
#include <odp/api/shared_memory.h>
#include <odp/api/spinlock.h>
-#include <odp/api/std_types.h>
+#include <odp/api/std.h>
#include <odp/api/sync.h>
#include <odp/api/time.h>
-#include <odp/api/plat/time_inlines.h>
#include <odp/api/timer.h>
+
+/* Inlined API functions */
+#include <odp/api/plat/atomic_inlines.h>
+#include <odp/api/plat/event_inlines.h>
+#include <odp/api/plat/queue_inlines.h>
+#include <odp/api/plat/time_inlines.h>
+#include <odp/api/plat/timer_inlines.h>
+
+#include <odp/api/plat/timer_inline_types.h>
+
+#include <odp_align_internal.h>
+#include <odp_atomic_internal.h>
+#include <odp_debug_internal.h>
+#include <odp_errno_define.h>
+#include <odp_event_internal.h>
+#include <odp_global_data.h>
+#include <odp_init_internal.h>
#include <odp_libconfig_internal.h>
+#include <odp_pool_internal.h>
#include <odp_queue_if.h>
#include <odp_timer_internal.h>
-#include <odp/api/plat/queue_inlines.h>
-#include <odp_global_data.h>
-#include <odp_event_internal.h>
+#include <odp_types_internal.h>
-/* Inlined API functions */
-#include <odp/api/plat/event_inlines.h>
+#include <errno.h>
+#include <inttypes.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/syscall.h>
+#include <time.h>
+#include <unistd.h>
/* One divided by one nanosecond in Hz */
#define GIGA_HZ 1000000000
@@ -94,6 +97,18 @@
#define IDX2LOCK(idx) (&timer_global->locks[(idx) % NUM_LOCKS])
#endif
+#include <odp/visibility_begin.h>
+
+/* Fill in timeout header field offsets for inline functions */
+const _odp_timeout_inline_offset_t
+_odp_timeout_inline_offset ODP_ALIGNED_CACHE = {
+ .expiration = offsetof(odp_timeout_hdr_t, expiration),
+ .timer = offsetof(odp_timeout_hdr_t, timer),
+ .user_ptr = offsetof(odp_timeout_hdr_t, user_ptr)
+};
+
+#include <odp/visibility_end.h>
+
typedef struct
#ifdef ODP_ATOMIC_U128
ODP_ALIGNED(16) /* 16-byte atomic operations need properly aligned addresses */
@@ -689,7 +704,7 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event,
/* Atomic CAS will fail if we experienced torn reads,
* retry update sequence until CAS succeeds */
} while (!_odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb,
- (_uint128_t *)&old, (_uint128_t *)&new,
+ (_odp_u128_t *)&old, (_odp_u128_t *)&new,
_ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX));
#elif __GCC_ATOMIC_LLONG_LOCK_FREE >= 2 && \
defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8
@@ -760,8 +775,8 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_event_t *tmo_event,
/* We are releasing the new timeout event to some other
* thread */
_odp_atomic_u128_xchg_mm((_odp_atomic_u128_t *)tb,
- (_uint128_t *)&new,
- (_uint128_t *)&old,
+ (_odp_u128_t *)&new,
+ (_odp_u128_t *)&old,
_ODP_MEMMODEL_ACQ_RLS);
old_event = old.tmo_event;
#else
@@ -804,7 +819,7 @@ static odp_event_t timer_set_unused(timer_pool_t *tp, uint32_t idx)
new.tmo_event = ODP_EVENT_INVALID;
_odp_atomic_u128_xchg_mm((_odp_atomic_u128_t *)tb,
- (_uint128_t *)&new, (_uint128_t *)&old,
+ (_odp_u128_t *)&new, (_odp_u128_t *)&old,
_ODP_MEMMODEL_RLX);
old_event = old.tmo_event;
#else
@@ -858,8 +873,8 @@ static odp_event_t timer_cancel(timer_pool_t *tp, uint32_t idx)
/* Atomic CAS will fail if we experienced torn reads,
* retry update sequence until CAS succeeds */
} while (!_odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb,
- (_uint128_t *)&old,
- (_uint128_t *)&new,
+ (_odp_u128_t *)&old,
+ (_odp_u128_t *)&new,
_ODP_MEMMODEL_RLS,
_ODP_MEMMODEL_RLX));
old_event = old.tmo_event;
@@ -916,7 +931,7 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick)
new.tmo_event = ODP_EVENT_INVALID;
int succ = _odp_atomic_u128_cmp_xchg_mm((_odp_atomic_u128_t *)tb,
- (_uint128_t *)&old, (_uint128_t *)&new,
+ (_odp_u128_t *)&old, (_odp_u128_t *)&new,
_ODP_MEMMODEL_RLS, _ODP_MEMMODEL_RLX);
if (succ)
tmo_event = old.tmo_event;
@@ -1468,22 +1483,6 @@ void odp_timer_pool_destroy(odp_timer_pool_t tpid)
odp_timer_pool_del(timer_pool_from_hdl(tpid));
}
-uint64_t odp_timer_tick_to_ns(odp_timer_pool_t tpid, uint64_t ticks)
-{
- (void)tpid;
-
- /* Timer ticks in API are nsec */
- return ticks;
-}
-
-uint64_t odp_timer_ns_to_tick(odp_timer_pool_t tpid, uint64_t ns)
-{
- (void)tpid;
-
- /* Timer ticks in API are nsec */
- return ns;
-}
-
uint64_t odp_timer_current_tick(odp_timer_pool_t tpid)
{
timer_pool_t *tp = timer_pool_from_hdl(tpid);
@@ -1812,19 +1811,6 @@ uint64_t odp_timer_to_u64(odp_timer_t hdl)
return _odp_pri(hdl);
}
-odp_timeout_t odp_timeout_from_event(odp_event_t ev)
-{
- /* This check not mandated by the API specification */
- if (odp_event_type(ev) != ODP_EVENT_TIMEOUT)
- ODP_ABORT("Event not a timeout");
- return (odp_timeout_t)ev;
-}
-
-odp_event_t odp_timeout_to_event(odp_timeout_t tmo)
-{
- return (odp_event_t)tmo;
-}
-
uint64_t odp_timeout_to_u64(odp_timeout_t tmo)
{
return _odp_pri(tmo);
@@ -1847,21 +1833,6 @@ int odp_timeout_fresh(odp_timeout_t tmo)
return hdr->expiration == (exp_tck & ~TMO_INACTIVE);
}
-odp_timer_t odp_timeout_timer(odp_timeout_t tmo)
-{
- return timeout_hdr(tmo)->timer;
-}
-
-uint64_t odp_timeout_tick(odp_timeout_t tmo)
-{
- return timeout_hdr(tmo)->expiration;
-}
-
-void *odp_timeout_user_ptr(odp_timeout_t tmo)
-{
- return (void *)(uintptr_t)timeout_hdr(tmo)->user_ptr;
-}
-
odp_timeout_t odp_timeout_alloc(odp_pool_t pool_hdl)
{
odp_event_t event;
diff --git a/platform/linux-generic/odp_timer_api.c b/platform/linux-generic/odp_timer_api.c
new file mode 100644
index 000000000..cd657956b
--- /dev/null
+++ b/platform/linux-generic/odp_timer_api.c
@@ -0,0 +1,11 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/timer.h>
+
+/* Non-inlined functions for ABI compat mode */
+#define _ODP_NO_INLINE
+#include <odp/api/plat/timer_inlines.h>
diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c
index fbf33e117..f02297ec9 100644
--- a/platform/linux-generic/odp_traffic_mngr.c
+++ b/platform/linux-generic/odp_traffic_mngr.c
@@ -55,45 +55,60 @@ _odp_int_name_kind_t PROFILE_TO_HANDLE_KIND[ODP_TM_NUM_PROFILES] = {
static const pkt_desc_t EMPTY_PKT_DESC = { .word = 0 };
#define MAX_PRIORITIES ODP_TM_MAX_PRIORITIES
-#define NUM_SHAPER_COLORS ODP_NUM_SHAPER_COLORS
/* Shaper BW limits in bits/sec */
#define TM_MIN_SHAPER_BW 8000ULL
#define TM_MAX_SHAPER_BW (100ULL * 1000ULL * 1000ULL * 1000ULL)
+/* Possible values for running the shaper algorithm. TM_SHAPER_GREEN means that
+ * the traffic is within the commit specification (rate and burst size),
+ * TM_SHAPER_YELLOW means that the traffic is within the peak specification
+ * (rate and burst size) and TM_SHAPER_RED means that the traffic is exceeding
+ * both its commit and peak specifications. Note that packets can also have an
+ * assigned packet color of ODP_PACKET_GREEN, ODP_PACKET_YELLOW or
+ * ODP_PACKET_RED, which has a different meaning and purpose than the shaper
+ * colors.
+ */
+typedef enum {
+ TM_SHAPER_GREEN, TM_SHAPER_YELLOW, TM_SHAPER_RED
+} tm_shaper_color_t;
+
+/* Number of enumeration values defined in tm_shaper_color_t type. */
+#define NUM_SHAPER_COLORS 3
+
static const tm_prop_t basic_prop_tbl[MAX_PRIORITIES][NUM_SHAPER_COLORS] = {
[0] = {
- [ODP_TM_SHAPER_GREEN] = { 0, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 0, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 0, DELAY_PKT } },
+ [TM_SHAPER_GREEN] = { 0, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 0, DECR_BOTH },
+ [TM_SHAPER_RED] = { 0, DELAY_PKT } },
[1] = {
- [ODP_TM_SHAPER_GREEN] = { 1, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 1, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 1, DELAY_PKT } },
+ [TM_SHAPER_GREEN] = { 1, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 1, DECR_BOTH },
+ [TM_SHAPER_RED] = { 1, DELAY_PKT } },
[2] = {
- [ODP_TM_SHAPER_GREEN] = { 2, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 2, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 2, DELAY_PKT } },
+ [TM_SHAPER_GREEN] = { 2, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 2, DECR_BOTH },
+ [TM_SHAPER_RED] = { 2, DELAY_PKT } },
[3] = {
- [ODP_TM_SHAPER_GREEN] = { 3, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 3, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 3, DELAY_PKT } },
+ [TM_SHAPER_GREEN] = { 3, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 3, DECR_BOTH },
+ [TM_SHAPER_RED] = { 3, DELAY_PKT } },
[4] = {
- [ODP_TM_SHAPER_GREEN] = { 4, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 4, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 4, DELAY_PKT } },
+ [TM_SHAPER_GREEN] = { 4, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 4, DECR_BOTH },
+ [TM_SHAPER_RED] = { 4, DELAY_PKT } },
[5] = {
- [ODP_TM_SHAPER_GREEN] = { 5, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 5, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 5, DELAY_PKT } },
+ [TM_SHAPER_GREEN] = { 5, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 5, DECR_BOTH },
+ [TM_SHAPER_RED] = { 5, DELAY_PKT } },
[6] = {
- [ODP_TM_SHAPER_GREEN] = { 6, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 6, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 6, DELAY_PKT } },
+ [TM_SHAPER_GREEN] = { 6, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 6, DECR_BOTH },
+ [TM_SHAPER_RED] = { 6, DELAY_PKT } },
[7] = {
- [ODP_TM_SHAPER_GREEN] = { 7, DECR_BOTH },
- [ODP_TM_SHAPER_YELLOW] = { 7, DECR_BOTH },
- [ODP_TM_SHAPER_RED] = { 7, DELAY_PKT } }
+ [TM_SHAPER_GREEN] = { 7, DECR_BOTH },
+ [TM_SHAPER_YELLOW] = { 7, DECR_BOTH },
+ [TM_SHAPER_RED] = { 7, DELAY_PKT } }
};
#define MAX_SHAPER_PROFILES 128
@@ -1076,30 +1091,30 @@ static odp_bool_t run_shaper(tm_system_t *tm_system,
pkt_desc_t *pkt_desc,
uint8_t priority)
{
- odp_tm_shaper_color_t shaper_color;
+ tm_shaper_color_t shaper_color;
tm_shaper_params_t *shaper_params;
odp_bool_t output_change;
tm_prop_t propagation;
shaper_params = shaper_obj->shaper_params;
- shaper_color = ODP_TM_SHAPER_GREEN;
+ shaper_color = TM_SHAPER_GREEN;
if (shaper_params) {
update_shaper_elapsed_time(tm_system, shaper_params,
shaper_obj);
if (shaper_params->enabled) {
if (0 < shaper_obj->commit_cnt)
- shaper_color = ODP_TM_SHAPER_GREEN;
+ shaper_color = TM_SHAPER_GREEN;
else if (!shaper_params->dual_rate)
- shaper_color = ODP_TM_SHAPER_RED;
+ shaper_color = TM_SHAPER_RED;
else if (shaper_obj->peak_cnt <= 0)
- shaper_color = ODP_TM_SHAPER_RED;
+ shaper_color = TM_SHAPER_RED;
else
- shaper_color = ODP_TM_SHAPER_YELLOW;
+ shaper_color = TM_SHAPER_YELLOW;
- if (shaper_color == ODP_TM_SHAPER_GREEN)
+ if (shaper_color == TM_SHAPER_GREEN)
tm_system->shaper_green_cnt++;
- else if (shaper_color == ODP_TM_SHAPER_YELLOW)
+ else if (shaper_color == TM_SHAPER_YELLOW)
tm_system->shaper_yellow_cnt++;
else
tm_system->shaper_red_cnt++;
@@ -1994,6 +2009,12 @@ static void tm_queue_cnts_decrement(tm_system_t *tm_system,
odp_atomic_sub_u64(&queue_cnts->byte_cnt, frame_len);
}
+static inline void activate_packet_aging(odp_packet_hdr_t *pkt_hdr)
+{
+ if (odp_unlikely(pkt_hdr->p.flags.tx_aging))
+ pkt_hdr->tx_aging_ns = pkt_hdr->tx_aging_ns + odp_time_global_ns();
+}
+
static int tm_enqueue(tm_system_t *tm_system,
tm_queue_obj_t *tm_queue_obj,
odp_packet_t pkt)
@@ -2028,6 +2049,7 @@ static int tm_enqueue(tm_system_t *tm_system,
if (tm_queue_obj->ordered_enqueue)
_odp_sched_fn->order_lock();
+ activate_packet_aging(packet_hdr(pkt));
rc = input_work_queue_append(tm_system, &work_item);
if (tm_queue_obj->ordered_enqueue)
@@ -2224,6 +2246,11 @@ static void tm_egress_marking(tm_system_t *tm_system, odp_packet_t odp_pkt)
}
}
+static inline odp_bool_t is_packet_aged(odp_packet_hdr_t *pkt_hdr)
+{
+ return pkt_hdr->p.flags.tx_aging && pkt_hdr->tx_aging_ns < odp_time_global_ns();
+}
+
static void tm_send_pkt(tm_system_t *tm_system, uint32_t max_sends)
{
tm_queue_obj_t *tm_queue_obj;
@@ -2250,9 +2277,13 @@ static void tm_send_pkt(tm_system_t *tm_system, uint32_t max_sends)
tm_system->egress_pkt_desc = EMPTY_PKT_DESC;
if (tm_system->egress.egress_kind == ODP_TM_EGRESS_PKT_IO) {
- ret = odp_pktout_send(tm_system->pktout, &odp_pkt, 1);
+ pktio_entry = get_pktio_entry(tm_system->pktout.pktio);
+ if (odp_unlikely(_odp_pktio_tx_aging_enabled(pktio_entry) &&
+ is_packet_aged(packet_hdr(odp_pkt))))
+ ret = 0; /* Aged packet handled as a discard */
+ else
+ ret = odp_pktout_send(tm_system->pktout, &odp_pkt, 1);
if (odp_unlikely(ret != 1)) {
- pktio_entry = get_pktio_entry(tm_system->pktout.pktio);
if (odp_unlikely(_odp_pktio_tx_compl_enabled(pktio_entry)))
_odp_pktio_allocate_and_send_tx_compl_events(pktio_entry,
&odp_pkt, 1);
@@ -2581,7 +2612,11 @@ static int tm_capabilities(odp_tm_capabilities_t capabilities[],
cap_ptr->vlan_marking_supported = true;
cap_ptr->ecn_marking_supported = true;
cap_ptr->drop_prec_marking_supported = true;
- cap_ptr->tm_queue_threshold = true;
+
+ cap_ptr->tm_queue_threshold.byte = true;
+ cap_ptr->tm_queue_threshold.packet = true;
+ cap_ptr->tm_queue_threshold.byte_and_packet = true;
+
cap_ptr->tm_queue_query_flags = (ODP_TM_QUERY_PKT_CNT |
ODP_TM_QUERY_BYTE_CNT |
ODP_TM_QUERY_THRESHOLDS);
@@ -2617,7 +2652,10 @@ static int tm_capabilities(odp_tm_capabilities_t capabilities[],
per_level_cap->tm_node_dual_slope_supported = true;
per_level_cap->fair_queuing_supported = true;
per_level_cap->weights_supported = true;
- per_level_cap->tm_node_threshold = true;
+
+ per_level_cap->tm_node_threshold.byte = true;
+ per_level_cap->tm_node_threshold.packet = true;
+ per_level_cap->tm_node_threshold.byte_and_packet = true;
}
cap_ptr->queue_stats.counter.discards = 1;
@@ -2691,7 +2729,11 @@ static void tm_system_capabilities_set(odp_tm_capabilities_t *cap_ptr,
cap_ptr->ecn_marking_supported = req_ptr->ecn_marking_needed;
cap_ptr->drop_prec_marking_supported =
req_ptr->drop_prec_marking_needed;
- cap_ptr->tm_queue_threshold = threshold;
+
+ cap_ptr->tm_queue_threshold.byte = threshold;
+ cap_ptr->tm_queue_threshold.packet = threshold;
+ cap_ptr->tm_queue_threshold.byte_and_packet = threshold;
+
cap_ptr->tm_queue_query_flags = (ODP_TM_QUERY_PKT_CNT |
ODP_TM_QUERY_BYTE_CNT |
ODP_TM_QUERY_THRESHOLDS);
@@ -2743,7 +2785,10 @@ static void tm_system_capabilities_set(odp_tm_capabilities_t *cap_ptr,
per_level_cap->tm_node_dual_slope_supported = dual_slope;
per_level_cap->fair_queuing_supported = true;
per_level_cap->weights_supported = true;
- per_level_cap->tm_node_threshold = threshold;
+
+ per_level_cap->tm_node_threshold.byte = threshold;
+ per_level_cap->tm_node_threshold.packet = threshold;
+ per_level_cap->tm_node_threshold.byte_and_packet = threshold;
}
cap_ptr->queue_stats.counter.discards = 1;
diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c
index 815ec61b9..030560b0d 100644
--- a/platform/linux-generic/pktio/dpdk.c
+++ b/platform/linux-generic/pktio/dpdk.c
@@ -57,12 +57,14 @@
#include <rte_string_fns.h>
#include <rte_version.h>
-#if RTE_VERSION < RTE_VERSION_NUM(19, 8, 0, 0)
- #define rte_ether_addr ether_addr
- #define rte_ipv4_hdr ipv4_hdr
- #define rte_ipv6_hdr ipv6_hdr
- #define rte_tcp_hdr tcp_hdr
- #define rte_udp_hdr udp_hdr
+#if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0)
+ #define RTE_MBUF_F_RX_RSS_HASH PKT_RX_RSS_HASH
+ #define RTE_MBUF_F_TX_IPV4 PKT_TX_IPV4
+ #define RTE_MBUF_F_TX_IPV6 PKT_TX_IPV6
+ #define RTE_MBUF_F_TX_IP_CKSUM PKT_TX_IP_CKSUM
+ #define RTE_MBUF_F_TX_UDP_CKSUM PKT_TX_UDP_CKSUM
+ #define RTE_MBUF_F_TX_TCP_CKSUM PKT_TX_TCP_CKSUM
+ #define RTE_MEMPOOL_REGISTER_OPS MEMPOOL_REGISTER_OPS
#endif
/* NUMA is not supported on all platforms */
@@ -72,17 +74,7 @@
#define numa_num_configured_nodes() 1
#endif
-#if RTE_VERSION < RTE_VERSION_NUM(17, 5, 0, 0)
-#define rte_log_set_global_level rte_set_log_level
-#endif
-
-/* Release notes v19.11: "Changed the mempool allocation behaviour
- * so that objects no longer cross pages by default" */
-#if RTE_VERSION >= RTE_VERSION_NUM(19, 11, 0, 0)
-#define MEMPOOL_FLAGS MEMPOOL_F_NO_IOVA_CONTIG
-#else
#define MEMPOOL_FLAGS 0
-#endif
#if _ODP_DPDK_ZERO_COPY
ODP_STATIC_ASSERT(CONFIG_PACKET_HEADROOM == RTE_PKTMBUF_HEADROOM,
@@ -107,11 +99,7 @@ ODP_STATIC_ASSERT((DPDK_NB_MBUF % DPDK_MEMPOOL_CACHE_SIZE == 0) &&
#define DPDK_MIN_RX_BURST 4
/* Limits for setting link MTU */
-#if RTE_VERSION >= RTE_VERSION_NUM(19, 11, 0, 0)
#define DPDK_MTU_MIN (RTE_ETHER_MIN_MTU + _ODP_ETHHDR_LEN)
-#else
-#define DPDK_MTU_MIN (68 + _ODP_ETHHDR_LEN)
-#endif
#define DPDK_MTU_MAX (9000 + _ODP_ETHHDR_LEN)
/** DPDK runtime configuration options */
@@ -334,12 +322,16 @@ static void pktmbuf_init(struct rte_mempool *mp, void *opaque_arg ODP_UNUSED,
static struct rte_mempool *mbuf_pool_create(const char *name,
pool_t *pool_entry)
{
+ odp_shm_info_t shm_info;
struct rte_mempool *mp = NULL;
struct rte_pktmbuf_pool_private mbp_priv;
struct rte_mempool_objsz sz;
unsigned int elt_size = pool_entry->dpdk_elt_size;
- unsigned int num = pool_entry->num;
+ unsigned int num = pool_entry->num, populated = 0;
uint32_t total_size;
+ uint64_t page_size, offset = 0, remainder = 0;
+ uint8_t *addr;
+ int ret;
if (!(pool_entry->mem_from_huge_pages)) {
ODP_ERR("DPDK requires memory is allocated from huge pages\n");
@@ -348,11 +340,17 @@ static struct rte_mempool *mbuf_pool_create(const char *name,
if (pool_entry->seg_len < RTE_MBUF_DEFAULT_BUF_SIZE) {
ODP_ERR("Some NICs need at least %dB buffers to not segment "
- "standard ethernet frames. Increase pool seg_len.\n",
- RTE_MBUF_DEFAULT_BUF_SIZE);
+ "standard ethernet frames. Increase pool seg_len.\n",
+ RTE_MBUF_DEFAULT_BUF_SIZE);
goto fail;
}
+ if (odp_shm_info(pool_entry->shm, &shm_info)) {
+ ODP_ERR("Failed to query SHM info.\n");
+ goto fail;
+ }
+
+ page_size = shm_info.page_size;
total_size = rte_mempool_calc_obj_size(elt_size, MEMPOOL_FLAGS, &sz);
if (total_size != pool_entry->block_size) {
ODP_ERR("DPDK pool block size not matching to ODP pool: "
@@ -361,10 +359,7 @@ static struct rte_mempool *mbuf_pool_create(const char *name,
goto fail;
}
- /* Skipped buffers have to be taken into account to populate pool
- * properly. */
- mp = rte_mempool_create_empty(name, num + pool_entry->skipped_blocks,
- elt_size, cache_size(num),
+ mp = rte_mempool_create_empty(name, num, elt_size, cache_size(num),
sizeof(struct rte_pktmbuf_pool_private),
rte_socket_id(), MEMPOOL_FLAGS);
if (mp == NULL) {
@@ -380,16 +375,34 @@ static struct rte_mempool *mbuf_pool_create(const char *name,
}
mbp_priv.mbuf_data_room_size = pool_entry->headroom +
- pool_entry->seg_len;
+ pool_entry->seg_len + pool_entry->tailroom;
mbp_priv.mbuf_priv_size = RTE_ALIGN(sizeof(odp_packet_hdr_t),
RTE_MBUF_PRIV_ALIGN);
rte_pktmbuf_pool_init(mp, &mbp_priv);
- num = rte_mempool_populate_iova(mp, (char *)pool_entry->base_addr,
- RTE_BAD_IOVA, pool_entry->shm_size,
- NULL, NULL);
- if (num <= 0) {
- ODP_ERR("Failed to populate mempool: %d\n", num);
+ /* DPDK expects buffers that would be crossing a hugepage boundary to be aligned to the
+ * boundary. This isn't the case with ODP pools as boundary-crossing buffers are skipped
+ * and unused but still part of the pool. Thus, populate the mempool with several virtually
+ * and physically contiguous chunks as dictated by the skipped buffers. */
+ for (uint64_t i = 0; i < pool_entry->shm_size; i += page_size) {
+ remainder = (page_size - offset) % total_size;
+ addr = pool_entry->base_addr + i + offset;
+ ret = rte_mempool_populate_iova(mp, (char *)addr, rte_mem_virt2iova(addr),
+ page_size - remainder - offset,
+ NULL, NULL);
+
+ if (ret <= 0) {
+ ODP_ERR("Failed to populate mempool: %d\n", ret);
+ goto fail;
+ }
+
+ populated += ret;
+ offset = remainder ? total_size - remainder : 0;
+ }
+
+ if (populated != num) {
+ ODP_ERR("Failed to populate mempool with all requested blocks, populated: %u, "
+ "requested: %u\n", populated, num);
goto fail;
}
@@ -556,7 +569,7 @@ static struct rte_mempool_ops odp_pool_ops = {
.get_count = pool_get_count
};
-MEMPOOL_REGISTER_OPS(odp_pool_ops)
+RTE_MEMPOOL_REGISTER_OPS(odp_pool_ops)
static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry,
odp_packet_t pkt_table[],
@@ -574,9 +587,9 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry,
pkt_dpdk_t *pkt_dpdk = pkt_priv(pktio_entry);
odp_pool_t pool = pkt_dpdk->pool;
odp_pktin_config_opt_t pktin_cfg = pktio_entry->s.config.pktin;
- odp_proto_layer_t parse_layer = pktio_entry->s.config.parser.layer;
odp_pktio_t input = pktio_entry->s.handle;
uint16_t frame_offset = pktio_entry->s.pktin_frame_offset;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
/* Allocate maximum sized packets */
max_len = pkt_dpdk->data_room;
@@ -591,8 +604,6 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry,
}
for (i = 0; i < num; i++) {
- odp_packet_hdr_t parsed_hdr;
-
mbuf = mbuf_table[i];
if (odp_unlikely(mbuf->nb_segs != 1)) {
ODP_ERR("Segmented buffers not supported\n");
@@ -603,31 +614,33 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry,
odp_prefetch(data);
pkt_len = rte_pktmbuf_pkt_len(mbuf);
+ pkt = pkt_table[i];
+ pkt_hdr = packet_hdr(pkt);
- if (pktio_cls_enabled(pktio_entry)) {
+ if (layer) {
uint32_t supported_ptypes = pkt_dpdk->supported_ptypes;
- packet_parse_reset(&parsed_hdr, 1);
- packet_set_len(&parsed_hdr, pkt_len);
- if (_odp_dpdk_packet_parse_common(&parsed_hdr.p, data,
- pkt_len, pkt_len,
- mbuf,
- ODP_PROTO_LAYER_ALL,
- supported_ptypes,
+ packet_parse_reset(pkt_hdr, 1);
+ if (_odp_dpdk_packet_parse_common(&pkt_hdr->p, data,
+ pkt_len, pkt_len, mbuf,
+ layer, supported_ptypes,
pktin_cfg)) {
odp_packet_free(pkt_table[i]);
rte_pktmbuf_free(mbuf);
continue;
}
- if (_odp_cls_classify_packet(pktio_entry,
- (const uint8_t *)data,
- pkt_len, pkt_len, &pool,
- &parsed_hdr, false))
- goto fail;
+
+ if (pktio_cls_enabled(pktio_entry)) {
+ if (_odp_cls_classify_packet(pktio_entry,
+ (const uint8_t *)data,
+ &pool, pkt_hdr)) {
+ odp_packet_free(pkt_table[i]);
+ rte_pktmbuf_free(mbuf);
+ continue;
+ }
+ }
}
- pkt = pkt_table[i];
- pkt_hdr = packet_hdr(pkt);
pull_tail(pkt_hdr, max_len - pkt_len);
if (frame_offset)
pull_head(pkt_hdr, frame_offset);
@@ -637,22 +650,7 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry,
pkt_hdr->input = input;
- if (pktio_cls_enabled(pktio_entry)) {
- _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr);
- } else if (parse_layer != ODP_PROTO_LAYER_NONE) {
- uint32_t supported_ptypes = pkt_dpdk->supported_ptypes;
-
- if (_odp_dpdk_packet_parse_layer(pkt_hdr, mbuf,
- parse_layer,
- supported_ptypes,
- pktin_cfg)) {
- odp_packet_free(pkt);
- rte_pktmbuf_free(mbuf);
- continue;
- }
- }
-
- if (mbuf->ol_flags & PKT_RX_RSS_HASH)
+ if (mbuf->ol_flags & RTE_MBUF_F_RX_RSS_HASH)
packet_set_flow_hash(pkt_hdr, mbuf->hash.rss);
packet_set_ts(pkt_hdr, ts);
@@ -753,12 +751,12 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg,
mbuf->l2_len = pkt_p->l3_offset - pkt_p->l2_offset;
if (l3_proto_v4)
- mbuf->ol_flags = PKT_TX_IPV4;
+ mbuf->ol_flags = RTE_MBUF_F_TX_IPV4;
else
- mbuf->ol_flags = PKT_TX_IPV6;
+ mbuf->ol_flags = RTE_MBUF_F_TX_IPV6;
if (ipv4_chksum_pkt) {
- mbuf->ol_flags |= PKT_TX_IP_CKSUM;
+ mbuf->ol_flags |= RTE_MBUF_F_TX_IP_CKSUM;
((struct rte_ipv4_hdr *)l3_hdr)->hdr_checksum = 0;
mbuf->l3_len = _ODP_IPV4HDR_IHL(*(uint8_t *)l3_hdr) * 4;
@@ -772,12 +770,12 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg,
l4_hdr = (void *)(mbuf_data + pkt_p->l4_offset);
if (udp_chksum_pkt) {
- mbuf->ol_flags |= PKT_TX_UDP_CKSUM;
+ mbuf->ol_flags |= RTE_MBUF_F_TX_UDP_CKSUM;
((struct rte_udp_hdr *)l4_hdr)->dgram_cksum =
phdr_csum(l3_proto_v4, l3_hdr, mbuf->ol_flags);
} else if (tcp_chksum_pkt) {
- mbuf->ol_flags |= PKT_TX_TCP_CKSUM;
+ mbuf->ol_flags |= RTE_MBUF_F_TX_TCP_CKSUM;
((struct rte_tcp_hdr *)l4_hdr)->cksum =
phdr_csum(l3_proto_v4, l3_hdr, mbuf->ol_flags);
@@ -863,9 +861,9 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry,
int i, nb_pkts;
odp_pool_t pool;
odp_pktin_config_opt_t pktin_cfg;
- odp_proto_layer_t parse_layer;
odp_pktio_t input;
pkt_dpdk_t *pkt_dpdk;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
prefetch_pkt(mbuf_table[0]);
@@ -874,15 +872,12 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry,
pool = pkt_dpdk->pool;
set_flow_hash = pkt_dpdk->opt.set_flow_hash;
pktin_cfg = pktio_entry->s.config.pktin;
- parse_layer = pktio_entry->s.config.parser.layer;
input = pktio_entry->s.handle;
if (odp_likely(mbuf_num > 1))
prefetch_pkt(mbuf_table[1]);
for (i = 0; i < mbuf_num; i++) {
- odp_packet_hdr_t parsed_hdr;
-
if (odp_likely((i + 2) < mbuf_num))
prefetch_pkt(mbuf_table[i + 2]);
@@ -895,30 +890,27 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry,
data = rte_pktmbuf_mtod(mbuf, char *);
pkt_len = rte_pktmbuf_pkt_len(mbuf);
-
pkt_hdr = pkt_hdr_from_mbuf(mbuf);
+ packet_init(pkt_hdr, pkt_len);
- if (pktio_cls_enabled(pktio_entry)) {
+ if (layer) {
uint32_t supported_ptypes = pkt_dpdk->supported_ptypes;
- packet_parse_reset(&parsed_hdr, 1);
- packet_set_len(&parsed_hdr, pkt_len);
- if (_odp_dpdk_packet_parse_common(&parsed_hdr.p, data,
- pkt_len, pkt_len,
- mbuf,
- ODP_PROTO_LAYER_ALL,
- supported_ptypes,
+ if (_odp_dpdk_packet_parse_common(&pkt_hdr->p, data,
+ pkt_len, pkt_len, mbuf,
+ layer, supported_ptypes,
pktin_cfg)) {
rte_pktmbuf_free(mbuf);
continue;
}
- if (_odp_cls_classify_packet(pktio_entry,
- (const uint8_t *)data,
- pkt_len, pkt_len, &pool,
- &parsed_hdr, false)) {
- ODP_ERR("Unable to classify packet\n");
- rte_pktmbuf_free(mbuf);
- continue;
+
+ if (pktio_cls_enabled(pktio_entry)) {
+ if (_odp_cls_classify_packet(pktio_entry,
+ (const uint8_t *)data,
+ &pool, pkt_hdr)) {
+ rte_pktmbuf_free(mbuf);
+ continue;
+ }
}
}
@@ -926,23 +918,9 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry,
* are supported. */
pkt_hdr->seg_data = data;
- packet_init(pkt_hdr, pkt_len);
pkt_hdr->input = input;
- if (pktio_cls_enabled(pktio_entry)) {
- _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr);
- } else if (parse_layer != ODP_PROTO_LAYER_NONE) {
- uint32_t supported_ptypes = pkt_dpdk->supported_ptypes;
-
- if (_odp_dpdk_packet_parse_layer(pkt_hdr, mbuf,
- parse_layer,
- supported_ptypes,
- pktin_cfg)) {
- rte_pktmbuf_free(mbuf);
- continue;
- }
- }
- if (set_flow_hash && (mbuf->ol_flags & PKT_RX_RSS_HASH))
+ if (set_flow_hash && (mbuf->ol_flags & RTE_MBUF_F_RX_RSS_HASH))
packet_set_flow_hash(pkt_hdr, mbuf->hash.rss);
packet_set_ts(pkt_hdr, ts);
@@ -1227,11 +1205,6 @@ static int dpdk_close(pktio_entry_t *pktio_entry)
rte_pktmbuf_free(pkt_dpdk->rx_cache[i].s.pkt[idx++]);
}
-#if RTE_VERSION < RTE_VERSION_NUM(17, 8, 0, 0)
- if (pktio_entry->s.state != PKTIO_STATE_OPENED)
- rte_eth_dev_close(pkt_dpdk->port_id);
-#endif
-
return 0;
}
@@ -1381,18 +1354,6 @@ static void dpdk_mempool_free(struct rte_mempool *mp, void *arg ODP_UNUSED)
rte_mempool_free(mp);
}
-/* RTE_ETH_FOREACH_DEV was introduced in v17.8, but causes a build error in
- * v18.2 (only a warning, but our build system treats warnings as errors). */
-#if (RTE_VERSION >= RTE_VERSION_NUM(18, 2, 0, 0)) && \
- (RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0))
- #define ETH_FOREACH_DEV(p) \
- for (p = rte_eth_find_next(0); \
- (unsigned int)p < (unsigned int)RTE_MAX_ETHPORTS; \
- p = rte_eth_find_next(p + 1))
-#elif RTE_VERSION >= RTE_VERSION_NUM(17, 8, 0, 0)
- #define ETH_FOREACH_DEV(p) RTE_ETH_FOREACH_DEV(p)
-#endif
-
static int dpdk_pktio_term(void)
{
uint16_t port_id;
@@ -1400,11 +1361,9 @@ static int dpdk_pktio_term(void)
if (!odp_global_rw->dpdk_initialized)
return 0;
-#if RTE_VERSION >= RTE_VERSION_NUM(17, 8, 0, 0)
- ETH_FOREACH_DEV(port_id) {
+ RTE_ETH_FOREACH_DEV(port_id) {
rte_eth_dev_close(port_id);
}
-#endif
if (!_ODP_DPDK_ZERO_COPY)
rte_mempool_walk(dpdk_mempool_free, NULL);
@@ -1574,16 +1533,18 @@ static int dpdk_init_capability(pktio_entry_t *pktio_entry,
/* Check if setting MTU is supported */
ret = rte_eth_dev_set_mtu(pkt_dpdk->port_id, pkt_dpdk->mtu - _ODP_ETHHDR_LEN);
- if (ret == 0) {
+ /* From DPDK 21.11 onwards, calling rte_eth_dev_set_mtu() before device is configured with
+ * rte_eth_dev_configure() will result in failure. The least hacky (unfortunately still
+ * very hacky) way to continue checking the support is to take into account that the
+ * function will fail earlier with -ENOTSUP if MTU setting is not supported by device than
+ * if the device was not yet configured. */
+ if (ret != -ENOTSUP) {
capa->set_op.op.maxlen = 1;
capa->maxlen.equal = true;
capa->maxlen.min_input = DPDK_MTU_MIN;
capa->maxlen.max_input = pkt_dpdk->mtu_max;
capa->maxlen.min_output = DPDK_MTU_MIN;
capa->maxlen.max_output = pkt_dpdk->mtu_max;
- } else if (ret != -ENOTSUP) {
- ODP_ERR("Failed to set interface MTU: %d\n", ret);
- return -1;
}
ptype_cnt = rte_eth_dev_get_supported_ptypes(pkt_dpdk->port_id,
@@ -1669,15 +1630,6 @@ static int dpdk_init_capability(pktio_entry_t *pktio_entry,
* mode change. Use system call for them. */
static void promisc_mode_check(pkt_dpdk_t *pkt_dpdk)
{
-#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0)
- /* Enable and disable calls do not have return value */
- rte_eth_promiscuous_enable(pkt_dpdk->port_id);
-
- if (!rte_eth_promiscuous_get(pkt_dpdk->port_id))
- pkt_dpdk->vdev_sysc_promisc = 1;
-
- rte_eth_promiscuous_disable(pkt_dpdk->port_id);
-#else
int ret;
ret = rte_eth_promiscuous_enable(pkt_dpdk->port_id);
@@ -1689,7 +1641,6 @@ static void promisc_mode_check(pkt_dpdk_t *pkt_dpdk)
if (ret)
pkt_dpdk->vdev_sysc_promisc = 1;
-#endif
}
static int dpdk_open(odp_pktio_t id ODP_UNUSED,
@@ -1735,18 +1686,17 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED,
pkt_dpdk->pool = pool;
- /* rte_eth_dev_count() was removed in v18.05 */
-#if RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0)
- if (rte_eth_dev_count() == 0) {
-#else
if (rte_eth_dev_count_avail() == 0) {
-#endif
ODP_ERR("No DPDK ports found\n");
return -1;
}
memset(&dev_info, 0, sizeof(struct rte_eth_dev_info));
- rte_eth_dev_info_get(pkt_dpdk->port_id, &dev_info);
+ ret = rte_eth_dev_info_get(pkt_dpdk->port_id, &dev_info);
+ if (ret) {
+ ODP_ERR("Failed to read device info: %d\n", ret);
+ return -1;
+ }
/* Initialize runtime options */
if (init_options(pktio_entry, &dev_info)) {
@@ -1765,18 +1715,10 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED,
promisc_mode_check(pkt_dpdk);
-#if RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0)
- ret = 0;
- if (pkt_dpdk->opt.multicast_en)
- rte_eth_allmulticast_enable(pkt_dpdk->port_id);
- else
- rte_eth_allmulticast_disable(pkt_dpdk->port_id);
-#else
if (pkt_dpdk->opt.multicast_en)
ret = rte_eth_allmulticast_enable(pkt_dpdk->port_id);
else
ret = rte_eth_allmulticast_disable(pkt_dpdk->port_id);
-#endif
/* Not supported by all PMDs, so ignore the return value */
if (ret)
diff --git a/platform/linux-generic/pktio/dpdk_parse.c b/platform/linux-generic/pktio/dpdk_parse.c
index 0984f06c5..f28f18921 100644
--- a/platform/linux-generic/pktio/dpdk_parse.c
+++ b/platform/linux-generic/pktio/dpdk_parse.c
@@ -1,5 +1,5 @@
/* Copyright (c) 2018, Linaro Limited
- * Copyright (c) 2021, Nokia
+ * Copyright (c) 2021-2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -26,9 +26,19 @@
#if defined(__PPC64__) && defined(vector)
#undef vector
#endif
+#include <rte_version.h>
+
+#if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0)
+ #define RTE_MBUF_F_RX_IP_CKSUM_MASK PKT_RX_IP_CKSUM_MASK
+ #define RTE_MBUF_F_RX_IP_CKSUM_GOOD PKT_RX_IP_CKSUM_GOOD
+ #define RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN PKT_RX_IP_CKSUM_UNKNOWN
+ #define RTE_MBUF_F_RX_L4_CKSUM_MASK PKT_RX_L4_CKSUM_MASK
+ #define RTE_MBUF_F_RX_L4_CKSUM_GOOD PKT_RX_L4_CKSUM_GOOD
+ #define RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN PKT_RX_L4_CKSUM_UNKNOWN
+#endif
-#define IP4_CSUM_RESULT(ol_flags) (ol_flags & PKT_RX_IP_CKSUM_MASK)
-#define L4_CSUM_RESULT(ol_flags) (ol_flags & PKT_RX_L4_CKSUM_MASK)
+#define IP4_CSUM_RESULT(ol_flags) ((ol_flags) & RTE_MBUF_F_RX_IP_CKSUM_MASK)
+#define L4_CSUM_RESULT(ol_flags) ((ol_flags) & RTE_MBUF_F_RX_L4_CKSUM_MASK)
/** Parser helper function for Ethernet packets */
static inline uint16_t dpdk_parse_eth(packet_parser_t *prs,
@@ -174,9 +184,9 @@ static inline uint8_t dpdk_parse_ipv4(packet_parser_t *prs,
if (do_csum) {
uint64_t packet_csum_result = IP4_CSUM_RESULT(mbuf_ol);
- if (packet_csum_result == PKT_RX_IP_CKSUM_GOOD) {
+ if (packet_csum_result == RTE_MBUF_F_RX_IP_CKSUM_GOOD) {
prs->input_flags.l3_chksum_done = 1;
- } else if (packet_csum_result != PKT_RX_IP_CKSUM_UNKNOWN) {
+ } else if (packet_csum_result != RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN) {
prs->input_flags.l3_chksum_done = 1;
prs->flags.ip_err = 1;
prs->flags.l3_chksum_err = 1;
@@ -302,9 +312,9 @@ static inline void dpdk_parse_tcp(packet_parser_t *prs,
if (do_csum) {
uint64_t packet_csum_result = L4_CSUM_RESULT(mbuf_ol);
- if (packet_csum_result == PKT_RX_L4_CKSUM_GOOD) {
+ if (packet_csum_result == RTE_MBUF_F_RX_L4_CKSUM_GOOD) {
prs->input_flags.l4_chksum_done = 1;
- } else if (packet_csum_result != PKT_RX_L4_CKSUM_UNKNOWN) {
+ } else if (packet_csum_result != RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN) {
prs->input_flags.l4_chksum_done = 1;
prs->flags.tcp_err = 1;
prs->flags.l4_chksum_err = 1;
@@ -332,9 +342,9 @@ static inline void dpdk_parse_udp(packet_parser_t *prs,
if (do_csum) {
uint64_t packet_csum_result = L4_CSUM_RESULT(mbuf_ol);
- if (packet_csum_result == PKT_RX_L4_CKSUM_GOOD) {
+ if (packet_csum_result == RTE_MBUF_F_RX_L4_CKSUM_GOOD) {
prs->input_flags.l4_chksum_done = 1;
- } else if (packet_csum_result != PKT_RX_L4_CKSUM_UNKNOWN) {
+ } else if (packet_csum_result != RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN) {
if (prs->input_flags.ipv4 && !udp->chksum) {
prs->input_flags.l4_chksum_done = 1;
} else {
diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c
index b3232cded..3e21efecd 100644
--- a/platform/linux-generic/pktio/loop.c
+++ b/platform/linux-generic/pktio/loop.c
@@ -18,6 +18,7 @@
#include <odp/api/plat/packet_flag_inlines.h>
#include <odp/api/plat/queue_inlines.h>
+#include <odp_parse_internal.h>
#include <odp_classification_internal.h>
#include <odp_debug_internal.h>
#include <odp_errno_define.h>
@@ -26,6 +27,7 @@
#include <odp_ipsec_internal.h>
#include <odp_packet_internal.h>
#include <odp_packet_io_internal.h>
+#include <odp_macros_internal.h>
#include <odp_queue_if.h>
#include <protocols/eth.h>
@@ -158,7 +160,11 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
odp_time_t ts_val;
odp_time_t *ts = NULL;
int num_rx = 0;
- int failed = 0;
+ int packets = 0, errors = 0;
+ uint32_t octets = 0;
+ const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
+ const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin;
if (odp_unlikely(num > QUEUE_MULTI_MAX))
num = QUEUE_MULTI_MAX;
@@ -168,8 +174,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
queue = pkt_priv(pktio_entry)->loopq;
nbr = odp_queue_deq_multi(queue, (odp_event_t *)hdr_tbl, num);
- if (pktio_entry->s.config.pktin.bit.ts_all ||
- pktio_entry->s.config.pktin.bit.ts_ptp) {
+ if (opt.bit.ts_all || opt.bit.ts_ptp) {
ts_val = odp_time_global();
ts = &ts_val;
}
@@ -181,61 +186,64 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
pkt_len = odp_packet_len(pkt);
pkt_hdr = packet_hdr(pkt);
- packet_parse_reset(pkt_hdr, 1);
- if (pktio_cls_enabled(pktio_entry)) {
- odp_packet_t new_pkt;
- odp_pool_t new_pool;
+ if (layer) {
uint8_t *pkt_addr;
- uint8_t buf[PACKET_PARSE_SEG_LEN];
+ uint8_t buf[PARSE_BYTES];
int ret;
uint32_t seg_len = odp_packet_seg_len(pkt);
+ uint64_t l4_part_sum = 0;
/* Make sure there is enough data for the packet
* parser in the case of a segmented packet. */
- if (odp_unlikely(seg_len < PACKET_PARSE_SEG_LEN &&
- pkt_len > PACKET_PARSE_SEG_LEN)) {
- odp_packet_copy_to_mem(pkt, 0,
- PACKET_PARSE_SEG_LEN,
- buf);
- seg_len = PACKET_PARSE_SEG_LEN;
+ if (odp_unlikely(seg_len < PARSE_BYTES &&
+ pkt_len > seg_len)) {
+ seg_len = MIN(pkt_len, PARSE_BYTES);
+ odp_packet_copy_to_mem(pkt, 0, seg_len, buf);
pkt_addr = buf;
} else {
pkt_addr = odp_packet_data(pkt);
}
- ret = _odp_cls_classify_packet(pktio_entry, pkt_addr,
- pkt_len, seg_len,
- &new_pool, pkt_hdr, true);
- if (ret) {
- failed++;
+ packet_parse_reset(pkt_hdr, 1);
+ ret = _odp_packet_parse_common(&pkt_hdr->p, pkt_addr, pkt_len,
+ seg_len, layer, chksums,
+ &l4_part_sum, opt);
+ if (ret)
+ errors++;
+
+ if (ret < 0) {
odp_packet_free(pkt);
continue;
}
- if (new_pool != odp_packet_pool(pkt)) {
- new_pkt = odp_packet_copy(pkt, new_pool);
-
- odp_packet_free(pkt);
+ if (pktio_cls_enabled(pktio_entry)) {
+ odp_packet_t new_pkt;
+ odp_pool_t new_pool;
- if (new_pkt == ODP_PACKET_INVALID) {
- failed++;
+ ret = _odp_cls_classify_packet(pktio_entry, pkt_addr,
+ &new_pool, pkt_hdr);
+ if (ret) {
+ odp_packet_free(pkt);
continue;
}
- pkt = new_pkt;
- pkt_hdr = packet_hdr(new_pkt);
+ if (new_pool != odp_packet_pool(pkt)) {
+ new_pkt = odp_packet_copy(pkt, new_pool);
+
+ odp_packet_free(pkt);
+
+ if (new_pkt == ODP_PACKET_INVALID) {
+ pktio_entry->s.stats.in_discards++;
+ continue;
+ }
+
+ pkt = new_pkt;
+ pkt_hdr = packet_hdr(new_pkt);
+ }
}
- } else {
- odp_packet_parse_param_t param;
-
- /*
- * Use odp_packet_parse() which can handle segmented
- * packets.
- */
- param.proto = ODP_PROTO_ETH;
- param.last_layer = pktio_entry->s.config.parser.layer;
- param.chksums = pktio_entry->s.in_chksums;
- odp_packet_parse(packet_handle(pkt_hdr), 0, &param);
+
+ if (layer >= ODP_PROTO_LAYER_L4)
+ _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum);
}
packet_set_ts(pkt_hdr, ts);
@@ -247,12 +255,17 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
odp_packet_has_ipsec(pkt))
_odp_ipsec_try_inline(&pkt);
- pktio_entry->s.stats.in_octets += pkt_len;
+ if (!pkt_hdr->p.flags.all.error) {
+ octets += pkt_len;
+ packets++;
+ }
+
pkts[num_rx++] = pkt;
}
- pktio_entry->s.stats.in_errors += failed;
- pktio_entry->s.stats.in_packets += num_rx - failed;
+ pktio_entry->s.stats.in_octets += octets;
+ pktio_entry->s.stats.in_packets += packets;
+ pktio_entry->s.stats.in_errors += errors;
odp_ticketlock_unlock(&pktio_entry->s.rxl);
@@ -533,6 +546,7 @@ static int loopback_init_capability(pktio_entry_t *pktio_entry)
capa->stats.pktio.counter.in_octets = 1;
capa->stats.pktio.counter.in_packets = 1;
capa->stats.pktio.counter.in_errors = 1;
+ capa->stats.pktio.counter.in_discards = 1;
capa->stats.pktio.counter.out_octets = 1;
capa->stats.pktio.counter.out_packets = 1;
capa->stats.pktin_queue.counter.octets = 1;
diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c
index 0b1345e50..94b88e21e 100644
--- a/platform/linux-generic/pktio/netmap.c
+++ b/platform/linux-generic/pktio/netmap.c
@@ -16,6 +16,7 @@
#include <odp/api/time.h>
#include <odp/api/plat/time_inlines.h>
+#include <odp_parse_internal.h>
#include <odp_packet_io_internal.h>
#include <odp_packet_io_stats.h>
#include <odp_ethtool_stats.h>
@@ -822,11 +823,14 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry,
odp_packet_t pkt;
odp_pool_t pool = pkt_priv(pktio_entry)->pool;
odp_packet_hdr_t *pkt_hdr;
- odp_packet_hdr_t parsed_hdr;
int i;
int num;
uint32_t max_len;
uint16_t frame_offset = pktio_entry->s.pktin_frame_offset;
+ int num_rx = 0;
+ const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
+ const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin;
/* Allocate maximum sized packets */
max_len = pkt_priv(pktio_entry)->mtu;
@@ -837,45 +841,50 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry,
for (i = 0; i < num; i++) {
netmap_slot_t slot;
uint16_t len;
+ const uint8_t *buf;
+ uint64_t l4_part_sum = 0;
slot = slot_tbl[i];
len = slot.len;
+ buf = (const uint8_t *)slot.buf;
odp_prefetch(slot.buf);
- if (pktio_cls_enabled(pktio_entry)) {
- if (_odp_cls_classify_packet(pktio_entry,
- (const uint8_t *)slot.buf, len,
- len, &pool, &parsed_hdr, true))
- goto fail;
+ pkt = pkt_tbl[num_rx];
+ pkt_hdr = packet_hdr(pkt);
+
+ if (layer) {
+ if (_odp_packet_parse_common(&pkt_hdr->p, buf, len, len,
+ layer, chksums, &l4_part_sum, opt) < 0)
+ continue;
+
+ if (pktio_cls_enabled(pktio_entry)) {
+ if (_odp_cls_classify_packet(pktio_entry, buf, &pool,
+ pkt_hdr))
+ continue;
+ }
}
- pkt = pkt_tbl[i];
- pkt_hdr = packet_hdr(pkt);
pull_tail(pkt_hdr, max_len - len);
if (frame_offset)
pull_head(pkt_hdr, frame_offset);
if (odp_packet_copy_from_mem(pkt, 0, len, slot.buf) != 0)
- goto fail;
+ break;
pkt_hdr->input = pktio_entry->s.handle;
- if (pktio_cls_enabled(pktio_entry))
- _odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr);
- else
- _odp_packet_parse_layer(pkt_hdr,
- pktio_entry->s.config.parser.layer,
- pktio_entry->s.in_chksums);
+ if (layer >= ODP_PROTO_LAYER_L4)
+ _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum);
packet_set_ts(pkt_hdr, ts);
+ num_rx++;
}
- return i;
+ if (num_rx < num)
+ odp_packet_free_multi(&pkt_tbl[num_rx], num - num_rx);
-fail:
- odp_packet_free_multi(&pkt_tbl[i], num - i);
- return i;
+ return num_rx;
}
static inline int netmap_recv_desc(pktio_entry_t *pktio_entry,
@@ -1042,11 +1051,11 @@ static int netmap_recv_tmo(pktio_entry_t *pktio_entry, int index,
}
static int netmap_recv_mq_tmo(pktio_entry_t *pktio_entry[], int index[],
- int num_q, odp_packet_t pkt_table[], int num,
- unsigned *from, uint64_t usecs)
+ uint32_t num_q, odp_packet_t pkt_table[], int num,
+ uint32_t *from, uint64_t usecs)
{
struct timeval timeout;
- int i;
+ uint32_t i;
int ret;
int maxfd = -1, maxfd2;
fd_set readfds;
diff --git a/platform/linux-generic/pktio/null.c b/platform/linux-generic/pktio/null.c
index 90e113ec6..a6498da8c 100644
--- a/platform/linux-generic/pktio/null.c
+++ b/platform/linux-generic/pktio/null.c
@@ -71,9 +71,9 @@ static int null_recv_tmo(pktio_entry_t *pktio_entry ODP_UNUSED,
}
static int null_recv_mq_tmo(pktio_entry_t *pktio_entry[] ODP_UNUSED,
- int index[] ODP_UNUSED, int num_q ODP_UNUSED,
+ int index[] ODP_UNUSED, uint32_t num_q ODP_UNUSED,
odp_packet_t pkt_table[] ODP_UNUSED,
- int num ODP_UNUSED, unsigned *from ODP_UNUSED,
+ int num ODP_UNUSED, uint32_t *from ODP_UNUSED,
uint64_t usecs)
{
struct timeval timeout;
diff --git a/platform/linux-generic/pktio/pcap.c b/platform/linux-generic/pktio/pcap.c
index 7290626c2..af94ffa72 100644
--- a/platform/linux-generic/pktio/pcap.c
+++ b/platform/linux-generic/pktio/pcap.c
@@ -46,6 +46,7 @@
#include <odp/api/plat/packet_inlines.h>
+#include <odp_parse_internal.h>
#include <odp_classification_internal.h>
#include <odp_debug_internal.h>
#include <odp_global_data.h>
@@ -248,7 +249,12 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
pkt_pcap_t *pcap = pkt_priv(pktio_entry);
odp_time_t ts_val;
odp_time_t *ts = NULL;
+ int packets = 0, errors = 0;
+ uint32_t octets = 0;
uint16_t frame_offset = pktio_entry->s.pktin_frame_offset;
+ const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
+ const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin;
odp_ticketlock_lock(&pktio_entry->s.rxl);
@@ -256,8 +262,7 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
odp_ticketlock_unlock(&pktio_entry->s.rxl);
return 0;
}
- if (pktio_entry->s.config.pktin.bit.ts_all ||
- pktio_entry->s.config.pktin.bit.ts_ptp)
+ if (opt.bit.ts_all || opt.bit.ts_ptp)
ts = &ts_val;
for (i = 0; i < num; ) {
@@ -291,42 +296,64 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
break;
}
- if (pktio_cls_enabled(pktio_entry)) {
- odp_packet_t new_pkt;
+ if (layer) {
+ uint64_t l4_part_sum = 0;
- ret = _odp_cls_classify_packet(pktio_entry, data,
- pkt_len, pkt_len,
- &new_pool, pkt_hdr, true);
- if (ret) {
+ ret = _odp_packet_parse_common(&pkt_hdr->p, data, pkt_len,
+ pkt_len, layer, chksums,
+ &l4_part_sum, opt);
+ if (ret)
+ errors++;
+
+ if (ret < 0) {
odp_packet_free(pkt);
continue;
}
- if (new_pool != pcap->pool) {
- new_pkt = odp_packet_copy(pkt, new_pool);
- odp_packet_free(pkt);
+ if (pktio_cls_enabled(pktio_entry)) {
+ odp_packet_t new_pkt;
- if (odp_unlikely(new_pkt == ODP_PACKET_INVALID))
+ ret = _odp_cls_classify_packet(pktio_entry, data,
+ &new_pool, pkt_hdr);
+ if (ret) {
+ odp_packet_free(pkt);
continue;
+ }
+ if (new_pool != pcap->pool) {
+ new_pkt = odp_packet_copy(pkt, new_pool);
+
+ odp_packet_free(pkt);
+
+ if (odp_unlikely(new_pkt == ODP_PACKET_INVALID)) {
+ pktio_entry->s.stats.in_discards++;
+ continue;
+ }
- pkt = new_pkt;
- pkt_hdr = packet_hdr(new_pkt);
+ pkt = new_pkt;
+ pkt_hdr = packet_hdr(new_pkt);
+ }
}
- } else {
- _odp_packet_parse_layer(pkt_hdr,
- pktio_entry->s.config.parser.layer,
- pktio_entry->s.in_chksums);
+
+ if (layer >= ODP_PROTO_LAYER_L4)
+ _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum);
}
- pktio_entry->s.stats.in_octets += pkt_hdr->frame_len;
packet_set_ts(pkt_hdr, ts);
pkt_hdr->input = pktio_entry->s.handle;
+ if (!pkt_hdr->p.flags.all.error) {
+ octets += pkt_len;
+ packets++;
+ }
+
pkts[i] = pkt;
i++;
}
- pktio_entry->s.stats.in_packets += i;
+
+ pktio_entry->s.stats.in_octets += octets;
+ pktio_entry->s.stats.in_packets += packets;
+ pktio_entry->s.stats.in_errors += errors;
odp_ticketlock_unlock(&pktio_entry->s.rxl);
@@ -441,6 +468,8 @@ static int pcapif_capability(pktio_entry_t *pktio_entry ODP_UNUSED,
capa->stats.pktio.counter.in_octets = 1;
capa->stats.pktio.counter.in_packets = 1;
+ capa->stats.pktio.counter.in_discards = 1;
+ capa->stats.pktio.counter.in_errors = 1;
capa->stats.pktio.counter.out_octets = 1;
capa->stats.pktio.counter.out_packets = 1;
diff --git a/platform/linux-generic/pktio/pktio_common.c b/platform/linux-generic/pktio/pktio_common.c
index d2f4d7219..1f3cf951e 100644
--- a/platform/linux-generic/pktio/pktio_common.c
+++ b/platform/linux-generic/pktio/pktio_common.c
@@ -11,13 +11,13 @@
static int sock_recv_mq_tmo_select(pktio_entry_t * const *entry,
const int index[],
- unsigned int num_q, unsigned int *from,
+ uint32_t num_q, uint32_t *from,
odp_packet_t packets[], int num,
uint64_t usecs, fd_set *readfds,
int maxfd)
{
struct timeval timeout;
- unsigned int i;
+ uint32_t i;
int ret;
for (i = 0; i < num_q; i++) {
@@ -50,17 +50,17 @@ static int sock_recv_mq_tmo_select(pktio_entry_t * const *entry,
}
int _odp_sock_recv_mq_tmo_try_int_driven(const struct odp_pktin_queue_t queues[],
- unsigned int num_q, unsigned int *from,
+ uint32_t num_q, uint32_t *from,
odp_packet_t packets[], int num,
uint64_t usecs, int *trial_successful)
{
- unsigned int i;
+ uint32_t i;
pktio_entry_t *entry[num_q];
int index[num_q];
fd_set readfds;
int maxfd = -1;
- int (*impl)(pktio_entry_t *entry[], int index[], int num_q,
- odp_packet_t packets[], int num, unsigned int *from,
+ int (*impl)(pktio_entry_t *entry[], int index[], uint32_t num_q,
+ odp_packet_t packets[], int num, uint32_t *from,
uint64_t wait_usecs) = NULL;
int impl_set = 0;
diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c
index 81c178b9d..9d1bbe545 100644
--- a/platform/linux-generic/pktio/socket.c
+++ b/platform/linux-generic/pktio/socket.c
@@ -15,12 +15,14 @@
#include <odp/api/ticketlock.h>
#include <odp_socket_common.h>
+#include <odp_parse_internal.h>
#include <odp_packet_internal.h>
#include <odp_packet_io_internal.h>
#include <odp_packet_io_stats.h>
#include <odp_debug_internal.h>
#include <odp_errno_define.h>
#include <odp_classification_internal.h>
+#include <odp_macros_internal.h>
#include <sys/socket.h>
#include <stdio.h>
@@ -233,6 +235,9 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
int i;
uint16_t frame_offset = pktio_entry->s.pktin_frame_offset;
uint32_t alloc_len = pkt_sock->mtu + frame_offset;
+ const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
+ const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin;
memset(msgvec, 0, sizeof(msgvec));
@@ -249,8 +254,7 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
recv_msgs = recvmmsg(sockfd, msgvec, nb_pkts, MSG_DONTWAIT, NULL);
odp_ticketlock_unlock(&pkt_sock->rx_lock);
- if (pktio_entry->s.config.pktin.bit.ts_all ||
- pktio_entry->s.config.pktin.bit.ts_ptp) {
+ if (opt.bit.ts_all || opt.bit.ts_ptp) {
ts_val = odp_time_global();
ts = &ts_val;
}
@@ -262,25 +266,48 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
uint16_t pkt_len = msgvec[i].msg_len;
int ret;
+ uint64_t l4_part_sum = 0;
if (odp_unlikely(msgvec[i].msg_hdr.msg_flags & MSG_TRUNC)) {
odp_packet_free(pkt);
ODP_DBG("dropped truncated packet\n");
continue;
}
- if (pktio_cls_enabled(pktio_entry)) {
- uint16_t seg_len = pkt_len;
- if (msgvec[i].msg_hdr.msg_iov->iov_len < pkt_len)
- seg_len = msgvec[i].msg_hdr.msg_iov->iov_len;
+ ret = odp_packet_trunc_tail(&pkt, odp_packet_len(pkt) - pkt_len,
+ NULL, NULL);
+ if (ret < 0) {
+ ODP_ERR("trunc_tail failed");
+ odp_packet_free(pkt);
+ continue;
+ }
+
+ if (layer) {
+ uint8_t buf[PARSE_BYTES];
+ uint16_t seg_len = msgvec[i].msg_hdr.msg_iov->iov_len;
- if (_odp_cls_classify_packet(pktio_entry, base, pkt_len,
- seg_len, &pool, pkt_hdr,
- true)) {
- ODP_ERR("_odp_cls_classify_packet failed");
+ /* Make sure there is enough data for the packet
+ * parser in the case of a segmented packet. */
+ if (odp_unlikely(seg_len < PARSE_BYTES && pkt_len > seg_len)) {
+ seg_len = MIN(pkt_len, PARSE_BYTES);
+ odp_packet_copy_to_mem(pkt, 0, seg_len, buf);
+ base = buf;
+ }
+
+ if (_odp_packet_parse_common(&pkt_hdr->p, base, pkt_len,
+ seg_len, layer, chksums,
+ &l4_part_sum, opt) < 0) {
odp_packet_free(pkt);
continue;
}
+
+ if (pktio_cls_enabled(pktio_entry)) {
+ if (_odp_cls_classify_packet(pktio_entry, base, &pool,
+ pkt_hdr)) {
+ odp_packet_free(pkt);
+ continue;
+ }
+ }
}
/* Don't receive packets sent by ourselves */
@@ -290,20 +317,10 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
continue;
}
- ret = odp_packet_trunc_tail(&pkt, odp_packet_len(pkt) - pkt_len,
- NULL, NULL);
- if (ret < 0) {
- ODP_ERR("trunk_tail failed");
- odp_packet_free(pkt);
- continue;
- }
-
pkt_hdr->input = pktio_entry->s.handle;
- if (!pktio_cls_enabled(pktio_entry))
- _odp_packet_parse_layer(pkt_hdr,
- pktio_entry->s.config.parser.layer,
- pktio_entry->s.in_chksums);
+ if (layer >= ODP_PROTO_LAYER_L4)
+ _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum);
packet_set_ts(pkt_hdr, ts);
@@ -359,11 +376,11 @@ static int sock_recv_tmo(pktio_entry_t *pktio_entry, int index,
}
static int sock_recv_mq_tmo(pktio_entry_t *pktio_entry[], int index[],
- int num_q, odp_packet_t pkt_table[], int num,
- unsigned *from, uint64_t usecs)
+ uint32_t num_q, odp_packet_t pkt_table[], int num,
+ uint32_t *from, uint64_t usecs)
{
struct timeval timeout;
- int i;
+ uint32_t i;
int ret;
int maxfd = -1, maxfd2;
fd_set readfds;
diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c
index f84834610..7824b0e91 100644
--- a/platform/linux-generic/pktio/socket_mmap.c
+++ b/platform/linux-generic/pktio/socket_mmap.c
@@ -16,6 +16,7 @@
#include <odp/api/plat/packet_inlines.h>
#include <odp_socket_common.h>
+#include <odp_parse_internal.h>
#include <odp_packet_internal.h>
#include <odp_packet_io_internal.h>
#include <odp_packet_io_stats.h>
@@ -150,9 +151,11 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry,
odp_pool_t pool = pkt_sock->pool;
uint16_t frame_offset = pktio_entry->s.pktin_frame_offset;
uint16_t vlan_len = 0;
+ const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
+ const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin;
- if (pktio_entry->s.config.pktin.bit.ts_all ||
- pktio_entry->s.config.pktin.bit.ts_ptp)
+ if (opt.bit.ts_all || opt.bit.ts_ptp)
ts = &ts_val;
ring = &pkt_sock->rx_ring;
@@ -163,8 +166,8 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry,
struct tpacket2_hdr *tp_hdr;
odp_packet_t pkt;
odp_packet_hdr_t *hdr;
- odp_packet_hdr_t parsed_hdr;
int ret;
+ uint64_t l4_part_sum = 0;
tp_hdr = (void *)next_ptr;
@@ -212,18 +215,29 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry,
continue;
}
- if (pktio_cls_enabled(pktio_entry)) {
- if (_odp_cls_classify_packet(pktio_entry, pkt_buf, pkt_len,
- pkt_len, &pool, &parsed_hdr,
- true)) {
+ hdr = packet_hdr(pkt);
+
+ if (layer) {
+ if (_odp_packet_parse_common(&hdr->p, pkt_buf, pkt_len,
+ pkt_len, layer, chksums,
+ &l4_part_sum, opt) < 0) {
odp_packet_free(pkt);
tp_hdr->tp_status = TP_STATUS_KERNEL;
frame_num = next_frame_num;
continue;
}
+
+ if (pktio_cls_enabled(pktio_entry)) {
+ if (_odp_cls_classify_packet(pktio_entry, pkt_buf,
+ &pool, hdr)) {
+ odp_packet_free(pkt);
+ tp_hdr->tp_status = TP_STATUS_KERNEL;
+ frame_num = next_frame_num;
+ continue;
+ }
+ }
}
- hdr = packet_hdr(pkt);
if (frame_offset)
pull_head(hdr, frame_offset);
@@ -276,12 +290,8 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry,
hdr->input = pktio_entry->s.handle;
- if (pktio_cls_enabled(pktio_entry))
- _odp_packet_copy_cls_md(hdr, &parsed_hdr);
- else
- _odp_packet_parse_layer(hdr,
- pktio_entry->s.config.parser.layer,
- pktio_entry->s.in_chksums);
+ if (layer >= ODP_PROTO_LAYER_L4)
+ _odp_packet_l4_chksum(hdr, chksums, l4_part_sum);
packet_set_ts(hdr, ts);
@@ -727,11 +737,11 @@ static int sock_mmap_recv_tmo(pktio_entry_t *pktio_entry, int index,
}
static int sock_mmap_recv_mq_tmo(pktio_entry_t *pktio_entry[], int index[],
- int num_q, odp_packet_t pkt_table[], int num,
- unsigned *from, uint64_t usecs)
+ uint32_t num_q, odp_packet_t pkt_table[], int num,
+ uint32_t *from, uint64_t usecs)
{
struct timeval timeout;
- int i;
+ uint32_t i;
int ret;
int maxfd = -1, maxfd2;
fd_set readfds;
diff --git a/platform/linux-generic/pktio/tap.c b/platform/linux-generic/pktio/tap.c
index d298c8982..40c46b43c 100644
--- a/platform/linux-generic/pktio/tap.c
+++ b/platform/linux-generic/pktio/tap.c
@@ -38,6 +38,7 @@
#include <odp/api/plat/packet_inlines.h>
+#include <odp_parse_internal.h>
#include <odp_debug_internal.h>
#include <odp_socket_common.h>
#include <odp_packet_internal.h>
@@ -286,14 +287,27 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data,
odp_packet_hdr_t *pkt_hdr;
odp_packet_hdr_t parsed_hdr;
int num;
+ uint64_t l4_part_sum = 0;
uint16_t frame_offset = pktio_entry->s.pktin_frame_offset;
-
- if (pktio_cls_enabled(pktio_entry)) {
- if (_odp_cls_classify_packet(pktio_entry, data, len, len,
- &pkt_priv(pktio_entry)->pool,
- &parsed_hdr, true)) {
+ const odp_proto_chksums_t chksums = pktio_entry->s.in_chksums;
+ const odp_proto_layer_t layer = pktio_entry->s.parse_layer;
+ const odp_pktin_config_opt_t opt = pktio_entry->s.config.pktin;
+
+ if (layer) {
+ packet_parse_reset(&parsed_hdr, 1);
+ packet_set_len(&parsed_hdr, len);
+ if (_odp_packet_parse_common(&parsed_hdr.p, data, len, len, layer,
+ chksums, &l4_part_sum, opt) < 0) {
return ODP_PACKET_INVALID;
}
+
+ if (pktio_cls_enabled(pktio_entry)) {
+ if (_odp_cls_classify_packet(pktio_entry, data,
+ &pkt_priv(pktio_entry)->pool,
+ &parsed_hdr)) {
+ return ODP_PACKET_INVALID;
+ }
+ }
}
num = _odp_packet_alloc_multi(pkt_priv(pktio_entry)->pool,
@@ -312,12 +326,12 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data,
return ODP_PACKET_INVALID;
}
- if (pktio_cls_enabled(pktio_entry))
+ if (layer) {
_odp_packet_copy_cls_md(pkt_hdr, &parsed_hdr);
- else
- _odp_packet_parse_layer(pkt_hdr,
- pktio_entry->s.config.parser.layer,
- pktio_entry->s.in_chksums);
+
+ if (layer >= ODP_PROTO_LAYER_L4)
+ _odp_packet_l4_chksum(pkt_hdr, chksums, l4_part_sum);
+ }
packet_set_ts(pkt_hdr, ts);
pkt_hdr->input = pktio_entry->s.handle;
@@ -335,6 +349,7 @@ static int tap_pktio_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
uint8_t buf[mtu];
odp_time_t ts_val;
odp_time_t *ts = NULL;
+ int num_rx = 0;
odp_ticketlock_lock(&pktio_entry->s.rxl);
@@ -355,14 +370,15 @@ static int tap_pktio_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
break;
}
- pkts[i] = pack_odp_pkt(pktio_entry, buf, retval, ts);
- if (pkts[i] == ODP_PACKET_INVALID)
+ pkts[num_rx] = pack_odp_pkt(pktio_entry, buf, retval, ts);
+ if (pkts[num_rx] == ODP_PACKET_INVALID)
break;
+ num_rx++;
}
odp_ticketlock_unlock(&pktio_entry->s.rxl);
- return i;
+ return num_rx;
}
static int tap_pktio_send_lockless(pktio_entry_t *pktio_entry,
diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh
index 8939aa485..d9f50225b 100755
--- a/scripts/ci/build.sh
+++ b/scripts/ci/build.sh
@@ -40,8 +40,7 @@ fi
CC="${CC:-${TARGET_ARCH}-gcc}"
${CC} ${CFLAGS} ${EXTRA_CHECKS} ${OLDPWD}/example/sysinfo/odp_sysinfo.c -o odp_sysinfo_inst_dynamic \
- `PKG_CONFIG_PATH=/opt/odp/lib/pkgconfig ${PKG_CONFIG} --cflags --libs ${ODP_LIB_NAME}` \
- `${PKG_CONFIG} --cflags --libs libdpdk`
+ `PKG_CONFIG_PATH=/opt/odp/lib/pkgconfig:${PKG_CONFIG_PATH} ${PKG_CONFIG} --cflags --libs ${ODP_LIB_NAME}`
sysctl vm.nr_hugepages=1000
mkdir -p /mnt/huge
diff --git a/scripts/ci/build_arm64.sh b/scripts/ci/build_arm64.sh
index 79b55e531..f340a0ca0 100755
--- a/scripts/ci/build_arm64.sh
+++ b/scripts/ci/build_arm64.sh
@@ -6,18 +6,27 @@ if [[ $(uname -m) =~ ^(arm64|aarch64)$ ]]; then
export BUILD_ARCH=aarch64-linux-gnu
fi
-if [ "${CC#clang}" != "${CC}" ] ; then
- export CC="clang --target=${TARGET_ARCH}"
- export CXX="clang++ --target=${TARGET_ARCH}"
+if [ "$TARGET_ARCH" == "$BUILD_ARCH" ]; then
+ # Native build
+ if [ "${CC#clang}" != "${CC}" ] ; then
+ export CXX="clang++"
+ fi
else
- export CC="${TARGET_ARCH}-gcc"
- export CXX="${TARGET_ARCH}-g++"
-fi
-export CPPFLAGS="-I/usr/include/${TARGET_ARCH}/dpdk"
+ # Cross compilation
+ if [ "${CC#clang}" != "${CC}" ] ; then
+ export CC="clang --target=${TARGET_ARCH}"
+ export CXX="clang++ --target=${TARGET_ARCH}"
+ else
+ export CC="${TARGET_ARCH}-gcc"
+ export CXX="${TARGET_ARCH}-g++"
+ fi
+
+ export CPPFLAGS="-I/usr/include/${TARGET_ARCH}/dpdk"
-# Use target libraries
-export PKG_CONFIG_PATH=
-export PKG_CONFIG_LIBDIR=/usr/lib/${TARGET_ARCH}/pkgconfig:/usr/local/lib/${TARGET_ARCH}/pkgconfig
+ # Use target libraries
+ export PKG_CONFIG_PATH=
+ export PKG_CONFIG_LIBDIR=/usr/lib/${TARGET_ARCH}/pkgconfig:/usr/local/lib/${TARGET_ARCH}/pkgconfig
+fi
# ARMv8 crypto
export PKG_CONFIG_PATH=~/aarch64cryptolib/pkgconfig:$PKG_CONFIG_PATH
diff --git a/test/performance/odp_bench_packet.c b/test/performance/odp_bench_packet.c
index 0354ef9b8..23fd17bea 100644
--- a/test/performance/odp_bench_packet.c
+++ b/test/performance/odp_bench_packet.c
@@ -756,7 +756,7 @@ static int bench_packet_headroom(void)
for (i = 0; i < TEST_REPEAT_COUNT; i++)
ret += odp_packet_headroom(gbl_args->pkt_tbl[i]);
- return i;
+ return i + ret;
}
static int bench_packet_tailroom(void)
@@ -767,7 +767,7 @@ static int bench_packet_tailroom(void)
for (i = 0; i < TEST_REPEAT_COUNT; i++)
ret += odp_packet_tailroom(gbl_args->pkt_tbl[i]);
- return i;
+ return i + ret;
}
static int bench_packet_tail(void)
@@ -1412,7 +1412,7 @@ static int bench_packet_has_ref(void)
for (i = 0; i < TEST_REPEAT_COUNT; i++)
ret += odp_packet_has_ref(pkt_tbl[i]);
- return i;
+ return i + ret;
}
static int bench_packet_subtype(void)
diff --git a/test/performance/odp_crypto.c b/test/performance/odp_crypto.c
index a26d072e9..45e770d42 100644
--- a/test/performance/odp_crypto.c
+++ b/test/performance/odp_crypto.c
@@ -610,7 +610,6 @@ create_session_from_config(odp_crypto_session_t *session,
odp_crypto_session_param_init(&params);
memcpy(&params, &config->session, sizeof(odp_crypto_session_param_t));
params.op = ODP_CRYPTO_OP_ENCODE;
- params.pref_mode = ODP_CRYPTO_SYNC;
/* Lookup the packet pool */
pkt_pool = odp_pool_lookup("packet_pool");
diff --git a/test/performance/odp_ipsec.c b/test/performance/odp_ipsec.c
index 5f1f9d4ef..c464ba0d7 100644
--- a/test/performance/odp_ipsec.c
+++ b/test/performance/odp_ipsec.c
@@ -1,4 +1,6 @@
/* Copyright (c) 2018, Linaro Limited
+ * Copyright (c) 2022, Marvell
+ * Copyright (c) 2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -18,11 +20,14 @@
#include <odp_api.h>
#include <odp/helper/odph_api.h>
+#include <inttypes.h>
/** @def POOL_NUM_PKT
* Number of packets in the pool
*/
-#define POOL_NUM_PKT 64
+#define POOL_NUM_PKT 4096
+
+#define MAX_DEQUEUE_BURST 16
static uint8_t test_salt[16] = "0123456789abcdef";
@@ -97,11 +102,11 @@ typedef struct {
int in_flight;
/**
- * Number of iteration to repeat crypto operation to get good
- * average number. Specified through -i or --terations option.
+ * Number of packets to be IPsec processed to get good average number.
+ * Specified through -c or --count option.
* Default is 10000.
*/
- int iteration_count;
+ int packet_count;
/**
* Payload size to test. If 0 set of predefined payload sizes
@@ -139,6 +144,19 @@ typedef struct {
* Specified through -u argument.
*/
int ah;
+
+ /*
+ * Burst size.
+ * Prepare and submit as many packets for IPsec processing in each
+ * iteration of the loop.
+ */
+ int burst_size;
+
+ /*
+ * Use vector packet completion from IPsec APIs.
+ * Specified through -v or --vector argument.
+ */
+ uint32_t vec_pkt_size;
} ipsec_args_t;
/*
@@ -516,7 +534,7 @@ print_result(ipsec_args_t *cargs,
throughput = (1000000.0 / result->elapsed) * payload_length / 1024;
printf("%30.30s %15d %15d %15.3f %15.3f %15.3f %15d\n",
- config->name, cargs->iteration_count, payload_length,
+ config->name, cargs->packet_count, payload_length,
result->elapsed, result->rusage_self, result->rusage_thread,
throughput);
}
@@ -584,28 +602,49 @@ static uint8_t test_data[] = {
0x08, 0x00, 0xfb, 0x37, 0x12, 0x34, 0x00, 0x00
};
-static odp_packet_t
-make_packet(odp_pool_t pkt_pool, unsigned int payload_length)
+static inline void debug_packets(int debug, odp_packet_t *pkt, int num_pkts)
{
- odp_packet_t pkt;
+ if (odp_likely(!debug))
+ return;
+ for (int i = 0; i < num_pkts; i++)
+ odp_packet_print_data(pkt[i], 0, odp_packet_len(pkt[i]));
+}
- if (payload_length < sizeof(test_data))
- return ODP_PACKET_INVALID;
+static int
+make_packet_multi(odp_pool_t pkt_pool, unsigned int payload_length,
+ odp_packet_t pkt[], int num)
+{
+ int i, ret;
+
+ ret = odp_packet_alloc_multi(pkt_pool, payload_length, pkt, num);
+ if (ret != num) {
+ ODPH_ERR("Could not allocate buffer\n");
+ if (ret > 0)
+ odp_packet_free_sp(pkt, ret);
+ return -1;
+ }
+
+ for (i = 0; i < num; i++) {
+ odp_packet_copy_from_mem(pkt[i], 0, sizeof(test_data), test_data);
+ odp_packet_l3_offset_set(pkt[i], 0);
- pkt = odp_packet_alloc(pkt_pool, payload_length);
- if (pkt == ODP_PACKET_INVALID) {
- ODPH_ERR("failed to allocate buffer\n");
- return pkt;
+ uint8_t *mem = odp_packet_data(pkt[i]);
+ ((odph_ipv4hdr_t *)mem)->tot_len = odp_cpu_to_be_16(payload_length);
+ memset(mem + sizeof(test_data), 1, payload_length - sizeof(test_data));
}
- odp_packet_copy_from_mem(pkt, 0, sizeof(test_data), test_data);
- odp_packet_l3_offset_set(pkt, 0);
+ return 0;
+}
- uint8_t *mem = odp_packet_data(pkt);
- ((odph_ipv4hdr_t *)mem)->tot_len = odp_cpu_to_be_16(payload_length);
- memset(mem + sizeof(test_data), 1, payload_length - sizeof(test_data));
+static inline void check_ipsec_result(odp_packet_t ipsec_pkt)
+{
+ odp_ipsec_packet_result_t result;
- return pkt;
+ if (odp_unlikely(odp_ipsec_result(&result, ipsec_pkt)))
+ ODPH_ERR("odp_ipsec_result() failed\n");
+ else if (odp_unlikely(result.status.error.all))
+ ODPH_ERR("IPsec processing error: %" PRIu32 "\n",
+ result.status.error.all);
}
/**
@@ -619,10 +658,13 @@ run_measure_one(ipsec_args_t *cargs,
time_record_t *start,
time_record_t *end)
{
+ int in_flight, pkts_allowed, num_out, num_pkts, rc = 0;
+ const int max_in_flight = cargs->in_flight;
+ const int burst_size = cargs->burst_size;
+ const int packet_count = cargs->packet_count;
+ const int debug = cargs->debug_packets;
odp_ipsec_out_param_t param;
odp_pool_t pkt_pool;
- odp_packet_t pkt = ODP_PACKET_INVALID;
- int rc = 0;
pkt_pool = odp_pool_lookup("packet_pool");
if (pkt_pool == ODP_POOL_INVALID) {
@@ -630,6 +672,9 @@ run_measure_one(ipsec_args_t *cargs,
return -1;
}
+ if (payload_length < sizeof(test_data))
+ return -1;
+
int packets_sent = 0;
int packets_received = 0;
@@ -641,43 +686,53 @@ run_measure_one(ipsec_args_t *cargs,
fill_time_record(start);
- while ((packets_sent < cargs->iteration_count) ||
- (packets_received < cargs->iteration_count)) {
- if ((packets_sent < cargs->iteration_count) &&
- (packets_sent - packets_received <
- cargs->in_flight)) {
- odp_packet_t out_pkt;
- int num_out = 1;
+ while ((packets_sent < packet_count) ||
+ (packets_received < packet_count)) {
+ num_pkts = packet_count - packets_sent;
+
+ /* Enqueue up to burst size */
+ num_pkts = num_pkts > burst_size ? burst_size : num_pkts;
+
+ /* Enqueue up to (max in flight - current in flight) */
+ in_flight = packets_sent - packets_received;
+ pkts_allowed = max_in_flight - in_flight;
- pkt = make_packet(pkt_pool, payload_length);
- if (ODP_PACKET_INVALID == pkt)
+ /* Enqueue either a burst of packets or skip */
+ num_pkts = num_pkts > pkts_allowed ? 0 : num_pkts;
+
+ if (odp_likely(num_pkts)) {
+ odp_packet_t out_pkt[num_pkts];
+ odp_packet_t pkt[num_pkts];
+ int i;
+
+ if (odp_unlikely(make_packet_multi(pkt_pool,
+ payload_length,
+ pkt,
+ num_pkts)))
return -1;
- if (cargs->debug_packets)
- odp_packet_print_data(pkt, 0,
- odp_packet_len(pkt));
+ debug_packets(debug, pkt, num_pkts);
+ num_out = num_pkts;
- rc = odp_ipsec_out(&pkt, 1,
- &out_pkt, &num_out,
+ rc = odp_ipsec_out(pkt, num_pkts,
+ out_pkt, &num_out,
&param);
- if (rc <= 0) {
- ODPH_ERR("failed odp_ipsec_out: rc = %d\n", rc);
- odp_packet_free(pkt);
+ if (odp_unlikely(rc <= 0)) {
+ ODPH_ERR("Failed odp_ipsec_out: rc = %d\n", rc);
+ odp_packet_free_sp(pkt, num_pkts);
break;
}
- if (odp_packet_has_error(out_pkt)) {
- odp_ipsec_packet_result_t result;
- odp_ipsec_result(&result, out_pkt);
- ODPH_ERR("Received error packet: %d\n",
- result.status.error.all);
- }
+ for (i = 0; i < num_out; i++)
+ check_ipsec_result(out_pkt[i]);
+
packets_sent += rc;
packets_received += num_out;
- if (cargs->debug_packets)
- odp_packet_print_data(out_pkt, 0,
- odp_packet_len(out_pkt));
- odp_packet_free(out_pkt);
+ debug_packets(debug, out_pkt, num_out);
+
+ if (odp_unlikely(rc != num_pkts))
+ odp_packet_free_sp(&pkt[rc], num_pkts - rc);
+ odp_packet_free_sp(out_pkt, num_out);
}
}
@@ -686,6 +741,46 @@ run_measure_one(ipsec_args_t *cargs,
return rc < 0 ? rc : 0;
}
+static uint32_t dequeue_burst(odp_queue_t polled_queue,
+ odp_event_t *events,
+ int max_burst)
+{
+ int num = 0;
+
+ if (polled_queue != ODP_QUEUE_INVALID) {
+ int rc = odp_queue_deq_multi(polled_queue,
+ events,
+ max_burst);
+ num = odp_likely(rc >= 0) ? rc : 0;
+ } else {
+ num = odp_schedule_multi(NULL,
+ ODP_SCHED_NO_WAIT,
+ events,
+ max_burst);
+ }
+ return num;
+}
+
+static inline uint32_t vec_pkt_handle(int debug, odp_event_t ev)
+{
+ odp_packet_vector_t vec = odp_packet_vector_from_event(ev);
+ uint32_t vec_size = odp_packet_vector_size(vec);
+ odp_packet_t *pkt_tbl;
+ uint32_t j;
+
+ odp_packet_vector_tbl(vec, &pkt_tbl);
+
+ for (j = 0; j < vec_size; j++)
+ check_ipsec_result(pkt_tbl[j]);
+
+ debug_packets(debug, pkt_tbl, vec_size);
+
+ odp_packet_free_sp(pkt_tbl, vec_size);
+ odp_packet_vector_free(vec);
+
+ return vec_size;
+}
+
static int
run_measure_one_async(ipsec_args_t *cargs,
odp_ipsec_sa_t sa,
@@ -693,11 +788,14 @@ run_measure_one_async(ipsec_args_t *cargs,
time_record_t *start,
time_record_t *end)
{
+ int in_flight, packets_allowed, num_pkts, rc = 0;
+ const int max_in_flight = cargs->in_flight;
+ const int burst_size = cargs->burst_size;
+ const int packet_count = cargs->packet_count;
+ const int debug = cargs->debug_packets;
odp_ipsec_out_param_t param;
odp_pool_t pkt_pool;
- odp_queue_t out_queue;
- odp_packet_t pkt = ODP_PACKET_INVALID;
- int rc = 0;
+ odp_queue_t polled_queue = ODP_QUEUE_INVALID;
pkt_pool = odp_pool_lookup("packet_pool");
if (pkt_pool == ODP_POOL_INVALID) {
@@ -705,12 +803,17 @@ run_measure_one_async(ipsec_args_t *cargs,
return -1;
}
- out_queue = odp_queue_lookup("ipsec-out");
- if (out_queue == ODP_QUEUE_INVALID) {
- ODPH_ERR("ipsec-out queue not found\n");
- return -1;
+ if (cargs->poll) {
+ polled_queue = odp_queue_lookup("ipsec-out");
+ if (polled_queue == ODP_QUEUE_INVALID) {
+ ODPH_ERR("ipsec-out queue not found\n");
+ return -1;
+ }
}
+ if (payload_length < sizeof(test_data))
+ return -1;
+
int packets_sent = 0;
int packets_received = 0;
@@ -722,55 +825,78 @@ run_measure_one_async(ipsec_args_t *cargs,
fill_time_record(start);
- while ((packets_sent < cargs->iteration_count) ||
- (packets_received < cargs->iteration_count)) {
- odp_event_t ev;
+ while ((packets_sent < packet_count) ||
+ (packets_received < packet_count)) {
+
+ num_pkts = packet_count - packets_sent;
+
+ /* Enqueue up to burst size */
+ num_pkts = num_pkts > burst_size ? burst_size : num_pkts;
+
+ /* Enqueue up to (max in flight - current in flight) */
+ in_flight = packets_sent - packets_received;
+ packets_allowed = max_in_flight - in_flight;
- if ((packets_sent < cargs->iteration_count) &&
- (packets_sent - packets_received <
- cargs->in_flight)) {
- pkt = make_packet(pkt_pool, payload_length);
- if (ODP_PACKET_INVALID == pkt)
+ if (num_pkts > 0 && num_pkts <= packets_allowed) {
+ odp_packet_t pkt[num_pkts];
+
+ if (odp_unlikely(make_packet_multi(pkt_pool,
+ payload_length,
+ pkt,
+ num_pkts)))
return -1;
- if (cargs->debug_packets)
- odp_packet_print_data(pkt, 0,
- odp_packet_len(pkt));
+ debug_packets(debug, pkt, num_pkts);
- rc = odp_ipsec_out_enq(&pkt, 1,
- &param);
- if (rc <= 0) {
- ODPH_ERR("failed odp_crypto_packet_op_enq: rc = %d\n",
+ rc = odp_ipsec_out_enq(pkt, num_pkts, &param);
+ if (odp_unlikely(rc <= 0)) {
+ ODPH_ERR("Failed odp_ipsec_out_enq: rc = %d\n",
rc);
- odp_packet_free(pkt);
+ odp_packet_free_sp(pkt, num_pkts);
break;
}
+
+ if (odp_unlikely(rc != num_pkts))
+ odp_packet_free_sp(&pkt[rc], num_pkts - rc);
+
packets_sent += rc;
- }
+ } else {
+ odp_packet_t pkt_out[max_in_flight];
+ uint32_t i = 0;
+
+ /*
+ * Dequeue packets until we can enqueue the next burst
+ * or until we have received all remaining packets
+ * when there are no more packets to be sent.
+ */
+ while (num_pkts > packets_allowed ||
+ (num_pkts == 0 && packets_received < packet_count)) {
+ odp_event_t events[MAX_DEQUEUE_BURST];
+ uint32_t num;
+
+ num = dequeue_burst(polled_queue, events, MAX_DEQUEUE_BURST);
+
+ for (uint32_t n = 0; n < num; n++) {
+ if (odp_event_type(events[n]) == ODP_EVENT_PACKET_VECTOR) {
+ uint32_t vec_size;
+
+ vec_size = vec_pkt_handle(debug, events[n]);
+ packets_received += vec_size - 1;
+ packets_allowed += vec_size - 1;
+ } else {
+ pkt_out[i] = odp_ipsec_packet_from_event(events[n]);
+ check_ipsec_result(pkt_out[i]);
+ i++;
+ }
+
+ }
+ packets_received += num;
+ packets_allowed += num;
+ }
+ debug_packets(debug, pkt_out, i);
- if (cargs->schedule)
- ev = odp_schedule(NULL,
- ODP_SCHED_NO_WAIT);
- else
- ev = odp_queue_deq(out_queue);
-
- while (ev != ODP_EVENT_INVALID) {
- odp_packet_t out_pkt;
- odp_ipsec_packet_result_t result;
-
- out_pkt = odp_ipsec_packet_from_event(ev);
- odp_ipsec_result(&result, out_pkt);
-
- if (cargs->debug_packets)
- odp_packet_print_data(out_pkt, 0,
- odp_packet_len(out_pkt));
- odp_packet_free(out_pkt);
- packets_received++;
- if (cargs->schedule)
- ev = odp_schedule(NULL,
- ODP_SCHED_NO_WAIT);
- else
- ev = odp_queue_deq(out_queue);
+ if (i)
+ odp_packet_free_sp(pkt_out, i);
}
}
@@ -843,13 +969,13 @@ run_measure_one_config(ipsec_args_t *cargs,
break;
count = get_elapsed_usec(&start, &end);
- result.elapsed = count / cargs->iteration_count;
+ result.elapsed = count / cargs->packet_count;
count = get_rusage_self_diff(&start, &end);
- result.rusage_self = count / cargs->iteration_count;
+ result.rusage_self = count / cargs->packet_count;
count = get_rusage_thread_diff(&start, &end);
- result.rusage_thread = count / cargs->iteration_count;
+ result.rusage_thread = count / cargs->packet_count;
print_result(cargs, payloads[i],
config, &result);
@@ -913,7 +1039,9 @@ static void usage(char *progname)
print_config_names(" ");
printf(" -d, --debug Enable dump of processed packets.\n"
" -f, --flight <number> Max number of packet processed in parallel (default 1)\n"
- " -i, --iterations <number> Number of iterations.\n"
+ " -c, --count <number> Number of packets (default 10000)\n"
+ " -b, --burst <number> Number of packets in one IPsec API submission (default 1)\n"
+ " -v, --vector <number> Enable vector packet completion from IPsec APIs with specified vector size.\n"
" -l, --payload Payload length.\n"
" -s, --schedule Use scheduler for completion events.\n"
" -p, --poll Poll completion queue for completion events.\n"
@@ -932,7 +1060,9 @@ static void parse_args(int argc, char *argv[], ipsec_args_t *cargs)
{"debug", no_argument, NULL, 'd'},
{"flight", optional_argument, NULL, 'f'},
{"help", no_argument, NULL, 'h'},
- {"iterations", optional_argument, NULL, 'i'},
+ {"count", optional_argument, NULL, 'c'},
+ {"burst", optional_argument, NULL, 'b'},
+ {"vector", optional_argument, NULL, 'v'},
{"payload", optional_argument, NULL, 'l'},
{"sessions", optional_argument, NULL, 'm'},
{"poll", no_argument, NULL, 'p'},
@@ -942,11 +1072,13 @@ static void parse_args(int argc, char *argv[], ipsec_args_t *cargs)
{NULL, 0, NULL, 0}
};
- static const char *shortopts = "+a:c:df:hi:m:nl:sptu";
+ static const char *shortopts = "+a:b:c:df:hm:nl:sptuv:";
cargs->in_flight = 1;
cargs->debug_packets = 0;
- cargs->iteration_count = 10000;
+ cargs->packet_count = 10000;
+ cargs->burst_size = 1;
+ cargs->vec_pkt_size = 0;
cargs->payload_length = 0;
cargs->alg_config = NULL;
cargs->schedule = 0;
@@ -971,8 +1103,24 @@ static void parse_args(int argc, char *argv[], ipsec_args_t *cargs)
case 'd':
cargs->debug_packets = 1;
break;
- case 'i':
- cargs->iteration_count = atoi(optarg);
+ case 'c':
+ cargs->packet_count = atoi(optarg);
+ break;
+ case 'b':
+ if (optarg == NULL)
+ cargs->burst_size = 32;
+ else
+ cargs->burst_size = atoi(optarg);
+ if (cargs->burst_size > POOL_NUM_PKT) {
+ printf("Invalid burst size (max allowed: %d)\n", POOL_NUM_PKT);
+ exit(-1);
+ }
+ break;
+ case 'v':
+ if (optarg == NULL)
+ cargs->vec_pkt_size = 32;
+ else
+ cargs->vec_pkt_size = atoi(optarg);
break;
case 'f':
cargs->in_flight = atoi(optarg);
@@ -1001,6 +1149,11 @@ static void parse_args(int argc, char *argv[], ipsec_args_t *cargs)
}
}
+ if (cargs->in_flight < cargs->burst_size) {
+ printf("-f (flight) must be greater than or equal to -b (burst)\n");
+ exit(-1);
+ }
+
optind = 1; /* reset 'extern optind' from the getopt lib */
if (cargs->schedule && cargs->poll) {
@@ -1012,6 +1165,7 @@ static void parse_args(int argc, char *argv[], ipsec_args_t *cargs)
int main(int argc, char *argv[])
{
+ odp_pool_t vec_pool = ODP_POOL_INVALID;
ipsec_args_t cargs;
odp_pool_t pool;
odp_queue_param_t qparam;
@@ -1105,11 +1259,54 @@ int main(int argc, char *argv[])
exit(EXIT_FAILURE);
}
+ if (cargs.vec_pkt_size) {
+ if (capa.vector.max_pools < 1) {
+ ODPH_ERR("Vector packet pool not available");
+ exit(EXIT_FAILURE);
+ }
+
+ if (!ipsec_capa.vector.supported) {
+ ODPH_ERR("Vector packet completion not supported by IPsec.\n");
+ exit(EXIT_FAILURE);
+ }
+
+ if (capa.vector.max_size < cargs.vec_pkt_size) {
+ ODPH_ERR("Vector size larger than max size supported by vector pool.\n");
+ exit(EXIT_FAILURE);
+ }
+
+ if (!cargs.schedule && !cargs.poll) {
+ ODPH_ERR("Vector packet is not supported with sync APIs.\n");
+ exit(EXIT_FAILURE);
+ }
+
+ /* Create vector pool */
+ odp_pool_param_init(&param);
+ param.vector.num = POOL_NUM_PKT;
+ param.vector.max_size = cargs.vec_pkt_size;
+ param.type = ODP_POOL_VECTOR;
+ vec_pool = odp_pool_create("vector_pool", &param);
+
+ if (vec_pool == ODP_POOL_INVALID) {
+ ODPH_ERR("Vector packet pool create failed.\n");
+ exit(EXIT_FAILURE);
+ }
+
+ odp_pool_print(vec_pool);
+ }
+
odp_ipsec_config_init(&config);
config.max_num_sa = 2;
config.inbound.chksums.all_chksum = 0;
config.outbound.all_chksum = 0;
+ if (vec_pool != ODP_POOL_INVALID) {
+ config.vector.enable = true;
+ config.vector.pool = vec_pool;
+ config.vector.max_size = cargs.vec_pkt_size;
+ config.vector.max_tmo_ns = ipsec_capa.vector.max_tmo_ns;
+ }
+
odp_queue_param_init(&qparam);
if (cargs.schedule) {
odp_schedule_config(NULL);
@@ -1196,6 +1393,14 @@ int main(int argc, char *argv[])
if (cargs.schedule || cargs.poll)
odp_queue_destroy(out_queue);
+
+ if (cargs.vec_pkt_size) {
+ if (odp_pool_destroy(vec_pool)) {
+ ODPH_ERR("Error: vector pool destroy\n");
+ exit(EXIT_FAILURE);
+ }
+ }
+
if (odp_pool_destroy(pool)) {
ODPH_ERR("Error: pool destroy\n");
exit(EXIT_FAILURE);
diff --git a/test/performance/odp_ipsec_run.sh b/test/performance/odp_ipsec_run.sh
index 1974a7c55..2ddb48d07 100755
--- a/test/performance/odp_ipsec_run.sh
+++ b/test/performance/odp_ipsec_run.sh
@@ -7,9 +7,9 @@
TEST_DIR="${TEST_DIR:-$(dirname $0)}"
-# Run with a small number of iterations in make check
+# Run with a small number of packets in make check
-$TEST_DIR/odp_ipsec${EXEEXT} -i 100
+$TEST_DIR/odp_ipsec${EXEEXT} -c 100
if [ $? -ne 0 ] ; then
echo Test FAILED
diff --git a/test/performance/odp_pktio_ordered.c b/test/performance/odp_pktio_ordered.c
index 37a0899b6..130eb4688 100644
--- a/test/performance/odp_pktio_ordered.c
+++ b/test/performance/odp_pktio_ordered.c
@@ -1258,7 +1258,7 @@ int main(int argc, char *argv[])
exit(EXIT_FAILURE);
}
- if ((unsigned)gbl_args->appl.num_flows > capa.max_output_queues)
+ if ((uint32_t)gbl_args->appl.num_flows > capa.max_output_queues)
gbl_args->appl.num_flows = capa.max_output_queues;
}
diff --git a/test/performance/odp_random.c b/test/performance/odp_random.c
index 622a58c65..46134ac0c 100644
--- a/test/performance/odp_random.c
+++ b/test/performance/odp_random.c
@@ -14,20 +14,52 @@
#include <odp_api.h>
#include <odp/helper/odph_api.h>
+#define PSEUDO_RANDOM (-1)
+
#define MB (1024ull * 1024ull)
+typedef struct test_global_t test_global_t;
+
+typedef struct thread_arg_t {
+ test_global_t *global;
+ int thread_idx;
+ uint8_t *data;
+
+} thread_arg_t;
+
+struct test_global_t {
+ odp_barrier_t barrier;
+ odp_random_kind_t type;
+ uint8_t *data;
+ uint32_t rounds;
+
+ thread_arg_t thread_arg[ODP_THREAD_COUNT_MAX];
+
+ struct {
+ uint64_t nsec[ODP_THREAD_COUNT_MAX];
+ uint64_t sum[ODP_THREAD_COUNT_MAX];
+ uint64_t min[ODP_THREAD_COUNT_MAX];
+ uint64_t max[ODP_THREAD_COUNT_MAX];
+ } stat;
+};
+
/* Command line options */
typedef struct {
+ int mode;
int num_threads;
uint32_t size;
uint32_t rounds;
+ uint64_t delay;
+
} options_t;
static options_t options;
static const options_t options_def = {
+ .mode = 0,
.num_threads = 1,
.size = 256,
.rounds = 100000,
+ .delay = 0,
};
static void print_usage(void)
@@ -37,13 +69,18 @@ static void print_usage(void)
"\n"
"Usage: odp_random [options]\n"
"\n"
+ " -m, --mode Test mode select (default: 0):\n"
+ " 0: Data throughput\n"
+ " 1: Data generation latency (size: 8B by default)\n"
" -t, --threads Number of worker threads (default %u)\n"
" -s, --size Size of buffer in bytes (default %u)\n"
" -r, --rounds Number of test rounds (default %u)\n"
" Divided by 100 for ODP_RANDOM_TRUE\n"
+ " -d, --delay Delay (nsec) between buffer fills (default %" PRIu64 ").\n"
+ " Affects only latency mode.\n"
" -h, --help This help\n"
"\n",
- options_def.num_threads, options_def.size, options_def.rounds);
+ options_def.num_threads, options_def.size, options_def.rounds, options_def.delay);
}
static int parse_options(int argc, char *argv[])
@@ -53,16 +90,19 @@ static int parse_options(int argc, char *argv[])
int ret = 0;
static const struct option longopts[] = {
+ { "mode", required_argument, NULL, 'm' },
{ "threads", required_argument, NULL, 't' },
{ "size", required_argument, NULL, 's' },
{ "rounds", required_argument, NULL, 'r' },
+ { "delay", required_argument, NULL, 'd' },
{ "help", no_argument, NULL, 'h' },
{ NULL, 0, NULL, 0 }
};
- static const char *shortopts = "+t:s:r:h";
+ static const char *shortopts = "+m:t:s:r:d:h";
options = options_def;
+ options.size = 0;
while (1) {
opt = getopt_long(argc, argv, shortopts, longopts, &long_index);
@@ -71,6 +111,9 @@ static int parse_options(int argc, char *argv[])
break;
switch (opt) {
+ case 'm':
+ options.mode = atoi(optarg);
+ break;
case 't':
options.num_threads = atol(optarg);
break;
@@ -80,6 +123,9 @@ static int parse_options(int argc, char *argv[])
case 'r':
options.rounds = atol(optarg);
break;
+ case 'd':
+ options.delay = atol(optarg);
+ break;
case 'h':
/* fall through */
default:
@@ -89,170 +135,216 @@ static int parse_options(int argc, char *argv[])
}
}
- if (options.size < 1) {
- ODPH_ERR("Invalid size: %" PRIu32 "\n", options.size);
+ if (options.num_threads < 1 || options.num_threads > ODP_THREAD_COUNT_MAX) {
+ ODPH_ERR("Bad number of threads: %i\n", options.num_threads);
return -1;
}
+ if (options.size == 0) {
+ options.size = options_def.size;
+
+ if (options.mode)
+ options.size = 8;
+ }
+
+ printf("\nOptions:\n");
+ printf("------------------------\n");
+ printf(" mode: %i\n", options.mode);
+ printf(" threads: %i\n", options.num_threads);
+ printf(" size: %u\n", options.size);
+ printf(" rounds: %u\n", options.rounds);
+ printf(" delay: %" PRIu64 "\n", options.delay);
+ printf("\n");
+
return ret;
}
-const char *shm_name = "odp_random_test";
+static inline void random_data_loop(odp_random_kind_t type, uint32_t rounds,
+ uint8_t *data, uint32_t size)
+{
+ uint32_t i;
+ int32_t ret;
-typedef struct test_shm_t {
- odp_barrier_t barrier;
- odp_random_kind_t type;
- uint64_t nsec[ODP_THREAD_COUNT_MAX];
-} test_shm_t;
+ if ((int)type == PSEUDO_RANDOM) {
+ uint64_t seed = 0;
-static test_shm_t *shm_lookup(void)
-{
- test_shm_t *shm = NULL;
- odp_shm_t shm_hdl = odp_shm_lookup(shm_name);
+ for (i = 0; i < rounds; i++) {
+ uint32_t pos = 0;
- if (shm_hdl != ODP_SHM_INVALID)
- shm = (test_shm_t *)odp_shm_addr(shm_hdl);
+ while (pos < size) {
+ ret = odp_random_test_data(data + pos, size - pos, &seed);
- return shm;
-}
+ if (ret < 0) {
+ ODPH_ERR("odp_random_test_data() failed\n");
+ exit(EXIT_FAILURE);
+ }
-static uint32_t type_rounds(odp_random_kind_t type)
-{
- switch (type) {
- case ODP_RANDOM_TRUE:
- return options.rounds / 100;
- default:
- return options.rounds;
+ pos += ret;
+ }
+ }
+ } else {
+ for (i = 0; i < rounds; i++) {
+ uint32_t pos = 0;
+
+ while (pos < size) {
+ ret = odp_random_data(data + pos, size - pos, type);
+
+ if (ret < 0) {
+ ODPH_ERR("odp_random_data() failed\n");
+ exit(EXIT_FAILURE);
+ }
+
+ pos += ret;
+ }
+ }
}
}
-static int test_random(void *p)
+static int test_random_perf(void *ptr)
{
- (void)p;
-
- uint8_t *buf, *data;
- const unsigned long page = ODP_PAGE_SIZE;
odp_time_t start;
uint64_t nsec;
- uint32_t rounds;
- test_shm_t *shm = shm_lookup();
+ thread_arg_t *thread_arg = ptr;
+ test_global_t *global = thread_arg->global;
+ odp_random_kind_t type = global->type;
+ int thread_idx = thread_arg->thread_idx;
+ uint8_t *data = thread_arg->data;
+ uint32_t size = options.size;
+ uint32_t rounds = global->rounds;
- if (!shm) {
- ODPH_ERR("Failed to look up shm %s\n", shm_name);
- exit(EXIT_FAILURE);
- }
+ /* One warm up round */
+ random_data_loop(type, 1, data, size);
- rounds = type_rounds(shm->type);
+ odp_barrier_wait(&global->barrier);
- /* One extra page for alignment. */
- buf = (uint8_t *)malloc(options.size + page);
-
- if (!buf) {
- ODPH_ERR("Memory allocation failed.\n");
- exit(EXIT_FAILURE);
- }
-
- /* Align to start of page. */
- data = (uint8_t *)(((uintptr_t)buf + (page - 1)) & ~(page - 1));
-
- odp_barrier_wait(&shm->barrier);
+ /* Test run */
start = odp_time_local();
- for (uint32_t i = 0; i < rounds; i++) {
- uint32_t pos = 0;
-
- while (pos < options.size) {
- int32_t n = odp_random_data(data + pos,
- options.size - pos,
- shm->type);
-
- if (n < 0) {
- ODPH_ERR("odp_random_data() failed\n");
- exit(EXIT_FAILURE);
- }
-
- pos += n;
- }
- }
+ random_data_loop(type, rounds, data, size);
nsec = odp_time_diff_ns(odp_time_local(), start);
- shm->nsec[odp_thread_id()] = nsec;
- free(buf);
+
+ global->stat.nsec[thread_idx] = nsec;
return 0;
}
-static int test_random_test(void *p)
+static inline void random_data_latency(test_global_t *global, int thread_idx,
+ uint32_t rounds, uint8_t *data, uint32_t size)
{
- (void)p;
-
- uint8_t *buf, *data;
- const unsigned long page = ODP_PAGE_SIZE;
- odp_time_t start;
+ uint32_t i;
+ int32_t ret;
+ odp_time_t t1, t2, start;
uint64_t nsec;
+ odp_random_kind_t type = global->type;
+ uint64_t delay = options.delay;
+ uint64_t min = UINT64_MAX;
+ uint64_t max = 0;
+ uint64_t sum = 0;
uint64_t seed = 0;
- uint32_t rounds;
- test_shm_t *shm = shm_lookup();
-
- if (!shm) {
- ODPH_ERR("Failed to look up shm %s\n", shm_name);
- exit(EXIT_FAILURE);
- }
-
- rounds = type_rounds(shm->type);
-
- /* One extra page for alignment. */
- buf = (uint8_t *)malloc(options.size + page);
-
- if (!buf) {
- ODPH_ERR("Memory allocation failed.\n");
- exit(EXIT_FAILURE);
- }
-
- /* Align to start of page. */
- data = (uint8_t *)(((uintptr_t)buf + (page - 1)) & ~(page - 1));
- odp_barrier_wait(&shm->barrier);
start = odp_time_local();
- for (uint32_t i = 0; i < rounds; i++) {
+ for (i = 0; i < rounds; i++) {
uint32_t pos = 0;
- while (pos < options.size) {
- int32_t n = odp_random_test_data(data + pos,
- options.size - pos,
- &seed);
+ if (delay)
+ odp_time_wait_ns(delay);
- if (n < 0) {
- ODPH_ERR("odp_random_data() failed\n");
- exit(EXIT_FAILURE);
- }
+ if ((int)type == PSEUDO_RANDOM) {
+ t1 = odp_time_local_strict();
+ while (pos < size) {
+ ret = odp_random_test_data(data + pos, size - pos, &seed);
+
+ if (ret < 0) {
+ ODPH_ERR("odp_random_test_data() failed\n");
+ exit(EXIT_FAILURE);
+ }
- pos += n;
+ pos += ret;
+ }
+ t2 = odp_time_local_strict();
+ } else {
+ t1 = odp_time_local_strict();
+ while (pos < size) {
+ ret = odp_random_data(data + pos, size - pos, type);
+
+ if (ret < 0) {
+ ODPH_ERR("odp_random_data() failed\n");
+ exit(EXIT_FAILURE);
+ }
+
+ pos += ret;
+ }
+ t2 = odp_time_local_strict();
}
+
+ nsec = odp_time_diff_ns(t2, t1);
+ sum += nsec;
+
+ if (nsec > max)
+ max = nsec;
+ if (nsec < min)
+ min = nsec;
}
nsec = odp_time_diff_ns(odp_time_local(), start);
- shm->nsec[odp_thread_id()] = nsec;
- free(buf);
+
+ global->stat.nsec[thread_idx] = nsec;
+ global->stat.sum[thread_idx] = sum;
+ global->stat.min[thread_idx] = min;
+ global->stat.max[thread_idx] = max;
+}
+
+static int test_random_latency(void *ptr)
+{
+ thread_arg_t *thread_arg = ptr;
+ test_global_t *global = thread_arg->global;
+ odp_random_kind_t type = global->type;
+ int thread_idx = thread_arg->thread_idx;
+ uint8_t *data = thread_arg->data;
+ uint32_t size = options.size;
+ uint32_t rounds = global->rounds;
+
+ /* One warm up round */
+ random_data_loop(type, 1, data, size);
+
+ odp_barrier_wait(&global->barrier);
+
+ /* Test run */
+ random_data_latency(global, thread_idx, rounds, data, size);
return 0;
}
-static void test_type(odp_instance_t instance, test_shm_t *shm,
- odp_random_kind_t type)
+static uint32_t type_rounds(odp_random_kind_t type)
+{
+ switch (type) {
+ case ODP_RANDOM_TRUE:
+ return options.rounds / 100;
+ default:
+ return options.rounds;
+ }
+}
+
+static void test_type(odp_instance_t instance, test_global_t *global, odp_random_kind_t type)
{
- memset(shm, 0, sizeof(test_shm_t));
- shm->type = type;
- odp_barrier_init(&shm->barrier, options.num_threads);
+ int i;
+ int num_threads = options.num_threads;
+ uint32_t rounds = type_rounds(type);
+ uint32_t size = options.size;
+
+ memset(&global->stat, 0, sizeof(global->stat));
+ global->type = type;
+ global->rounds = rounds;
+ odp_barrier_init(&global->barrier, num_threads);
odp_cpumask_t cpumask;
odph_thread_common_param_t thr_common;
- odph_thread_param_t thr_param;
- odph_thread_t thr_worker[options.num_threads];
+ odph_thread_param_t thr_param[num_threads];
+ odph_thread_t thr_worker[num_threads];
- if (odp_cpumask_default_worker(&cpumask, options.num_threads) !=
- options.num_threads) {
+ if (odp_cpumask_default_worker(&cpumask, num_threads) != num_threads) {
ODPH_ERR("Failed to get default CPU mask.\n");
exit(EXIT_FAILURE);
}
@@ -260,35 +352,36 @@ static void test_type(odp_instance_t instance, test_shm_t *shm,
odph_thread_common_param_init(&thr_common);
thr_common.instance = instance;
thr_common.cpumask = &cpumask;
- thr_common.share_param = 1;
- odph_thread_param_init(&thr_param);
- thr_param.thr_type = ODP_THREAD_WORKER;
- thr_param.start = test_random;
+ for (i = 0; i < num_threads; i++) {
+ odph_thread_param_init(&thr_param[i]);
+ thr_param[i].thr_type = ODP_THREAD_WORKER;
+ thr_param[i].arg = &global->thread_arg[i];
- if (type == (odp_random_kind_t)-1)
- thr_param.start = test_random_test;
+ if (options.mode == 0)
+ thr_param[i].start = test_random_perf;
+ else
+ thr_param[i].start = test_random_latency;
+ }
memset(&thr_worker, 0, sizeof(thr_worker));
- if (odph_thread_create(thr_worker, &thr_common, &thr_param,
- options.num_threads) != options.num_threads) {
+ if (odph_thread_create(thr_worker, &thr_common, thr_param, num_threads) != num_threads) {
ODPH_ERR("Failed to create worker threads.\n");
exit(EXIT_FAILURE);
}
- if (odph_thread_join(thr_worker, options.num_threads) !=
- options.num_threads) {
+ if (odph_thread_join(thr_worker, num_threads) != num_threads) {
ODPH_ERR("Failed to join worker threads.\n");
exit(EXIT_FAILURE);
}
double mb, seconds, nsec = 0;
- for (int i = 0; i < ODP_THREAD_COUNT_MAX; i++)
- nsec += shm->nsec[i];
+ for (i = 0; i < num_threads; i++)
+ nsec += global->stat.nsec[i];
- nsec /= options.num_threads;
+ nsec /= num_threads;
switch (type) {
case ODP_RANDOM_BASIC:
@@ -304,25 +397,53 @@ static void test_type(odp_instance_t instance, test_shm_t *shm,
printf("odp_random_test_data\n");
}
- uint32_t rounds = type_rounds(type);
-
printf("--------------------\n");
- printf("threads: %d size: %u B rounds: %u ", options.num_threads,
- options.size, rounds);
- mb = (uint64_t)options.num_threads * (uint64_t)options.size *
- (uint64_t)rounds;
+ printf("threads: %d size: %u B rounds: %u ", num_threads, size, rounds);
+ mb = (uint64_t)num_threads * (uint64_t)size * (uint64_t)rounds;
mb /= MB;
seconds = (double)nsec / (double)ODP_TIME_SEC_IN_NS;
printf("MB: %.3f seconds: %.3f ", mb, seconds);
printf("MB/s: %.3f ", mb / seconds);
- printf("MB/s/thread: %.3f", mb / seconds / (double)options.num_threads);
- printf("\n\n");
+ printf("MB/s/thread: %.3f\n", mb / seconds / (double)num_threads);
+
+ if (options.mode) {
+ double ave;
+ uint64_t min = UINT64_MAX;
+ uint64_t max = 0;
+ uint64_t sum = 0;
+
+ printf(" latency (nsec)\n");
+ printf(" thread min max ave\n");
+ for (i = 0; i < num_threads; i++) {
+ ave = (double)global->stat.sum[i] / rounds;
+ sum += global->stat.sum[i];
+
+ if (global->stat.min[i] < min)
+ min = global->stat.min[i];
+
+ if (global->stat.max[i] > max)
+ max = global->stat.max[i];
+
+ printf("%8i %8" PRIu64 " %8" PRIu64 " %10.1f\n", i, global->stat.min[i],
+ global->stat.max[i], ave);
+ }
+
+ printf(" all %8" PRIu64 " %8" PRIu64 " %10.1f\n",
+ min, max, ((double)sum / rounds) / num_threads);
+ }
+
+ printf("\n");
}
int main(int argc, char **argv)
{
odp_instance_t instance;
odp_init_t init;
+ odp_shm_t shm_glb, shm_data;
+ test_global_t *global;
+ int num_threads, i;
+ uint64_t tot_size, size;
+ uint8_t *addr;
if (parse_options(argc, argv))
exit(EXIT_FAILURE);
@@ -352,31 +473,59 @@ int main(int argc, char **argv)
odp_sys_info_print();
- test_shm_t *shm = NULL;
- odp_shm_t shm_hdl = odp_shm_reserve(shm_name, sizeof(test_shm_t), 64,
- 0);
+ global = NULL;
+ shm_glb = odp_shm_reserve("test_globals", sizeof(test_global_t), ODP_CACHE_LINE_SIZE, 0);
+
+ if (shm_glb != ODP_SHM_INVALID)
+ global = (test_global_t *)odp_shm_addr(shm_glb);
+
+ if (!global) {
+ ODPH_ERR("Failed to reserve shm\n");
+ exit(EXIT_FAILURE);
+ }
+
+ memset(global, 0, sizeof(test_global_t));
+
+ num_threads = options.num_threads;
+ addr = NULL;
+ size = ODP_CACHE_LINE_SIZE + ODP_CACHE_LINE_ROUNDUP(options.size);
+ tot_size = num_threads * size;
+ shm_data = odp_shm_reserve("test_data", tot_size, ODP_CACHE_LINE_SIZE, 0);
- if (shm_hdl != ODP_SHM_INVALID)
- shm = (test_shm_t *)odp_shm_addr(shm_hdl);
+ if (shm_data != ODP_SHM_INVALID)
+ addr = odp_shm_addr(shm_data);
- if (!shm) {
- ODPH_ERR("Failed to reserve shm %s\n", shm_name);
+ if (!addr) {
+ ODPH_ERR("Failed to reserve shm: size %" PRIu64 " bytes\n", tot_size);
exit(EXIT_FAILURE);
}
+ for (i = 0; i < num_threads; i++) {
+ global->thread_arg[i].global = global;
+ global->thread_arg[i].thread_idx = i;
+ global->thread_arg[i].data = addr + i * size;
+ }
+
+ odp_shm_print_all();
+
switch (odp_random_max_kind()) {
case ODP_RANDOM_TRUE:
- test_type(instance, shm, ODP_RANDOM_TRUE);
+ test_type(instance, global, ODP_RANDOM_TRUE);
/* fall through */
case ODP_RANDOM_CRYPTO:
- test_type(instance, shm, ODP_RANDOM_CRYPTO);
+ test_type(instance, global, ODP_RANDOM_CRYPTO);
/* fall through */
default:
- test_type(instance, shm, ODP_RANDOM_BASIC);
- test_type(instance, shm, -1);
+ test_type(instance, global, ODP_RANDOM_BASIC);
+ test_type(instance, global, PSEUDO_RANDOM);
+ }
+
+ if (odp_shm_free(shm_data)) {
+ ODPH_ERR("odp_shm_free() failed\n");
+ exit(EXIT_FAILURE);
}
- if (odp_shm_free(shm_hdl)) {
+ if (odp_shm_free(shm_glb)) {
ODPH_ERR("odp_shm_free() failed\n");
exit(EXIT_FAILURE);
}
diff --git a/test/performance/odp_sched_pktio.c b/test/performance/odp_sched_pktio.c
index 77c1d260f..11be9d367 100644
--- a/test/performance/odp_sched_pktio.c
+++ b/test/performance/odp_sched_pktio.c
@@ -862,10 +862,9 @@ static int open_pktios(test_global_t *test_global)
odp_pktin_queue_param_t pktin_param;
odp_pktout_queue_param_t pktout_param;
odp_schedule_sync_t sched_sync;
- unsigned int num_queue;
+ uint32_t num_queue, j;
char *name;
int i, num_pktio, ret;
- unsigned int j;
num_pktio = test_global->opt.num_pktio;
num_queue = test_global->opt.num_pktio_queue;
diff --git a/test/validation/api/classification/classification.h b/test/validation/api/classification/classification.h
index 4c8613555..70dcc6230 100644
--- a/test/validation/api/classification/classification.h
+++ b/test/validation/api/classification/classification.h
@@ -23,6 +23,11 @@
#define CLS_DEFAULT_SMAC {0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c}
#define CLS_MAGIC_VAL 0xdeadbeef
+/* Config values for Drop CoS */
+#define TEST_DROP 1
+#define CLS_DROP 6
+#define CLS_DROP_PORT 4001
+
/* Config values for Error CoS */
#define TEST_ERROR 1
#define CLS_ERROR 1
@@ -47,7 +52,7 @@
/* Config values for CoS L2 Priority */
#define TEST_L2_QOS 1
-#define CLS_L2_QOS_0 6
+#define CLS_L2_QOS_0 7
#define CLS_L2_QOS_MAX 5
#define CLS_ENTRIES (CLS_L2_QOS_0 + CLS_L2_QOS_MAX)
diff --git a/test/validation/api/classification/odp_classification_basic.c b/test/validation/api/classification/odp_classification_basic.c
index 3aeb89462..fb5ec4ed0 100644
--- a/test/validation/api/classification/odp_classification_basic.c
+++ b/test/validation/api/classification/odp_classification_basic.c
@@ -52,6 +52,49 @@ static void classification_test_create_cos(void)
odp_queue_destroy(queue);
}
+static void classification_test_create_cos_max_common(odp_bool_t stats)
+{
+ uint32_t i;
+ odp_cls_cos_param_t cls_param;
+ odp_cls_capability_t capa;
+
+ CU_ASSERT_FATAL(odp_cls_capability(&capa) == 0);
+
+ uint32_t num = capa.max_cos;
+
+ if (stats && capa.max_cos_stats < num)
+ num = capa.max_cos_stats;
+
+ odp_cos_t cos[num];
+
+ for (i = 0; i < num; i++) {
+ odp_cls_cos_param_init(&cls_param);
+ cls_param.action = ODP_COS_ACTION_DROP;
+ cls_param.stats_enable = stats;
+
+ cos[i] = odp_cls_cos_create(NULL, &cls_param);
+ if (cos[i] == ODP_COS_INVALID) {
+ ODPH_ERR("odp_cls_cos_create() failed at CoS %u out of %u.\n", i + 1, num);
+ break;
+ }
+ }
+
+ CU_ASSERT(i == num);
+
+ for (uint32_t j = 0; j < i; j++)
+ CU_ASSERT(!odp_cos_destroy(cos[j]));
+}
+
+static void classification_test_create_cos_max(void)
+{
+ classification_test_create_cos_max_common(false);
+}
+
+static void classification_test_create_cos_max_stats(void)
+{
+ classification_test_create_cos_max_common(true);
+}
+
static void classification_test_destroy_cos(void)
{
odp_cos_t cos;
@@ -345,6 +388,8 @@ static void classification_test_pmr_composite_create(void)
odp_testinfo_t classification_suite_basic[] = {
ODP_TEST_INFO(classification_test_default_values),
ODP_TEST_INFO(classification_test_create_cos),
+ ODP_TEST_INFO(classification_test_create_cos_max),
+ ODP_TEST_INFO(classification_test_create_cos_max_stats),
ODP_TEST_INFO(classification_test_destroy_cos),
ODP_TEST_INFO(classification_test_create_pmr_match),
ODP_TEST_INFO(classification_test_cos_set_queue),
diff --git a/test/validation/api/classification/odp_classification_tests.c b/test/validation/api/classification/odp_classification_tests.c
index 41b4ab02f..4511fc1d7 100644
--- a/test/validation/api/classification/odp_classification_tests.c
+++ b/test/validation/api/classification/odp_classification_tests.c
@@ -23,6 +23,7 @@ static int global_num_l2_qos;
#define NUM_COS_PMR_CHAIN 2
#define NUM_COS_DEFAULT 1
+#define NUM_COS_DROP 1
#define NUM_COS_ERROR 1
#define NUM_COS_L2_PRIO CLS_L2_QOS_MAX
#define NUM_COS_PMR 1
@@ -453,6 +454,74 @@ void test_pktio_default_cos(odp_bool_t enable_pktv)
odp_packet_free(pkt);
}
+void configure_pktio_drop_cos(odp_bool_t enable_pktv, uint32_t max_cos_stats)
+{
+ uint16_t val;
+ uint16_t mask;
+ odp_pmr_param_t pmr_param;
+ odp_cls_cos_param_t cls_param;
+ char cosname[ODP_COS_NAME_LEN];
+
+ sprintf(cosname, "DropCoS");
+ odp_cls_cos_param_init(&cls_param);
+
+ cls_param.action = ODP_COS_ACTION_DROP;
+ cls_param.stats_enable = max_cos_stats > 0;
+
+ if (enable_pktv) {
+ cls_param.vector.enable = true;
+ cls_param.vector.pool = pktv_config.pool;
+ cls_param.vector.max_size = pktv_config.max_size;
+ cls_param.vector.max_tmo_ns = pktv_config.max_tmo_ns;
+ }
+
+ cos_list[CLS_DROP] = odp_cls_cos_create(cosname, &cls_param);
+ CU_ASSERT_FATAL(cos_list[CLS_DROP] != ODP_COS_INVALID);
+
+ val = odp_cpu_to_be_16(CLS_DROP_PORT);
+ mask = odp_cpu_to_be_16(0xffff);
+ odp_cls_pmr_param_init(&pmr_param);
+ pmr_param.term = find_first_supported_l3_pmr();
+ pmr_param.match.value = &val;
+ pmr_param.match.mask = &mask;
+ pmr_param.val_sz = sizeof(val);
+
+ pmr_list[CLS_DROP] = odp_cls_pmr_create(&pmr_param, 1,
+ cos_list[CLS_DEFAULT],
+ cos_list[CLS_DROP]);
+ CU_ASSERT_FATAL(pmr_list[CLS_DROP] != ODP_PMR_INVALID);
+}
+
+void test_pktio_drop_cos(odp_bool_t enable_pktv)
+{
+ odp_packet_t pkt;
+ odp_queue_t queue;
+ uint32_t seqno = 0;
+ cls_packet_info_t pkt_info;
+ odp_cls_capability_t capa;
+ odp_cls_cos_stats_t start, stop;
+
+ CU_ASSERT_FATAL(odp_cls_capability(&capa) == 0);
+ pkt_info = default_pkt_info;
+ pkt_info.l4_type = CLS_PKT_L4_UDP;
+ pkt = create_packet(pkt_info);
+ CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID);
+ seqno = cls_pkt_get_seq(pkt);
+ CU_ASSERT(seqno != TEST_SEQ_INVALID);
+ set_first_supported_pmr_port(pkt, CLS_DROP_PORT);
+ CU_ASSERT(odp_cls_cos_stats(cos_list[CLS_DROP], &start) == 0);
+ enqueue_pktio_interface(pkt, pktio_loop);
+ pkt = receive_packet(&queue, ODP_TIME_SEC_IN_NS, enable_pktv);
+ CU_ASSERT(odp_cls_cos_stats(cos_list[CLS_DROP], &stop) == 0);
+ CU_ASSERT_FATAL(pkt == ODP_PACKET_INVALID);
+ if (capa.stats.cos.counter.packets)
+ CU_ASSERT((stop.packets - start.packets) == 1);
+ if (capa.stats.cos.counter.discards)
+ CU_ASSERT((stop.discards - start.discards) == 0);
+ if (capa.stats.cos.counter.errors)
+ CU_ASSERT((stop.errors - start.errors) == 0);
+}
+
static int classification_check_queue_stats(void)
{
odp_cls_capability_t capa;
@@ -926,6 +995,11 @@ static void classification_test_pktio_configure_common(odp_bool_t enable_pktv)
tc.default_cos = 1;
num_cos -= NUM_COS_DEFAULT;
}
+ if (num_cos >= NUM_COS_DEFAULT && TEST_DROP) {
+ configure_pktio_drop_cos(enable_pktv, capa.max_cos_stats);
+ tc.drop_cos = 1;
+ num_cos -= NUM_COS_DROP;
+ }
if (num_cos >= NUM_COS_ERROR && TEST_ERROR) {
configure_pktio_error_cos(enable_pktv);
tc.error_cos = 1;
@@ -969,6 +1043,8 @@ static void classification_test_pktio_test_common(odp_bool_t enable_pktv)
/* Test Different CoS on the pktio interface */
if (tc.default_cos && TEST_DEFAULT)
test_pktio_default_cos(enable_pktv);
+ if (tc.drop_cos && TEST_DROP)
+ test_pktio_drop_cos(enable_pktv);
if (tc.error_cos && TEST_ERROR)
test_pktio_error_cos(enable_pktv);
if (tc.pmr_chain && TEST_PMR_CHAIN)
diff --git a/test/validation/api/classification/odp_classification_testsuites.h b/test/validation/api/classification/odp_classification_testsuites.h
index 2b5da94e2..6b00e138b 100644
--- a/test/validation/api/classification/odp_classification_testsuites.h
+++ b/test/validation/api/classification/odp_classification_testsuites.h
@@ -34,6 +34,7 @@ typedef struct cls_packet_info {
typedef union odp_cls_testcase {
struct {
uint32_t default_cos:1;
+ uint32_t drop_cos:1;
uint32_t error_cos:1;
uint32_t pmr_chain:1;
uint32_t l2_priority:1;
@@ -72,6 +73,8 @@ odp_pool_t pktv_pool_create(const char *poolname);
odp_queue_t queue_create(const char *queuename, bool sched);
void configure_pktio_default_cos(odp_bool_t enable_pktv);
void test_pktio_default_cos(odp_bool_t enable_pktv);
+void configure_pktio_drop_cos(odp_bool_t enable_pktv, uint32_t max_cos_stats);
+void test_pktio_drop_cos(odp_bool_t enable_pktv);
void configure_pktio_error_cos(odp_bool_t enable_pktv);
void test_pktio_error_cos(odp_bool_t enable_pktv);
void configure_cls_pmr_chain(odp_bool_t enable_pktv);
diff --git a/test/validation/api/crypto/odp_crypto_test_inp.c b/test/validation/api/crypto/odp_crypto_test_inp.c
index 721dfe8c9..97f721dd5 100644
--- a/test/validation/api/crypto/odp_crypto_test_inp.c
+++ b/test/validation/api/crypto/odp_crypto_test_inp.c
@@ -37,7 +37,9 @@ static void test_default_values(void)
CU_ASSERT_EQUAL(param.op, ODP_CRYPTO_OP_ENCODE);
CU_ASSERT_EQUAL(param.auth_cipher_text, false);
+#if ODP_DEPRECATED_API
CU_ASSERT_EQUAL(param.pref_mode, ODP_CRYPTO_SYNC);
+#endif
CU_ASSERT_EQUAL(param.op_mode, ODP_CRYPTO_SYNC);
CU_ASSERT_EQUAL(param.cipher_alg, ODP_CIPHER_ALG_NULL);
CU_ASSERT_EQUAL(param.cipher_iv_len, 0);
@@ -194,6 +196,7 @@ static const char *cipher_alg_name(odp_cipher_alg_t cipher)
}
}
+#if ODP_DEPRECATED_API
static int alg_op(odp_packet_t pkt,
odp_bool_t *ok,
odp_crypto_session_t session,
@@ -273,6 +276,7 @@ static int alg_op(odp_packet_t pkt,
return 0;
}
+#endif
static int alg_packet_op(odp_packet_t pkt,
odp_bool_t *ok,
@@ -370,10 +374,14 @@ static int crypto_op(odp_packet_t pkt,
int rc;
if (!suite_context.packet)
+#if ODP_DEPRECATED_API
rc = alg_op(pkt, ok, session,
cipher_iv, auth_iv,
cipher_range, auth_range,
aad, hash_result_offset);
+#else
+ rc = -1;
+#endif
else
rc = alg_packet_op(pkt, ok, session,
cipher_iv, auth_iv,
@@ -657,7 +665,9 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op,
ses_params.op = op;
ses_params.auth_cipher_text = false;
ses_params.op_mode = suite_context.op_mode;
+#if ODP_DEPRECATED_API
ses_params.pref_mode = suite_context.pref_mode;
+#endif
ses_params.cipher_alg = cipher_alg;
ses_params.auth_alg = auth_alg;
ses_params.compl_queue = suite_context.queue;
@@ -2154,6 +2164,7 @@ static odp_event_t plain_compl_queue_deq(void)
return odp_queue_deq(suite_context.queue);
}
+#if ODP_DEPRECATED_API
static int crypto_suite_sync_init(void)
{
suite_context.pool = odp_pool_lookup("packet_pool");
@@ -2206,6 +2217,7 @@ static int crypto_suite_async_sched_init(void)
return 0;
}
+#endif
static int crypto_suite_packet_sync_init(void)
{
@@ -2422,12 +2434,14 @@ odp_testinfo_t crypto_suite[] = {
};
odp_suiteinfo_t crypto_suites[] = {
+#if ODP_DEPRECATED_API
{"odp_crypto_sync_inp", crypto_suite_sync_init,
NULL, crypto_suite},
{"odp_crypto_async_plain_inp", crypto_suite_async_plain_init,
crypto_suite_term, crypto_suite},
{"odp_crypto_async_sched_inp", crypto_suite_async_sched_init,
crypto_suite_term, crypto_suite},
+#endif
{"odp_crypto_packet_sync_inp", crypto_suite_packet_sync_init,
NULL, crypto_suite},
{"odp_crypto_packet_async_plain_inp",
diff --git a/test/validation/api/ipsec/ipsec.c b/test/validation/api/ipsec/ipsec.c
index 88f204ea4..d12234a5a 100644
--- a/test/validation/api/ipsec/ipsec.c
+++ b/test/validation/api/ipsec/ipsec.c
@@ -281,10 +281,6 @@ int ipsec_check_esp_chacha20_poly1305(void)
int ipsec_check_test_sa_update_seq_num(void)
{
- odp_ipsec_capability_t capa;
-
- odp_ipsec_capability(&capa);
-
if (!capa.test.sa_operations.seq_num)
return ODP_TEST_INACTIVE;
@@ -942,9 +938,8 @@ static void verify_in(const ipsec_test_part *part,
result.orig_ip_len == len);
}
}
- ipsec_check_packet(part->out[i].pkt_res,
- pkto[i],
- false);
+ if (part->out[i].l3_type != ODP_PROTO_L3_TYPE_NONE)
+ ipsec_check_packet(part->out[i].pkt_res, pkto[i], false);
if (suite_context.inbound_op_mode == ODP_IPSEC_OP_MODE_INLINE)
expected_user_ptr = NULL;
CU_ASSERT(odp_packet_user_ptr(pkto[i]) == expected_user_ptr);
diff --git a/test/validation/api/pktio/pktio.c b/test/validation/api/pktio/pktio.c
index 9144e1260..f6408c788 100644
--- a/test/validation/api/pktio/pktio.c
+++ b/test/validation/api/pktio/pktio.c
@@ -770,8 +770,8 @@ static int recv_packets_tmo(odp_pktio_t pktio, odp_packet_t pkt_tbl[],
int num_q;
int i;
int n;
- unsigned int from_val = 0;
- unsigned *from = NULL;
+ uint32_t from_val = 0;
+ uint32_t *from = NULL;
if (mode == RECV_MQ_TMO)
from = &from_val;
@@ -787,8 +787,7 @@ static int recv_packets_tmo(odp_pktio_t pktio, odp_packet_t pkt_tbl[],
n = odp_pktin_recv_tmo(pktin[0], pkt_tmp, num - num_rx,
tmo);
else
- n = odp_pktin_recv_mq_tmo(pktin, (unsigned)num_q,
- from, pkt_tmp,
+ n = odp_pktin_recv_mq_tmo(pktin, (uint32_t)num_q, from, pkt_tmp,
num - num_rx, tmo);
ts2 = odp_time_global();
@@ -812,7 +811,7 @@ static int recv_packets_tmo(odp_pktio_t pktio, odp_packet_t pkt_tbl[],
odp_packet_free(pkt_tmp[i]);
}
if (mode == RECV_MQ_TMO)
- CU_ASSERT(from_val < (unsigned)num_q);
+ CU_ASSERT(from_val < (uint32_t)num_q);
} while (num_rx < num);
if (num_rx < num) {
@@ -1265,7 +1264,7 @@ static void test_recv_tmo(recv_tmo_mode_e mode)
odp_packet_t pkt_tbl[test_pkt_count];
uint32_t pkt_seq[test_pkt_count];
uint64_t ns;
- unsigned num_q;
+ uint32_t num_q;
int ret;
int i;
diff --git a/test/validation/api/traffic_mngr/traffic_mngr.c b/test/validation/api/traffic_mngr/traffic_mngr.c
index cab1fd06d..65f91192c 100644
--- a/test/validation/api/traffic_mngr/traffic_mngr.c
+++ b/test/validation/api/traffic_mngr/traffic_mngr.c
@@ -120,6 +120,12 @@ typedef enum {
SHAPER_PROFILE, SCHED_PROFILE, THRESHOLD_PROFILE, WRED_PROFILE
} profile_kind_t;
+typedef enum {
+ THRESHOLD_BYTE,
+ THRESHOLD_PACKET,
+ THRESHOLD_BYTE_AND_PACKET
+} threshold_type_t;
+
typedef struct {
uint32_t num_queues;
odp_tm_queue_t tm_queues[];
@@ -516,6 +522,7 @@ static int open_pktios(void)
uint32_t iface;
char pool_name[ODP_POOL_NAME_LEN];
int rc, ret;
+ int pkt_aging = 0;
int lso = 0;
odp_pool_param_init(&pool_param);
@@ -620,6 +627,12 @@ static int open_pktios(void)
odp_pktio_config_init(&pktio_config);
+ /* Enable packet aging if supported */
+ if (xmt_pktio_capa.max_tx_aging_tmo_ns) {
+ pkt_aging = 1;
+ pktio_config.pktout.bit.aging_ena = 1;
+ }
+
/* Enable LSO if supported */
if (xmt_pktio_capa.lso.max_profiles && xmt_pktio_capa.lso.max_profiles_per_pktio) {
lso = 1;
@@ -627,7 +640,7 @@ static int open_pktios(void)
}
/* Enable selected features */
- if (lso) {
+ if (lso || pkt_aging) {
if (odp_pktio_config(xmt_pktio, &pktio_config)) {
ODPH_ERR("pktio configure failed\n");
return -1;
@@ -693,15 +706,16 @@ static int get_unique_id(odp_packet_t odp_pkt,
/* For IPv4 pkts use the ident field to store the unique_id. */
ident_offset = l3_offset + offsetof(odph_ipv4hdr_t, id);
- odp_packet_copy_to_mem(odp_pkt, ident_offset, 2, &be_ip_ident);
+ CU_ASSERT_FATAL(odp_packet_copy_to_mem(odp_pkt, ident_offset, 2,
+ &be_ip_ident) == 0);
unique_id = odp_be_to_cpu_16(be_ip_ident);
is_ipv4 = true;
} else if (odp_packet_has_ipv6(odp_pkt)) {
/* For IPv6 pkts use the flow field to store the unique_id. */
flow_offset = l3_offset + offsetof(odph_ipv6hdr_t, ver_tc_flow);
- odp_packet_copy_to_mem(odp_pkt, flow_offset, 4,
- &be_ver_tc_flow);
+ CU_ASSERT_FATAL(odp_packet_copy_to_mem(odp_pkt, flow_offset, 4,
+ &be_ver_tc_flow) == 0);
ver_tc_flow = odp_be_to_cpu_32(be_ver_tc_flow);
unique_id = ver_tc_flow & ODPH_IPV6HDR_FLOW_LABEL_MASK;
is_ipv4 = false;
@@ -1818,11 +1832,15 @@ set_reqs_based_on_capas(odp_tm_requirements_t *req)
req->tm_queue_dual_slope_needed = true;
if (tm_capabilities.vlan_marking_supported)
req->vlan_marking_needed = true;
- if (tm_capabilities.tm_queue_threshold)
+ if (tm_capabilities.tm_queue_threshold.byte ||
+ tm_capabilities.tm_queue_threshold.packet ||
+ tm_capabilities.tm_queue_threshold.byte_and_packet)
req->tm_queue_threshold_needed = true;
for (j = 0; j < tm_capabilities.max_levels; j++) {
- if (tm_capabilities.per_level[j].tm_node_threshold)
+ if (tm_capabilities.per_level[j].tm_node_threshold.byte ||
+ tm_capabilities.per_level[j].tm_node_threshold.packet ||
+ tm_capabilities.per_level[j].tm_node_threshold.byte_and_packet)
req->per_level[j].tm_node_threshold_needed = true;
}
@@ -2638,7 +2656,8 @@ static void traffic_mngr_test_sched_profile(void)
}
static void check_threshold_profile(char *threshold_name,
- uint32_t threshold_idx)
+ uint32_t threshold_idx,
+ threshold_type_t threshold)
{
odp_tm_threshold_params_t threshold_params;
odp_tm_threshold_t profile;
@@ -2657,15 +2676,17 @@ static void check_threshold_profile(char *threshold_name,
if (ret)
return;
- CU_ASSERT(threshold_params.max_pkts ==
- threshold_idx * MIN_PKT_THRESHOLD);
- CU_ASSERT(threshold_params.max_bytes ==
- threshold_idx * MIN_BYTE_THRESHOLD);
- CU_ASSERT(threshold_params.enable_max_pkts == 1);
- CU_ASSERT(threshold_params.enable_max_bytes == 1);
+ if (threshold == THRESHOLD_PACKET || threshold == THRESHOLD_BYTE_AND_PACKET) {
+ CU_ASSERT(threshold_params.enable_max_pkts == 1);
+ CU_ASSERT(threshold_params.max_pkts == threshold_idx * MIN_PKT_THRESHOLD);
+ }
+ if (threshold == THRESHOLD_BYTE || threshold == THRESHOLD_BYTE_AND_PACKET) {
+ CU_ASSERT(threshold_params.enable_max_bytes == 1);
+ CU_ASSERT(threshold_params.max_bytes == threshold_idx * MIN_BYTE_THRESHOLD);
+ }
}
-static void traffic_mngr_test_threshold_profile(void)
+static void traffic_mngr_test_threshold_profile(threshold_type_t threshold)
{
odp_tm_threshold_params_t threshold_params;
odp_tm_threshold_t profile;
@@ -2673,14 +2694,19 @@ static void traffic_mngr_test_threshold_profile(void)
char threshold_name[TM_NAME_LEN];
odp_tm_threshold_params_init(&threshold_params);
- threshold_params.enable_max_pkts = 1;
- threshold_params.enable_max_bytes = 1;
+
+ if (threshold == THRESHOLD_PACKET || threshold == THRESHOLD_BYTE_AND_PACKET)
+ threshold_params.enable_max_pkts = 1;
+ if (threshold == THRESHOLD_BYTE || threshold == THRESHOLD_BYTE_AND_PACKET)
+ threshold_params.enable_max_bytes = 1;
for (idx = 1; idx <= NUM_THRESH_TEST_PROFILES; idx++) {
snprintf(threshold_name, sizeof(threshold_name),
"threshold_profile_%" PRIu32, idx);
- threshold_params.max_pkts = idx * MIN_PKT_THRESHOLD;
- threshold_params.max_bytes = idx * MIN_BYTE_THRESHOLD;
+ if (threshold == THRESHOLD_PACKET || threshold == THRESHOLD_BYTE_AND_PACKET)
+ threshold_params.max_pkts = idx * MIN_PKT_THRESHOLD;
+ if (threshold == THRESHOLD_BYTE || threshold == THRESHOLD_BYTE_AND_PACKET)
+ threshold_params.max_bytes = idx * MIN_BYTE_THRESHOLD;
profile = odp_tm_threshold_create(threshold_name,
&threshold_params);
@@ -2702,10 +2728,30 @@ static void traffic_mngr_test_threshold_profile(void)
threshold_idx = ((3 + 7 * idx) % NUM_THRESH_TEST_PROFILES) + 1;
snprintf(threshold_name, sizeof(threshold_name),
"threshold_profile_%" PRIu32, threshold_idx);
- check_threshold_profile(threshold_name, threshold_idx);
+ check_threshold_profile(threshold_name, threshold_idx, threshold);
+ }
+
+ for (i = 0; i < NUM_THRESH_TEST_PROFILES; i++) {
+ CU_ASSERT(odp_tm_threshold_destroy(threshold_profiles[i]) == 0);
+ num_threshold_profiles--;
}
}
+static void traffic_mngr_test_threshold_profile_byte(void)
+{
+ traffic_mngr_test_threshold_profile(THRESHOLD_BYTE);
+}
+
+static void traffic_mngr_test_threshold_profile_packet(void)
+{
+ traffic_mngr_test_threshold_profile(THRESHOLD_PACKET);
+}
+
+static void traffic_mngr_test_threshold_profile_byte_and_packet(void)
+{
+ traffic_mngr_test_threshold_profile(THRESHOLD_BYTE_AND_PACKET);
+}
+
static void check_wred_profile(char *wred_name,
uint32_t wred_idx,
uint32_t color)
@@ -3353,6 +3399,7 @@ static int test_sched_wfq(const char *sched_base_name,
if (FANIN_RATIO <= fanin)
fanin = 0;
}
+ CU_ASSERT(pkts_sent == pkt_cnt + 4);
busy_wait(1000000); /* wait 1 millisecond */
@@ -3433,7 +3480,8 @@ static int test_threshold(const char *threshold_name,
odp_tm_threshold_params_t threshold_params;
odp_tm_queue_t tm_queue;
pkt_info_t pkt_info;
- uint32_t num_pkts, pkt_len, pkts_sent;
+ uint32_t pkt_len, pkts_sent;
+ uint32_t num_pkts = 0;
odp_tm_threshold_params_init(&threshold_params);
if (max_pkts != 0) {
@@ -3442,16 +3490,19 @@ static int test_threshold(const char *threshold_name,
threshold_params.enable_max_pkts = true;
num_pkts = 2 * max_pkts;
pkt_len = 256;
- } else if (max_bytes != 0) {
+ }
+
+ if (max_bytes != 0) {
max_bytes = MIN(max_bytes, MAX_PKTS * MAX_PAYLOAD / 3);
threshold_params.max_bytes = max_bytes;
threshold_params.enable_max_bytes = true;
num_pkts = 2 * max_bytes / MAX_PAYLOAD;
pkt_len = MAX_PAYLOAD;
- } else {
- return -1;
}
+ if (max_pkts == 0 && max_bytes == 0)
+ return -1;
+
/* Pick a tm_queue and set the tm_queue's threshold profile and then
* send in twice the amount of traffic as suggested by the thresholds
* and make sure at least SOME pkts get dropped. */
@@ -4366,6 +4417,40 @@ static int test_fanin_info(const char *node_name)
return walk_tree_backwards(node_desc->node);
}
+static void test_packet_aging(uint64_t tmo_ns, uint32_t pkt_len, odp_bool_t is_dropping)
+{
+ odp_tm_queue_t tm_queue;
+ const char *node_name = "node_1_1_1";
+ const char *shaper_name = "test_shaper";
+ const uint64_t rate = 256 * 1000;
+ pkt_info_t pkt_info;
+ const uint16_t num_pkts = 4;
+ int recv_pkts;
+
+ tm_queue = find_tm_queue(0, node_name, 0);
+ CU_ASSERT_FATAL(tm_queue != ODP_TM_INVALID);
+ init_xmt_pkts(&pkt_info);
+ pkt_info.drop_eligible = false;
+ pkt_info.pkt_class = 1;
+ CU_ASSERT_FATAL(make_pkts(num_pkts, pkt_len, &pkt_info) == 0);
+
+ for (int i = 0; i < num_pkts; i++)
+ odp_packet_aging_tmo_set(xmt_pkts[i], tmo_ns);
+
+ CU_ASSERT_FATAL(set_shaper(node_name, shaper_name, rate, rate) == 0);
+ CU_ASSERT(send_pkts(tm_queue, num_pkts) == num_pkts);
+ recv_pkts = receive_pkts(odp_tm_systems[0], rcv_pktin, num_pkts, MBPS);
+
+ if (is_dropping)
+ CU_ASSERT(recv_pkts < num_pkts)
+ else
+ CU_ASSERT(recv_pkts == num_pkts);
+
+ set_shaper(node_name, NULL, 0, 0);
+ flush_leftover_pkts(odp_tm_systems[0], rcv_pktin);
+ CU_ASSERT(odp_tm_is_idle(odp_tm_systems[0]));
+}
+
static void traffic_mngr_test_default_values(void)
{
odp_tm_requirements_t req;
@@ -4517,25 +4602,51 @@ static void traffic_mngr_test_scheduler(void)
INCREASING_WEIGHTS) == 0);
}
-static int traffic_mngr_check_thresholds(void)
+static int traffic_mngr_check_thresholds_byte(void)
{
- /* Check only for tm queue threshold support as
- * we only test queue threshold.
- */
- if (!tm_capabilities.tm_queue_threshold)
+ /* Check only for TM queue threshold support as we only test queue threshold. */
+ if (!tm_capabilities.tm_queue_threshold.byte)
return ODP_TEST_INACTIVE;
return ODP_TEST_ACTIVE;
}
-static void traffic_mngr_test_thresholds(void)
+static int traffic_mngr_check_thresholds_packet(void)
{
- CU_ASSERT(test_threshold("thresh_A", "shaper_A", "node_1_2_1", 0,
- 16, 0) == 0);
- CU_ASSERT(test_threshold("thresh_B", "shaper_B", "node_1_2_1", 1,
+ /* Check only for TM queue threshold support as we only test queue threshold. */
+ if (!tm_capabilities.tm_queue_threshold.packet)
+ return ODP_TEST_INACTIVE;
+
+ return ODP_TEST_ACTIVE;
+}
+
+static int traffic_mngr_check_thresholds_byte_and_packet(void)
+{
+ /* Check only for TM queue threshold support as we only test queue threshold. */
+ if (!tm_capabilities.tm_queue_threshold.byte_and_packet)
+ return ODP_TEST_INACTIVE;
+
+ return ODP_TEST_ACTIVE;
+}
+
+static void traffic_mngr_test_thresholds_byte(void)
+{
+ CU_ASSERT(test_threshold("thresh_byte", "shaper_B", "node_1_2_1", 1,
0, 6400) == 0);
}
+static void traffic_mngr_test_thresholds_packet(void)
+{
+ CU_ASSERT(test_threshold("thresh_packet", "shaper_A", "node_1_2_1", 0,
+ 16, 0) == 0);
+}
+
+static void traffic_mngr_test_thresholds_byte_and_packet(void)
+{
+ CU_ASSERT(test_threshold("thresh_byte_and_packet", "shaper_A", "node_1_2_1", 0,
+ 16, 6400) == 0);
+}
+
static int traffic_mngr_check_queue_stats(void)
{
if (tm_capabilities.queue_stats.all_counters == 0)
@@ -4617,7 +4728,8 @@ static int traffic_mngr_check_wred(void)
static int traffic_mngr_check_byte_wred(void)
{
/* Check if wred is part of created odp_tm_t capabilities */
- if (!tm_capabilities.tm_queue_wred_supported)
+ if (!tm_capabilities.tm_queue_wred_supported ||
+ !tm_capabilities.tm_queue_threshold.byte)
return ODP_TEST_INACTIVE;
if ((tm_shaper_min_rate > 64 * 1000) ||
@@ -4631,7 +4743,8 @@ static int traffic_mngr_check_byte_wred(void)
static int traffic_mngr_check_pkt_wred(void)
{
/* Check if wred is part of created odp_tm_t capabilities */
- if (!tm_capabilities.tm_queue_wred_supported)
+ if (!tm_capabilities.tm_queue_wred_supported ||
+ !tm_capabilities.tm_queue_threshold.packet)
return ODP_TEST_INACTIVE;
if ((tm_shaper_min_rate > 64 * 1000) ||
@@ -4772,6 +4885,23 @@ static void traffic_mngr_test_ecn_drop_prec_marking(void)
CU_ASSERT(ip_marking_tests("node_1_4_2", true, true) == 0);
}
+static int traffic_mngr_check_tx_aging(void)
+{
+ return xmt_pktio_capa.max_tx_aging_tmo_ns ? ODP_TEST_ACTIVE : ODP_TEST_INACTIVE;
+}
+
+static void traffic_mngr_test_tx_aging_no_drop(void)
+{
+ /* Set very long aging tmo, packets should not be dropped due to aging */
+ test_packet_aging(60000000000, 128, false);
+}
+
+static void traffic_mngr_test_tx_aging_drop(void)
+{
+ /* Set very short aging tmo, there should be drops due to aging */
+ test_packet_aging(10, MAX_PAYLOAD, true);
+}
+
static void traffic_mngr_test_fanin_info(void)
{
CU_ASSERT(test_fanin_info("node_1") == 0);
@@ -4845,16 +4975,24 @@ odp_testinfo_t traffic_mngr_suite[] = {
ODP_TEST_INFO(traffic_mngr_test_tm_create),
ODP_TEST_INFO(traffic_mngr_test_shaper_profile),
ODP_TEST_INFO(traffic_mngr_test_sched_profile),
- ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_threshold_profile,
- traffic_mngr_check_thresholds),
+ ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_threshold_profile_byte,
+ traffic_mngr_check_thresholds_byte),
+ ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_threshold_profile_packet,
+ traffic_mngr_check_thresholds_packet),
+ ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_threshold_profile_byte_and_packet,
+ traffic_mngr_check_thresholds_byte_and_packet),
ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_wred_profile,
traffic_mngr_check_wred),
ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_shaper,
traffic_mngr_check_shaper),
ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_scheduler,
traffic_mngr_check_scheduler),
- ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_thresholds,
- traffic_mngr_check_thresholds),
+ ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_thresholds_byte,
+ traffic_mngr_check_thresholds_byte),
+ ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_thresholds_packet,
+ traffic_mngr_check_thresholds_packet),
+ ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_thresholds_byte_and_packet,
+ traffic_mngr_check_thresholds_byte_and_packet),
ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_byte_wred,
traffic_mngr_check_byte_wred),
ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_pkt_wred,
@@ -4871,6 +5009,10 @@ odp_testinfo_t traffic_mngr_suite[] = {
traffic_mngr_check_drop_prec_marking),
ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_ecn_drop_prec_marking,
traffic_mngr_check_ecn_drop_prec_marking),
+ ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_tx_aging_no_drop,
+ traffic_mngr_check_tx_aging),
+ ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_tx_aging_drop,
+ traffic_mngr_check_tx_aging),
ODP_TEST_INFO(traffic_mngr_test_fanin_info),
ODP_TEST_INFO_CONDITIONAL(traffic_mngr_test_lso_ipv4, traffic_mngr_check_lso_ipv4),
ODP_TEST_INFO(traffic_mngr_test_destroy),