aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMatias Elo <matias.elo@nokia.com>2023-02-24 09:39:16 +0200
committerGitHub <noreply@github.com>2023-02-24 09:39:16 +0200
commit0f2f3271f64b25d7ce101b7a5c82b7fc17ba2d32 (patch)
tree5d4df86a452c52b1421b8039835ec793ac4019c7
parent282188e33695bb6863e289df61e0dfbd87ca205c (diff)
parentc7c99cfd5635da8c0afc7f3803ce6e1c82c3465b (diff)
Merge ODP v1.40.0.0v1.40.0.0_DPDK_19.11
Merge ODP linux-generic v1.40.0.0 into linux-dpdk.
-rw-r--r--.github/workflows/ci-pipeline-arm64.yml15
-rw-r--r--.github/workflows/ci-pipeline.yml13
-rw-r--r--CHANGELOG68
-rw-r--r--DEPENDENCIES60
-rw-r--r--config/odp-linux-dpdk.conf28
-rw-r--r--config/odp-linux-generic.conf28
-rw-r--r--configure.ac2
-rw-r--r--doc/images/timeout_fsm.gv8
-rw-r--r--doc/images/timer_fsm.gv15
-rw-r--r--doc/users-guide/users-guide-timer.adoc10
-rw-r--r--example/debug/odp_debug.c10
-rw-r--r--example/sysinfo/odp_sysinfo.c40
-rw-r--r--example/timer/odp_timer_accuracy.c27
-rw-r--r--example/timer/odp_timer_simple.c10
-rw-r--r--example/timer/odp_timer_test.c11
-rw-r--r--include/Makefile.am18
-rw-r--r--include/odp/api/abi-default/buffer.h15
-rw-r--r--include/odp/api/abi-default/buffer_types.h34
-rw-r--r--include/odp/api/abi-default/crypto.h19
-rw-r--r--include/odp/api/abi-default/crypto_types.h37
-rw-r--r--include/odp/api/buffer_types.h28
-rw-r--r--include/odp/api/crypto.h12
-rw-r--r--include/odp/api/crypto_types.h28
-rw-r--r--include/odp/api/spec/buffer.h15
-rw-r--r--include/odp/api/spec/buffer_types.h45
-rw-r--r--include/odp/api/spec/crypto.h1162
-rw-r--r--include/odp/api/spec/crypto_types.h1102
-rw-r--r--include/odp/api/spec/ipsec.h2
-rw-r--r--include/odp/api/spec/ipsec_types.h2
-rw-r--r--include/odp/api/spec/packet_flags.h37
-rw-r--r--include/odp/api/spec/packet_io.h33
-rw-r--r--include/odp/api/spec/packet_io_stats.h15
-rw-r--r--include/odp/api/spec/packet_io_types.h1
-rw-r--r--include/odp/api/spec/shared_memory.h13
-rw-r--r--include/odp/api/spec/timer.h5
-rw-r--r--include/odp/api/spec/timer_types.h42
-rw-r--r--include/odp/api/spec/traffic_mngr.h45
-rw-r--r--include/odp/arch/arm32-linux/odp/api/abi/buffer_types.h7
-rw-r--r--include/odp/arch/arm32-linux/odp/api/abi/crypto_types.h7
-rw-r--r--include/odp/arch/arm64-linux/odp/api/abi/buffer_types.h7
-rw-r--r--include/odp/arch/arm64-linux/odp/api/abi/crypto_types.h7
-rw-r--r--include/odp/arch/default-linux/odp/api/abi/buffer_types.h7
-rw-r--r--include/odp/arch/default-linux/odp/api/abi/crypto_types.h7
-rw-r--r--include/odp/arch/power64-linux/odp/api/abi/buffer_types.h7
-rw-r--r--include/odp/arch/power64-linux/odp/api/abi/crypto_types.h7
-rw-r--r--include/odp/arch/x86_32-linux/odp/api/abi/buffer_types.h7
-rw-r--r--include/odp/arch/x86_32-linux/odp/api/abi/crypto_types.h7
-rw-r--r--include/odp/arch/x86_64-linux/odp/api/abi/buffer_types.h7
-rw-r--r--include/odp/arch/x86_64-linux/odp/api/abi/crypto_types.h7
-rw-r--r--include/odp/autoheader_external.h.in3
-rw-r--r--include/odp/autoheader_internal.h.in3
-rw-r--r--platform/linux-dpdk/Makefile.am9
l---------platform/linux-dpdk/include-abi/odp/api/abi/buffer_types.h1
l---------platform/linux-dpdk/include-abi/odp/api/abi/crypto_types.h1
-rw-r--r--platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h13
l---------platform/linux-dpdk/include/odp/api/plat/crypto_inlines.h1
l---------platform/linux-dpdk/include/odp/api/plat/event_validation_external.h1
-rw-r--r--platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h3
-rw-r--r--platform/linux-dpdk/include/odp/api/plat/packet_inlines.h11
-rw-r--r--platform/linux-dpdk/include/odp_config_internal.h10
-rw-r--r--platform/linux-dpdk/include/odp_event_internal.h9
-rw-r--r--platform/linux-dpdk/include/odp_packet_io_internal.h8
-rw-r--r--platform/linux-dpdk/include/odp_pool_internal.h5
-rw-r--r--platform/linux-dpdk/m4/configure.m43
l---------platform/linux-dpdk/m4/odp_event_validation.m41
-rw-r--r--platform/linux-dpdk/m4/odp_libconfig.m42
-rw-r--r--platform/linux-dpdk/odp_crypto.c106
-rw-r--r--platform/linux-dpdk/odp_event.c25
-rw-r--r--platform/linux-dpdk/odp_init.c16
-rw-r--r--platform/linux-dpdk/odp_packet.c7
-rw-r--r--platform/linux-dpdk/odp_packet_flags.c2
-rw-r--r--platform/linux-dpdk/odp_pool.c33
-rw-r--r--platform/linux-dpdk/odp_queue_basic.c47
-rw-r--r--platform/linux-dpdk/odp_shared_memory.c15
-rw-r--r--platform/linux-dpdk/odp_timer.c11
-rw-r--r--platform/linux-dpdk/test/alternate-timer.conf2
-rw-r--r--platform/linux-dpdk/test/crypto.conf2
-rw-r--r--platform/linux-dpdk/test/process-mode.conf2
-rw-r--r--platform/linux-dpdk/test/sched-basic.conf3
-rw-r--r--platform/linux-dpdk/test/stash-custom.conf8
-rw-r--r--platform/linux-generic/Makefile.am15
-rw-r--r--platform/linux-generic/README26
-rw-r--r--platform/linux-generic/arch/aarch64/odp_crypto_armv8.c128
-rw-r--r--platform/linux-generic/include-abi/odp/api/abi/buffer.h20
-rw-r--r--platform/linux-generic/include-abi/odp/api/abi/buffer_types.h40
-rw-r--r--platform/linux-generic/include-abi/odp/api/abi/crypto.h15
-rw-r--r--platform/linux-generic/include-abi/odp/api/abi/crypto_types.h42
-rw-r--r--platform/linux-generic/include/odp/api/plat/buffer_inlines.h5
-rw-r--r--platform/linux-generic/include/odp/api/plat/crypto_inlines.h65
-rw-r--r--platform/linux-generic/include/odp/api/plat/event_validation_external.h111
-rw-r--r--platform/linux-generic/include/odp/api/plat/packet_inline_types.h1
-rw-r--r--platform/linux-generic/include/odp/api/plat/packet_inlines.h3
-rw-r--r--platform/linux-generic/include/odp/api/plat/pool_inline_types.h1
-rw-r--r--platform/linux-generic/include/odp/api/plat/queue_inlines.h10
-rw-r--r--platform/linux-generic/include/odp_config_internal.h10
-rw-r--r--platform/linux-generic/include/odp_event_internal.h7
-rw-r--r--platform/linux-generic/include/odp_event_validation_internal.h52
-rw-r--r--platform/linux-generic/include/odp_init_internal.h3
-rw-r--r--platform/linux-generic/include/odp_packet_io_internal.h8
-rw-r--r--platform/linux-generic/include/odp_pool_internal.h1
-rw-r--r--platform/linux-generic/include/odp_queue_basic_internal.h5
-rw-r--r--platform/linux-generic/include/odp_ring_mpmc_internal.h210
-rw-r--r--platform/linux-generic/include/odp_ring_mpmc_u32_internal.h25
-rw-r--r--platform/linux-generic/include/odp_ring_mpmc_u64_internal.h25
-rw-r--r--platform/linux-generic/libodp-linux.pc.in2
-rw-r--r--platform/linux-generic/m4/configure.m46
-rw-r--r--platform/linux-generic/m4/odp_crypto.m412
-rw-r--r--platform/linux-generic/m4/odp_event_validation.m423
-rw-r--r--platform/linux-generic/m4/odp_ipsec_mb.m419
-rw-r--r--platform/linux-generic/m4/odp_libconfig.m42
-rw-r--r--platform/linux-generic/odp_crypto_api.c11
-rw-r--r--platform/linux-generic/odp_crypto_ipsecmb.c881
-rw-r--r--platform/linux-generic/odp_crypto_null.c68
-rw-r--r--platform/linux-generic/odp_crypto_openssl.c395
-rw-r--r--platform/linux-generic/odp_event.c25
-rw-r--r--platform/linux-generic/odp_event_validation.c260
-rw-r--r--platform/linux-generic/odp_init.c16
-rw-r--r--platform/linux-generic/odp_ipsec_sad.c39
-rw-r--r--platform/linux-generic/odp_packet.c21
-rw-r--r--platform/linux-generic/odp_packet_flags.c2
-rw-r--r--platform/linux-generic/odp_packet_io.c29
-rw-r--r--platform/linux-generic/odp_pool.c31
-rw-r--r--platform/linux-generic/odp_queue_basic.c67
-rw-r--r--platform/linux-generic/odp_schedule_basic.c22
-rw-r--r--platform/linux-generic/odp_shared_memory.c9
-rw-r--r--platform/linux-generic/odp_stash.c665
-rw-r--r--platform/linux-generic/odp_timer.c9
-rw-r--r--platform/linux-generic/odp_traffic_mngr.c6
-rw-r--r--platform/linux-generic/pktio/dpdk.c10
-rw-r--r--platform/linux-generic/pktio/loop.c323
-rw-r--r--platform/linux-generic/pktio/stats/packet_io_stats.c5
-rw-r--r--platform/linux-generic/test/inline-timer.conf2
-rw-r--r--platform/linux-generic/test/packet_align.conf2
-rw-r--r--platform/linux-generic/test/process-mode.conf2
-rw-r--r--platform/linux-generic/test/sched-basic.conf3
-rw-r--r--platform/linux-generic/test/stash-custom.conf8
-rw-r--r--test/Makefile.inc1
-rw-r--r--test/common/Makefile.am8
-rw-r--r--test/common/packet_common.c133
-rw-r--r--test/common/packet_common.h67
-rw-r--r--test/performance/.gitignore1
-rw-r--r--test/performance/Makefile.am2
-rw-r--r--test/performance/odp_bench_buffer.c4
-rw-r--r--test/performance/odp_bench_misc.c1069
-rw-r--r--test/performance/odp_bench_packet.c2
-rw-r--r--test/performance/odp_crypto.c16
-rw-r--r--test/performance/odp_ipsec.c4
-rw-r--r--test/performance/odp_sched_perf.c3
-rw-r--r--test/performance/odp_sched_pktio.c22
-rw-r--r--test/performance/odp_timer_perf.c20
-rw-r--r--test/validation/api/classification/odp_classification_common.c2
-rw-r--r--test/validation/api/crypto/Makefile.am1
-rw-r--r--test/validation/api/crypto/odp_crypto_test_inp.c810
-rw-r--r--test/validation/api/dma/dma.c8
-rw-r--r--test/validation/api/packet/packet.c107
-rw-r--r--test/validation/api/pool/pool.c20
-rw-r--r--test/validation/api/system/system.c17
-rw-r--r--test/validation/api/time/time.c1
-rw-r--r--test/validation/api/timer/timer.c68
159 files changed, 6981 insertions, 2710 deletions
diff --git a/.github/workflows/ci-pipeline-arm64.yml b/.github/workflows/ci-pipeline-arm64.yml
index b4fb1e271..3d2f106a8 100644
--- a/.github/workflows/ci-pipeline-arm64.yml
+++ b/.github/workflows/ci-pipeline-arm64.yml
@@ -129,7 +129,8 @@ jobs:
cc: [gcc, clang]
conf: ['', '--enable-abi-compat', '--enable-deprecated --enable-helper-deprecated --enable-debug=full',
'--enable-dpdk-zero-copy --disable-static-applications',
- '--disable-host-optimization', '--disable-host-optimization --enable-abi-compat',
+ '--disable-host-optimization --enable-event-validation=warn',
+ '--disable-host-optimization --enable-abi-compat',
'--without-openssl --without-pcap']
steps:
- uses: AutoModality/action-clean@v1.1.0
@@ -185,6 +186,18 @@ jobs:
if: ${{ failure() }}
run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done
+ Run_stash_config:
+ if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }}
+ runs-on: [self-hosted, ARM64]
+ steps:
+ - uses: AutoModality/action-clean@v1.1.0
+ - uses: actions/checkout@v3
+ - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}"
+ -e CONF="${CONF}" -e ODP_CONFIG_FILE=/odp/platform/linux-dpdk/test/stash-custom.conf $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/check.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done
+
Run_scheduler_sp:
if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }}
runs-on: [self-hosted, ARM64]
diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml
index 425ee17b6..dff61a187 100644
--- a/.github/workflows/ci-pipeline.yml
+++ b/.github/workflows/ci-pipeline.yml
@@ -277,7 +277,8 @@ jobs:
cc: [gcc, clang]
conf: ['', '--enable-abi-compat', '--enable-deprecated --enable-helper-deprecated --enable-debug=full',
'--disable-static-applications',
- '--disable-host-optimization', '--disable-host-optimization --enable-abi-compat',
+ '--disable-host-optimization --enable-event-validation=warn',
+ '--disable-host-optimization --enable-abi-compat',
'--without-openssl --without-pcap']
steps:
- uses: actions/checkout@v3
@@ -312,6 +313,16 @@ jobs:
if: ${{ failure() }}
run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done
+ Run_stash_config:
+ runs-on: ubuntu-20.04
+ steps:
+ - uses: actions/checkout@v3
+ - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}"
+ -e CONF="${CONF}" -e ODP_CONFIG_FILE=/odp/platform/linux-dpdk/test/stash-custom.conf $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/check.sh
+ - name: Failure log
+ if: ${{ failure() }}
+ run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done
+
Run_scheduler_sp:
runs-on: ubuntu-20.04
steps:
diff --git a/CHANGELOG b/CHANGELOG
index 863b41002..8d6a88ddf 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,71 @@
+== OpenDataPlane (1.40.0.0)
+
+=== Backward incompatible API changes
+==== Packet
+* Specify which packet metadata flags cannot be set simultaneously using
+`odp_packet_has_XX_set()` functions.
+
+==== Timer
+* Use `ODP_DEPRECATE()` macro for `odp_timer_set_abs()` and
+`odp_timer_set_rel()` functions. Previously, the deprecation was only mentioned
+in the function documentation.
+* Deprecate old timer pool clock sources `ODP_CLOCK_CPU` and `ODP_CLOCK_EXT`,
+which have been replaced by `ODP_CLOCK_DEFAULT` and `ODP_CLOCK_SRC_1`.
+* Deprecate old timer set return values `ODP_TIMER_TOOEARLY`,
+`ODP_TIMER_TOOLATE`, and `ODP_TIMER_NOEVENT`, which have been replaced by
+`ODP_TIMER_TOO_NEAR`, `ODP_TIMER_TOO_FAR`, and `ODP_TIMER_FAIL`.
+
+=== Backward compatible API changes
+==== Crypto
+* Add new operation type session parameter
+(`odp_crypto_session_param_t.op_type`) that controls how crypto operations
+interpret their parameters and handle output packets. Defaults to backward
+compatible `ODP_CRYPTO_OP_TYPE_LEGACY` mode.
+* Add `ODP_CRYPTO_OP_TYPE_BASIC` operation type with simplified interface
+compared to `ODP_CRYPTO_OP_TYPE_LEGACY`.
+* Add `ODP_CRYPTO_OP_TYPE_OOP` operation type that writes the output of the
+crypto operation into a caller provided output packet and does not consume the
+input packet.
+* Clarify that `odp_crypto_op()` copies all packet data and metadata from the
+input packet to the output packet in `ODP_CRYPTO_OP_TYPE_LEGACY` and
+`ODP_CRYPTO_OP_TYPE_BASIC` modes.
+* Require that `odp_crypto_result()` is called before packet data of
+asynchronously processed packets can be assumed to be valid in non-legacy modes.
+* Fix EIA2 IV length in API documentation. EIA2 uses 64-bit IV, not 128-bit as
+previously mentioned in the API text.
+
+==== Packet
+* Clarify `odp_packet_has_vlan()` and `odp_packet_has_vlan_qinq()`
+specifications.
+
+=== Remove deprecated APIs
+==== Crypto
+* Remove deprecated per-session IV configuration.
+
+==== Packet IO
+* Remove deprecated `in_unknown_protos` field from `odp_pktio_stats_t`.
+* Remove deprecated `odp_pktin_ts_res()` function.
+* Remove deprecated `odp_pktin_ts_from_ns()` function.
+
+==== Shared Memory
+* Remove deprecated `ODP_SHM_SW_ONLY` define.
+
+==== Traffic Manager
+* Remove deprecated `odp_tm_capabilities()` function.
+* Remove deprecated `commit_bps` field from `odp_tm_shaper_params_t`.
+* Remove deprecated `peak_bps` field from `odp_tm_shaper_params_t`.
+
+=== Implementation
+==== Crypto
+* Add Multi-Buffer Crypto for IPsec library (Arm optimized) based crypto
+implementation. See `DEPENDENCIES` for additional information.
+
+==== Debug
+* Add support for runtime event validation (buffer endmark checking). Event
+validation can be enabled during configure with
+`--enable-event-validation [warn/abort]` or with `--enabled-debug=full`. See
+`README` for additional information.
+
== OpenDataPlane (1.39.0.0)
=== Backward incompatible API changes
diff --git a/DEPENDENCIES b/DEPENDENCIES
index 0a1bea32b..816639bee 100644
--- a/DEPENDENCIES
+++ b/DEPENDENCIES
@@ -124,11 +124,39 @@ Prerequisites for building the OpenDataPlane (ODP) API
$ ./configure --with-crypto=armv8crypto
$ make
-3.4 Netmap packet I/O support (optional)
+3.4 Multi-buffer Crypto for IPsec Library (optional)
+
+ Multi-Buffer Crypto for IPsec Library is a set of functions that
+ implement authentication and encryption processing for IPsec, these functions
+ take advantage of SIMD instructions to improve performance.
+
+ Note ODP assumes that IPSec MB library is compiled with SAFE_PARAM enabled
+ (enabled by default), otherwise crypto operations with too long cipher/auth
+ ranges will have undefined behaviour.
+
+3.4.1 Building Multi-buffer Crypto for IPSec Library for Arm
+
+ # Checkout and build Arm code
+ $ git clone https://git.gitlab.arm.com/arm-reference-solutions/ipsec-mb.git
+ $ cd ipsec-mb/
+ $ git checkout SECLIB-IPSEC-2022.12.13
+ $ make
+ $ sudo make install
+
+ For additional instructions, refer to README.md in crypto library repository.
+
+3.4.2 Building ODP with Multi-buffer IPSec Library
+ $ ./bootstrap
+
+ # Compile and build ODP with Multi-buffer IPSec library
+ $ ./configure --with-crypto=ipsecmb
+ $ make
+
+3.5 Netmap packet I/O support (optional)
Netmap accelerated ODP packet I/O.
-3.4.1 Building netmap kernel modules
+3.5.1 Building netmap kernel modules
ODP works at least with the latest release version of netmap, which is
currently v13.0. However, if possible one should try to use the latest netmap
@@ -172,14 +200,14 @@ Prerequisites for building the OpenDataPlane (ODP) API
$ ./configure --kernel-sources=<path_to_kernel_src>
$ make
-3.4.2 Building ODP
+3.5.2 Building ODP
$ cd <odp_dir>
$ ./bootstrap
$ ./configure --with-netmap-path=<netmap_dir>
$ make
-3.4.3 Inserting netmap kernel modules
+3.5.3 Inserting netmap kernel modules
In order to use netmap I/O you need to insert at least the core netmap
kernel module.
@@ -197,13 +225,13 @@ Prerequisites for building the OpenDataPlane (ODP) API
To restore the original drivers you should be able to use modprobe.
-3.4.4 Running ODP with netmap I/O
+3.5.4 Running ODP with netmap I/O
ODP applications will use netmap for packet I/O by default as long as the
netmap kernel module is loaded. If socket I/O is desired instead, it can be
activated by setting the environment variable ODP_PKTIO_DISABLE_NETMAP.
-3.5 DPDK packet I/O support (optional)
+3.6 DPDK packet I/O support (optional)
Use DPDK for ODP packet I/O. Currently supported DPDK versions are v19.11,
v20.11 (recommended), v21.11.
@@ -212,7 +240,7 @@ Prerequisites for building the OpenDataPlane (ODP) API
https://github.com/OpenDataPlane/odp-dpdk.git
for a full DPDK based ODP implementation.
-3.5.1 DPDK pktio requirements
+3.6.1 DPDK pktio requirements
DPDK pktio adds a dependency to NUMA library.
# Debian/Ubuntu
@@ -221,11 +249,11 @@ Prerequisites for building the OpenDataPlane (ODP) API
# CentOS/RedHat/Fedora
$ sudo yum install numactl-devel
-3.5.2 Native DPDK install
+3.6.2 Native DPDK install
# Debian/Ubuntu starting from 20.04
$ sudo apt-get install dpdk-dev
-3.5.3 Build DPDK v19.11 from source
+3.6.3 Build DPDK v19.11 from source
$ git clone https://dpdk.org/git/dpdk-stable --branch 19.11 --depth 1 ./<dpdk-dir>
# Make and edit DPDK configuration
@@ -243,7 +271,7 @@ Prerequisites for building the OpenDataPlane (ODP) API
# Configure ODP
$ ./configure --with-dpdk-path=<dpdk-dir>
-3.5.4 Build DPDK v20.11 and onwards from source
+3.6.4 Build DPDK v20.11 and onwards from source
$ git clone https://dpdk.org/git/dpdk-stable --branch <version, e.g. 21.11> --depth 1 ./<dpdk-dir>
# Prepare the build directory
@@ -263,7 +291,7 @@ Prerequisites for building the OpenDataPlane (ODP) API
# Or, if DPDK was not installed to the default location, set PKG_CONFIG_PATH:
$ PKG_CONFIG_PATH=<dpdk-dir>/install/lib/x86_64-linux-gnu/pkgconfig ./configure --enable-dpdk
-3.5.5 Setup system
+3.6.5 Setup system
# Load DPDK modules
$ sudo modprobe uio
@@ -275,7 +303,7 @@ Prerequisites for building the OpenDataPlane (ODP) API
512 x 2MB huge pages. All this can be done with the DPDK setup script
(<dpdk-dir>/usertools/dpdk-setup.sh).
-3.5.6 Running ODP with DPDK pktio
+3.6.6 Running ODP with DPDK pktio
ODP applications will try use DPDK for packet I/O by default. If some other
I/O type is desired instead, DPDK I/O can be disabled by setting the
@@ -291,7 +319,7 @@ Prerequisites for building the OpenDataPlane (ODP) API
1024MB of memory:
$ sudo ODP_PKTIO_DPDK_PARAMS="-m 1024" ./test/performance/odp_l2fwd -i 0 -c 1
-3.6 AF_XDP socket based packet I/O support (optional)
+3.7 AF_XDP socket based packet I/O support (optional)
Use AF_XDP socket for packet I/O. At the moment, only zero-copy variant is
supported, requiring a kernel version 5.4 or higher. Additionally, if packet
@@ -308,7 +336,7 @@ Prerequisites for building the OpenDataPlane (ODP) API
Note that, currently, AF_XDP socket packet I/O cannot be instantiated if
DPDK zero-copy is enabled.
-3.6.1 AF_XDP socket packet I/O requirements
+3.7.1 AF_XDP socket packet I/O requirements
AF_XDP socket packet I/O implementation requires libxdp and libbpf libraries.
They can be fetched from XDP-project in GitHub:
@@ -330,14 +358,14 @@ Prerequisites for building the OpenDataPlane (ODP) API
$ cd <path to built libbpf>
$ make install
-3.6.2 Build ODP with AF_XDP socket packet I/O support
+3.7.2 Build ODP with AF_XDP socket packet I/O support
After building and installing libxdp and libbpf, ODP can be configured to be
built with AF_XDP support (pass PKG_CONFIG_PATH if needed).
$ ./configure --enable-xdp
-3.6.3 Running ODP with AF_XDP socket packet I/O
+3.7.3 Running ODP with AF_XDP socket packet I/O
AF_XDP socket packet I/Os bind to TRX-combined queues. Based on the packet
prosessing needs, NIC(s) of the environment should be configured
diff --git a/config/odp-linux-dpdk.conf b/config/odp-linux-dpdk.conf
index d28f728f2..8fe298e5b 100644
--- a/config/odp-linux-dpdk.conf
+++ b/config/odp-linux-dpdk.conf
@@ -16,7 +16,7 @@
# Mandatory fields
odp_implementation = "linux-dpdk"
-config_file_version = "0.1.18"
+config_file_version = "0.1.21"
# System options
system: {
@@ -192,6 +192,15 @@ sched_basic: {
worker = 1
control = 1
}
+
+ # Ordered queue reorder stash size
+ #
+ # Number of events each thread can stash internally before having to
+ # wait for the right order context. Reorder stash can improve
+ # performance if threads process events in bursts. If 'order_stash_size'
+ # > 0, events may be dropped by the implementation if the target queue
+ # is full. To prevent this set 'order_stash_size' to 0.
+ order_stash_size = 512
}
sched_eventdev: {
@@ -209,6 +218,23 @@ sched_eventdev: {
num_ports = 0
}
+stash: {
+ # Maximum number of stashes
+ max_num = 512
+
+ # Maximum number of objects in a stash
+ #
+ # The value may be rounded up by the implementation. For optimal memory
+ # usage set value to a power of two - 1.
+ max_num_obj = 4095
+
+ # Strict size
+ #
+ # When set to 0, application can attempt to store more handles into a
+ # stash than it specified in the creation parameters.
+ strict_size = 1
+}
+
timer: {
# Inline timer poll interval
#
diff --git a/config/odp-linux-generic.conf b/config/odp-linux-generic.conf
index f8accd07f..df5328d1f 100644
--- a/config/odp-linux-generic.conf
+++ b/config/odp-linux-generic.conf
@@ -16,7 +16,7 @@
# Mandatory fields
odp_implementation = "linux-generic"
-config_file_version = "0.1.22"
+config_file_version = "0.1.25"
# System options
system: {
@@ -234,6 +234,32 @@ sched_basic: {
worker = 1
control = 1
}
+
+ # Ordered queue reorder stash size
+ #
+ # Number of events each thread can stash internally before having to
+ # wait for the right order context. Reorder stash can improve
+ # performance if threads process events in bursts. If 'order_stash_size'
+ # > 0, events may be dropped by the implementation if the target queue
+ # is full. To prevent this set 'order_stash_size' to 0.
+ order_stash_size = 512
+}
+
+stash: {
+ # Maximum number of stashes
+ max_num = 512
+
+ # Maximum number of objects in a stash
+ #
+ # The value may be rounded up by the implementation. For optimal memory
+ # usage set value to a power of two - 1.
+ max_num_obj = 4095
+
+ # Strict size
+ #
+ # When set to 0, application can attempt to store more handles into a
+ # stash than it specified in the creation parameters.
+ strict_size = 1
}
timer: {
diff --git a/configure.ac b/configure.ac
index 9b3863cef..19b23e4b5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3,7 +3,7 @@ AC_PREREQ([2.5])
# ODP API version
##########################################################################
m4_define([odp_version_generation], [1])
-m4_define([odp_version_major], [39])
+m4_define([odp_version_major], [40])
m4_define([odp_version_minor], [0])
m4_define([odp_version_patch], [0])
diff --git a/doc/images/timeout_fsm.gv b/doc/images/timeout_fsm.gv
index dab24d0d6..179434bce 100644
--- a/doc/images/timeout_fsm.gv
+++ b/doc/images/timeout_fsm.gv
@@ -8,17 +8,13 @@ digraph timer_state_machine {
TO_Unalloc -> TO_Alloc [label="odp_timeout_alloc()"];
TO_Alloc -> TO_Unalloc [label="odp_timeout_free()"];
TO_Alloc -> TO_Pending [fontcolor=green,
- label="odp_timer_set_abs()"];
- TO_Alloc -> TO_Pending [fontcolor=green,
- label="odp_timer_set_rel()"];
+ label="odp_timer_start()"];
TO_Pending -> TO_Alloc [fontcolor=green,
label="odp_timer_cancel()"];
TO_Pending -> TO_Enqueued [fontcolor=green, label="timer expires"];
TO_Enqueued -> TO_Delivered [label="odp_schedule()"];
TO_Delivered -> TO_Pending [fontcolor=green,
- label="odp_timer_set_abs()"];
- TO_Delivered -> TO_Pending [fontcolor=green,
- label="odp_timer_set_rel()"];
+ label="odp_timer_start()"];
TO_Delivered -> TO_Delivered [label="odp_timeout_from_event()"];
TO_Delivered -> TO_Delivered [label="odp_timeout_timer()"];
TO_Delivered -> TO_Unalloc
diff --git a/doc/images/timer_fsm.gv b/doc/images/timer_fsm.gv
index 819d0f979..eb1df8ae9 100644
--- a/doc/images/timer_fsm.gv
+++ b/doc/images/timer_fsm.gv
@@ -3,17 +3,12 @@ digraph timer_state_machine {
node [fontsize=28];
edge [fontsize=28];
node [shape=doublecircle]; Timer_Unalloc;
- node [shape=circle]; Timer_Alloc Timer_Set Timer_Expired
+ node [shape=circle]; Timer_Alloc Timer_Active Timer_Expired
Timer_Unalloc -> Timer_Alloc [label="odp_timer_alloc()"];
Timer_Alloc -> Timer_Unalloc [label="odp_timer_free()"];
- Timer_Alloc -> Timer_Set [fontcolor=green,label="odp_timer_set_abs()"];
- Timer_Alloc -> Timer_Set [fontcolor=green,label="odp_timer_set_rel()"];
- Timer_Set -> Timer_Alloc [fontcolor=green,label="odp_timer_cancel()"];
- Timer_Set -> Timer_Expired [fontcolor=green,label="timer expires"];
+ Timer_Alloc -> Timer_Active [fontcolor=green,label="odp_timer_start()"];
+ Timer_Active -> Timer_Alloc [fontcolor=green,label="odp_timer_cancel()"];
+ Timer_Active -> Timer_Expired [fontcolor=green,label="timer expires"];
Timer_Expired -> Timer_Unalloc [label="odp_timer_free()"];
- Timer_Expired -> Timer_Set [fontcolor=green,
- label="odp_timer_set_abs()"];
- Timer_Expired -> Timer_Set [fontcolor=green,
- label="odp_timer_set_rel()"];
-
+ Timer_Expired -> Timer_Active [fontcolor=green, label="odp_timer_start()"];
}
diff --git a/doc/users-guide/users-guide-timer.adoc b/doc/users-guide/users-guide-timer.adoc
index 4a8bad74f..e854b5f2d 100644
--- a/doc/users-guide/users-guide-timer.adoc
+++ b/doc/users-guide/users-guide-timer.adoc
@@ -46,11 +46,11 @@ as an input parameter to enable the pool-specific conversion ratios to be
used.
Associated with each timer pool is a free running tick counter that can be
-sampled at any time via the `odp_timer_current_tick()` API. Timers can be set
-to an absolute future tick value via `odp_timer_set_abs()` or to a future tick
-value relative to the current tick via `odp_timer_set_rel()`. Implementations
-may impose minimum and maximum future values supported by a given timer pool
-and timer set operations will fail if the requested value is outside of the
+sampled at any time via the `odp_timer_current_tick()` API. Timers are started
+with `odp_timer_start()` and the expiration time can be an absolute future tick
+value or a future tick value relative to the current tick. Implementations may
+impose minimum and maximum future values supported by a given timer pool and
+timer start operations will fail if the requested value is outside of the
supported range.
Before a set timer expires, it can be canceled via the `odp_timer_cancel()`
diff --git a/example/debug/odp_debug.c b/example/debug/odp_debug.c
index 6b4ea977f..6683e46b4 100644
--- a/example/debug/odp_debug.c
+++ b/example/debug/odp_debug.c
@@ -372,6 +372,7 @@ static int timer_debug(void)
odp_timer_capability_t timer_capa;
odp_timer_pool_t timer_pool;
odp_timer_pool_param_t timer_param;
+ odp_timer_start_t start_param;
odp_timer_t timer;
odp_queue_t queue;
odp_queue_param_t queue_param;
@@ -452,8 +453,13 @@ static int timer_debug(void)
odp_timeout_print(timeout);
event = odp_timeout_to_event(timeout);
- if (odp_timer_set_rel(timer, tick, &event) != ODP_TIMER_SUCCESS)
- ODPH_ERR("Timer set failed.\n");
+
+ start_param.tick_type = ODP_TIMER_TICK_REL;
+ start_param.tick = tick;
+ start_param.tmo_ev = event;
+
+ if (odp_timer_start(timer, &start_param) != ODP_TIMER_SUCCESS)
+ ODPH_ERR("Timer start failed.\n");
printf("\n");
odp_timer_print(timer);
diff --git a/example/sysinfo/odp_sysinfo.c b/example/sysinfo/odp_sysinfo.c
index 523f70496..093b2f1bf 100644
--- a/example/sysinfo/odp_sysinfo.c
+++ b/example/sysinfo/odp_sysinfo.c
@@ -134,12 +134,26 @@ static const char *cipher_alg_name(odp_cipher_alg_t cipher)
return "aes_cbc";
case ODP_CIPHER_ALG_AES_CTR:
return "aes_ctr";
+ case ODP_CIPHER_ALG_AES_ECB:
+ return "aes_ecb";
+ case ODP_CIPHER_ALG_AES_CFB128:
+ return "aes_cfb128";
+ case ODP_CIPHER_ALG_AES_XTS:
+ return "aes_xts";
case ODP_CIPHER_ALG_AES_GCM:
return "aes_gcm";
case ODP_CIPHER_ALG_AES_CCM:
return "aes_ccm";
case ODP_CIPHER_ALG_CHACHA20_POLY1305:
return "chacha20_poly1305";
+ case ODP_CIPHER_ALG_KASUMI_F8:
+ return "kasumi_f8";
+ case ODP_CIPHER_ALG_SNOW3G_UEA2:
+ return "snow3g_uea2";
+ case ODP_CIPHER_ALG_AES_EEA2:
+ return "aes_eea2";
+ case ODP_CIPHER_ALG_ZUC_EEA3:
+ return "zuc_eea3";
default:
return "Unknown";
}
@@ -154,14 +168,14 @@ static const char *auth_alg_name(odp_auth_alg_t auth)
return "md5_hmac";
case ODP_AUTH_ALG_SHA1_HMAC:
return "sha1_hmac";
+ case ODP_AUTH_ALG_SHA224_HMAC:
+ return "sha224_hmac";
case ODP_AUTH_ALG_SHA256_HMAC:
return "sha256_hmac";
case ODP_AUTH_ALG_SHA384_HMAC:
return "sha384_hmac";
case ODP_AUTH_ALG_SHA512_HMAC:
return "sha512_hmac";
- case ODP_AUTH_ALG_AES_XCBC_MAC:
- return "aes_xcbc_mac";
case ODP_AUTH_ALG_AES_GCM:
return "aes_gcm";
case ODP_AUTH_ALG_AES_GMAC:
@@ -170,8 +184,30 @@ static const char *auth_alg_name(odp_auth_alg_t auth)
return "aes_ccm";
case ODP_AUTH_ALG_AES_CMAC:
return "aes_cmac";
+ case ODP_AUTH_ALG_AES_XCBC_MAC:
+ return "aes_xcbc_mac";
case ODP_AUTH_ALG_CHACHA20_POLY1305:
return "chacha20_poly1305";
+ case ODP_AUTH_ALG_KASUMI_F9:
+ return "kasumi_f9";
+ case ODP_AUTH_ALG_SNOW3G_UIA2:
+ return "snow3g_uia2";
+ case ODP_AUTH_ALG_AES_EIA2:
+ return "aes_eia2";
+ case ODP_AUTH_ALG_ZUC_EIA3:
+ return "zuc_eia3";
+ case ODP_AUTH_ALG_MD5:
+ return "md5";
+ case ODP_AUTH_ALG_SHA1:
+ return "sha1";
+ case ODP_AUTH_ALG_SHA224:
+ return "sha224";
+ case ODP_AUTH_ALG_SHA256:
+ return "sha256";
+ case ODP_AUTH_ALG_SHA384:
+ return "sha384";
+ case ODP_AUTH_ALG_SHA512:
+ return "sha512";
default:
return "Unknown";
}
diff --git a/example/timer/odp_timer_accuracy.c b/example/timer/odp_timer_accuracy.c
index 84031b9e9..0952320bb 100644
--- a/example/timer/odp_timer_accuracy.c
+++ b/example/timer/odp_timer_accuracy.c
@@ -233,7 +233,7 @@ static int start_timers(test_global_t *test_global)
odp_timer_t timer;
odp_queue_t queue;
odp_queue_param_t queue_param;
- uint64_t tick, start_tick;
+ uint64_t start_tick;
uint64_t period_ns, res_ns, res_hz, start_ns, nsec, offset_ns;
uint64_t max_res_ns, max_res_hz;
odp_event_t event;
@@ -452,17 +452,17 @@ static int start_timers(test_global_t *test_global)
for (i = 0; i < num_tmo; i++) {
for (j = 0; j < burst; j++) {
- nsec = offset_ns + (i * period_ns) + (j * burst_gap);
- tick = start_tick + odp_timer_ns_to_tick(timer_pool,
- nsec);
-
timer_ctx_t *ctx = &test_global->timer_ctx[idx];
+ odp_timer_start_t start_param;
- timer = ctx->timer;
- event = ctx->event;
+ nsec = offset_ns + (i * period_ns) + (j * burst_gap);
ctx->nsec = start_ns + nsec;
- ret = odp_timer_set_abs(timer, tick, &event);
+ start_param.tick_type = ODP_TIMER_TICK_ABS;
+ start_param.tick = start_tick + odp_timer_ns_to_tick(timer_pool, nsec);
+ start_param.tmo_ev = ctx->event;
+
+ ret = odp_timer_start(ctx->timer, &start_param);
if (ret != ODP_TIMER_SUCCESS) {
printf("Timer[%" PRIu64 "] set failed: %i\n",
@@ -647,6 +647,7 @@ static void run_test(test_global_t *test_global)
odp_timer_pool_t tp = test_global->timer_pool;
unsigned int retries = test_global->opt.early_retry;
uint64_t start_ns = test_global->start_ns;
+ odp_timer_start_t start_param;
tim = ctx->timer;
@@ -659,18 +660,20 @@ static void run_test(test_global_t *test_global)
nsec = ctx->nsec - start_ns;
tick = test_global->start_tick +
odp_timer_ns_to_tick(tp, nsec);
-
- ret = odp_timer_set_abs(tim, tick, &ev);
+ start_param.tick_type = ODP_TIMER_TICK_ABS;
} else {
/* Relative time */
tick = test_global->period_tick;
time = odp_time_local();
time_ns = odp_time_to_ns(time);
ctx->nsec = time_ns + period_ns;
-
- ret = odp_timer_set_rel(tim, tick, &ev);
+ start_param.tick_type = ODP_TIMER_TICK_REL;
}
+ start_param.tmo_ev = ev;
+ start_param.tick = tick;
+
+ ret = odp_timer_start(tim, &start_param);
if (ret == ODP_TIMER_TOO_NEAR)
stat->num_too_near++;
else
diff --git a/example/timer/odp_timer_simple.c b/example/timer/odp_timer_simple.c
index f44977476..4e3120653 100644
--- a/example/timer/odp_timer_simple.c
+++ b/example/timer/odp_timer_simple.c
@@ -129,13 +129,19 @@ int main(int argc ODP_UNUSED, char *argv[] ODP_UNUSED)
for (i = 0; i < 5; i++) {
odp_time_t time;
+ odp_timer_start_t start_param;
/* Program timeout action on current tick + period */
tick = odp_timer_current_tick(timer_pool);
- rc = odp_timer_set_abs(tim, tick + period, &ev);
+
+ start_param.tick_type = ODP_TIMER_TICK_ABS;
+ start_param.tick = tick + period;
+ start_param.tmo_ev = ev;
+
+ rc = odp_timer_start(tim, &start_param);
/* Too early or too late timeout requested */
if (odp_unlikely(rc != ODP_TIMER_SUCCESS))
- ODPH_ABORT("odp_timer_set_abs() failed: %d\n", rc);
+ ODPH_ABORT("odp_timer_start() failed: %d\n", rc);
/* Wait for 2 seconds for timeout action to be generated */
ev = odp_schedule(&queue, sched_tmo);
diff --git a/example/timer/odp_timer_test.c b/example/timer/odp_timer_test.c
index 277bbf6ae..f298c9fe8 100644
--- a/example/timer/odp_timer_test.c
+++ b/example/timer/odp_timer_test.c
@@ -121,14 +121,19 @@ static void test_abs_timeouts(int thr, test_globals_t *gbls)
int wait = 0;
odp_event_t ev;
odp_timer_set_t rc;
+ odp_timer_start_t start_param;
if (ttp) {
tick += period;
- rc = odp_timer_set_abs(ttp->tim, tick, &ttp->ev);
+
+ start_param.tick_type = ODP_TIMER_TICK_ABS;
+ start_param.tick = tick;
+ start_param.tmo_ev = ttp->ev;
+
+ rc = odp_timer_start(ttp->tim, &start_param);
if (odp_unlikely(rc != ODP_TIMER_SUCCESS)) {
/* Too early or too late timeout requested */
- ODPH_ABORT("odp_timer_set_abs() failed: %s\n",
- timerset2str(rc));
+ ODPH_ABORT("odp_timer_start() failed: %s\n", timerset2str(rc));
}
}
diff --git a/include/Makefile.am b/include/Makefile.am
index 49ccf552e..cc2f17ae2 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -13,6 +13,7 @@ odpapiinclude_HEADERS = \
odp/api/atomic.h \
odp/api/barrier.h \
odp/api/buffer.h \
+ odp/api/buffer_types.h \
odp/api/byteorder.h \
odp/api/chksum.h \
odp/api/classification.h \
@@ -20,6 +21,7 @@ odpapiinclude_HEADERS = \
odp/api/cpu.h \
odp/api/cpumask.h \
odp/api/crypto.h \
+ odp/api/crypto_types.h \
odp/api/debug.h \
odp/api/deprecated.h \
odp/api/dma.h \
@@ -77,6 +79,7 @@ odpapispecinclude_HEADERS = \
odp/api/spec/atomic.h \
odp/api/spec/barrier.h \
odp/api/spec/buffer.h \
+ odp/api/spec/buffer_types.h \
odp/api/spec/byteorder.h \
odp/api/spec/chksum.h \
odp/api/spec/classification.h \
@@ -84,6 +87,7 @@ odpapispecinclude_HEADERS = \
odp/api/spec/cpu.h \
odp/api/spec/cpumask.h \
odp/api/spec/crypto.h \
+ odp/api/spec/crypto_types.h \
odp/api/spec/debug.h \
odp/api/spec/dma.h \
odp/api/spec/dma_types.h \
@@ -144,12 +148,14 @@ odpapiabidefaultinclude_HEADERS = \
odp/api/abi-default/atomic.h \
odp/api/abi-default/barrier.h \
odp/api/abi-default/buffer.h \
+ odp/api/abi-default/buffer_types.h \
odp/api/abi-default/byteorder.h \
odp/api/abi-default/classification.h \
odp/api/abi-default/comp.h \
odp/api/abi-default/cpu.h \
odp/api/abi-default/cpumask.h \
odp/api/abi-default/crypto.h \
+ odp/api/abi-default/crypto_types.h \
odp/api/abi-default/debug.h \
odp/api/abi-default/dma_types.h \
odp/api/abi-default/errno.h \
@@ -202,12 +208,14 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/arm32-linux/odp/api/abi/atomic.h \
odp/arch/arm32-linux/odp/api/abi/barrier.h \
odp/arch/arm32-linux/odp/api/abi/buffer.h \
+ odp/arch/arm32-linux/odp/api/abi/buffer_types.h \
odp/arch/arm32-linux/odp/api/abi/byteorder.h \
odp/arch/arm32-linux/odp/api/abi/classification.h \
odp/arch/arm32-linux/odp/api/abi/comp.h \
odp/arch/arm32-linux/odp/api/abi/cpu.h \
odp/arch/arm32-linux/odp/api/abi/cpumask.h \
odp/arch/arm32-linux/odp/api/abi/crypto.h \
+ odp/arch/arm32-linux/odp/api/abi/crypto_types.h \
odp/arch/arm32-linux/odp/api/abi/debug.h \
odp/arch/arm32-linux/odp/api/abi/dma_types.h \
odp/arch/arm32-linux/odp/api/abi/errno.h \
@@ -256,12 +264,14 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/arm64-linux/odp/api/abi/atomic.h \
odp/arch/arm64-linux/odp/api/abi/barrier.h \
odp/arch/arm64-linux/odp/api/abi/buffer.h \
+ odp/arch/arm64-linux/odp/api/abi/buffer_types.h \
odp/arch/arm64-linux/odp/api/abi/byteorder.h \
odp/arch/arm64-linux/odp/api/abi/classification.h \
odp/arch/arm64-linux/odp/api/abi/comp.h \
odp/arch/arm64-linux/odp/api/abi/cpu.h \
odp/arch/arm64-linux/odp/api/abi/cpumask.h \
odp/arch/arm64-linux/odp/api/abi/crypto.h \
+ odp/arch/arm64-linux/odp/api/abi/crypto_types.h \
odp/arch/arm64-linux/odp/api/abi/debug.h \
odp/arch/arm64-linux/odp/api/abi/dma_types.h \
odp/arch/arm64-linux/odp/api/abi/errno.h \
@@ -310,12 +320,14 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/default-linux/odp/api/abi/atomic.h \
odp/arch/default-linux/odp/api/abi/barrier.h \
odp/arch/default-linux/odp/api/abi/buffer.h \
+ odp/arch/default-linux/odp/api/abi/buffer_types.h \
odp/arch/default-linux/odp/api/abi/byteorder.h \
odp/arch/default-linux/odp/api/abi/classification.h \
odp/arch/default-linux/odp/api/abi/comp.h \
odp/arch/default-linux/odp/api/abi/cpu.h \
odp/arch/default-linux/odp/api/abi/cpumask.h \
odp/arch/default-linux/odp/api/abi/crypto.h \
+ odp/arch/default-linux/odp/api/abi/crypto_types.h \
odp/arch/default-linux/odp/api/abi/debug.h \
odp/arch/default-linux/odp/api/abi/dma_types.h \
odp/arch/default-linux/odp/api/abi/errno.h \
@@ -364,12 +376,14 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/power64-linux/odp/api/abi/atomic.h \
odp/arch/power64-linux/odp/api/abi/barrier.h \
odp/arch/power64-linux/odp/api/abi/buffer.h \
+ odp/arch/power64-linux/odp/api/abi/buffer_types.h \
odp/arch/power64-linux/odp/api/abi/byteorder.h \
odp/arch/power64-linux/odp/api/abi/classification.h \
odp/arch/power64-linux/odp/api/abi/comp.h \
odp/arch/power64-linux/odp/api/abi/cpu.h \
odp/arch/power64-linux/odp/api/abi/cpumask.h \
odp/arch/power64-linux/odp/api/abi/crypto.h \
+ odp/arch/power64-linux/odp/api/abi/crypto_types.h \
odp/arch/power64-linux/odp/api/abi/debug.h \
odp/arch/power64-linux/odp/api/abi/dma_types.h \
odp/arch/power64-linux/odp/api/abi/errno.h \
@@ -418,12 +432,14 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/x86_32-linux/odp/api/abi/atomic.h \
odp/arch/x86_32-linux/odp/api/abi/barrier.h \
odp/arch/x86_32-linux/odp/api/abi/buffer.h \
+ odp/arch/x86_32-linux/odp/api/abi/buffer_types.h \
odp/arch/x86_32-linux/odp/api/abi/byteorder.h \
odp/arch/x86_32-linux/odp/api/abi/classification.h \
odp/arch/x86_32-linux/odp/api/abi/comp.h \
odp/arch/x86_32-linux/odp/api/abi/cpu.h \
odp/arch/x86_32-linux/odp/api/abi/cpumask.h \
odp/arch/x86_32-linux/odp/api/abi/crypto.h \
+ odp/arch/x86_32-linux/odp/api/abi/crypto_types.h \
odp/arch/x86_32-linux/odp/api/abi/debug.h \
odp/arch/x86_32-linux/odp/api/abi/dma_types.h \
odp/arch/x86_32-linux/odp/api/abi/errno.h \
@@ -472,12 +488,14 @@ odpapiabiarchinclude_HEADERS = \
odp/arch/x86_64-linux/odp/api/abi/atomic.h \
odp/arch/x86_64-linux/odp/api/abi/barrier.h \
odp/arch/x86_64-linux/odp/api/abi/buffer.h \
+ odp/arch/x86_64-linux/odp/api/abi/buffer_types.h \
odp/arch/x86_64-linux/odp/api/abi/byteorder.h \
odp/arch/x86_64-linux/odp/api/abi/classification.h \
odp/arch/x86_64-linux/odp/api/abi/comp.h \
odp/arch/x86_64-linux/odp/api/abi/cpu.h \
odp/arch/x86_64-linux/odp/api/abi/cpumask.h \
odp/arch/x86_64-linux/odp/api/abi/crypto.h \
+ odp/arch/x86_64-linux/odp/api/abi/crypto_types.h \
odp/arch/x86_64-linux/odp/api/abi/debug.h \
odp/arch/x86_64-linux/odp/api/abi/dma_types.h \
odp/arch/x86_64-linux/odp/api/abi/errno.h \
diff --git a/include/odp/api/abi-default/buffer.h b/include/odp/api/abi-default/buffer.h
index 2bc27bba7..59b81186b 100644
--- a/include/odp/api/abi-default/buffer.h
+++ b/include/odp/api/abi-default/buffer.h
@@ -11,20 +11,7 @@
extern "C" {
#endif
-/** @internal Dummy type for strong typing */
-typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_buffer_t;
-
-/** @ingroup odp_buffer
- * @{
- */
-
-typedef _odp_abi_buffer_t *odp_buffer_t;
-
-#define ODP_BUFFER_INVALID ((odp_buffer_t)0)
-
-/**
- * @}
- */
+/* Empty header required due to the inline functions */
#ifdef __cplusplus
}
diff --git a/include/odp/api/abi-default/buffer_types.h b/include/odp/api/abi-default/buffer_types.h
new file mode 100644
index 000000000..f01466151
--- /dev/null
+++ b/include/odp/api/abi-default/buffer_types.h
@@ -0,0 +1,34 @@
+/* Copyright (c) 2017-2018, Linaro Limited
+ * Copyright (c) 2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_ABI_BUFFER_TYPES_H_
+#define ODP_ABI_BUFFER_TYPES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** @internal Dummy type for strong typing */
+typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_buffer_t;
+
+/** @ingroup odp_buffer
+ * @{
+ */
+
+typedef _odp_abi_buffer_t *odp_buffer_t;
+
+#define ODP_BUFFER_INVALID ((odp_buffer_t)0)
+
+/**
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/include/odp/api/abi-default/crypto.h b/include/odp/api/abi-default/crypto.h
index e7c0a651c..0e9e8506a 100644
--- a/include/odp/api/abi-default/crypto.h
+++ b/include/odp/api/abi-default/crypto.h
@@ -1,4 +1,5 @@
/* Copyright (c) 2017-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -11,23 +12,7 @@
extern "C" {
#endif
-#include <stdint.h>
-
-/** @internal Dummy type for strong typing */
-typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_crypto_compl_t;
-
-/** @ingroup odp_crypto
- * @{
- */
-
-#define ODP_CRYPTO_SESSION_INVALID (0xffffffffffffffffULL)
-
-typedef uint64_t odp_crypto_session_t;
-typedef _odp_abi_crypto_compl_t *odp_crypto_compl_t;
-
-/**
- * @}
- */
+/* Empty header required due to the inline functions */
#ifdef __cplusplus
}
diff --git a/include/odp/api/abi-default/crypto_types.h b/include/odp/api/abi-default/crypto_types.h
new file mode 100644
index 000000000..cc9155bfa
--- /dev/null
+++ b/include/odp/api/abi-default/crypto_types.h
@@ -0,0 +1,37 @@
+/* Copyright (c) 2017-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_ABI_CRYPTO_TYPES_H_
+#define ODP_ABI_CRYPTO_TYPES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+/** @internal Dummy type for strong typing */
+typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_crypto_compl_t;
+
+/** @ingroup odp_crypto
+ * @{
+ */
+
+#define ODP_CRYPTO_SESSION_INVALID (0xffffffffffffffffULL)
+
+typedef uint64_t odp_crypto_session_t;
+typedef _odp_abi_crypto_compl_t *odp_crypto_compl_t;
+
+/**
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/include/odp/api/buffer_types.h b/include/odp/api/buffer_types.h
new file mode 100644
index 000000000..767aae560
--- /dev/null
+++ b/include/odp/api/buffer_types.h
@@ -0,0 +1,28 @@
+/* Copyright (c) 2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODP buffer types
+ */
+
+#ifndef ODP_API_BUFFER_TYPES_H_
+#define ODP_API_BUFFER_TYPES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp/api/abi/buffer_types.h>
+
+#include <odp/api/spec/buffer_types.h>
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/include/odp/api/crypto.h b/include/odp/api/crypto.h
index ec7c8370b..336304553 100644
--- a/include/odp/api/crypto.h
+++ b/include/odp/api/crypto.h
@@ -17,19 +17,7 @@
extern "C" {
#endif
-#include <odp/api/std_types.h>
-#include <odp/api/abi/packet_types.h>
#include <odp/api/abi/crypto.h>
-#include <odp/api/abi/buffer.h>
-#include <odp/api/abi/queue_types.h>
-
-/** @ingroup odp_crypto
- * @{
- */
-
-/**
- * @}
- */
#include <odp/api/spec/crypto.h>
diff --git a/include/odp/api/crypto_types.h b/include/odp/api/crypto_types.h
new file mode 100644
index 000000000..225ff7df0
--- /dev/null
+++ b/include/odp/api/crypto_types.h
@@ -0,0 +1,28 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODP crypto
+ */
+
+#ifndef ODP_API_CRYPTO_TYPES_H_
+#define ODP_API_CRYPTO_TYPES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp/api/abi/crypto_types.h>
+
+#include <odp/api/spec/crypto_types.h>
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/include/odp/api/spec/buffer.h b/include/odp/api/spec/buffer.h
index cfb85df17..dd3749afe 100644
--- a/include/odp/api/spec/buffer.h
+++ b/include/odp/api/spec/buffer.h
@@ -1,5 +1,5 @@
/* Copyright (c) 2013-2018, Linaro Limited
- * Copyright (c) 2022, Nokia
+ * Copyright (c) 2022-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -8,7 +8,7 @@
/**
* @file
*
- * ODP buffer descriptor
+ * ODP buffer
*/
#ifndef ODP_API_SPEC_BUFFER_H_
@@ -19,6 +19,7 @@
extern "C" {
#endif
+#include <odp/api/buffer_types.h>
#include <odp/api/event_types.h>
#include <odp/api/pool_types.h>
#include <odp/api/std_types.h>
@@ -29,16 +30,6 @@ extern "C" {
*/
/**
- * @typedef odp_buffer_t
- * ODP buffer
- */
-
-/**
- * @def ODP_BUFFER_INVALID
- * Invalid buffer
- */
-
-/**
* Get buffer handle from event
*
* Converts an ODP_EVENT_BUFFER type event to a buffer.
diff --git a/include/odp/api/spec/buffer_types.h b/include/odp/api/spec/buffer_types.h
new file mode 100644
index 000000000..7a7bc770f
--- /dev/null
+++ b/include/odp/api/spec/buffer_types.h
@@ -0,0 +1,45 @@
+/* Copyright (c) 2013-2018, Linaro Limited
+ * Copyright (c) 2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODP buffer types
+ */
+
+#ifndef ODP_API_SPEC_BUFFER_TYPES_H_
+#define ODP_API_SPEC_BUFFER_TYPES_H_
+#include <odp/visibility_begin.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** @addtogroup odp_buffer
+ * @{
+ */
+
+/**
+ * @typedef odp_buffer_t
+ * ODP buffer
+ */
+
+/**
+ * @def ODP_BUFFER_INVALID
+ * Invalid buffer
+ */
+
+/**
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <odp/visibility_end.h>
+#endif
diff --git a/include/odp/api/spec/crypto.h b/include/odp/api/spec/crypto.h
index a79a05ad3..4292b78c1 100644
--- a/include/odp/api/spec/crypto.h
+++ b/include/odp/api/spec/crypto.h
@@ -1,5 +1,5 @@
/* Copyright (c) 2014-2018, Linaro Limited
- * Copyright (c) 2021-2022, Nokia
+ * Copyright (c) 2021-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -15,9 +15,9 @@
#define ODP_API_SPEC_CRYPTO_H_
#include <odp/visibility_begin.h>
+#include <odp/api/crypto_types.h>
#include <odp/api/deprecated.h>
#include <odp/api/packet_types.h>
-#include <odp/api/pool_types.h>
#include <odp/api/std_types.h>
#ifdef __cplusplus
@@ -30,1062 +30,6 @@ extern "C" {
*/
/**
- * @def ODP_CRYPTO_SESSION_INVALID
- * Invalid session handle
- */
-
-/**
- * @typedef odp_crypto_session_t
- * Crypto API opaque session handle
- */
-
-/**
- * @typedef odp_crypto_compl_t
-* @deprecated Crypto API completion event (platform dependent).
-*/
-
-/**
- * Crypto API operation mode
- */
-typedef enum {
- /** Synchronous, return results immediately */
- ODP_CRYPTO_SYNC,
- /** Asynchronous, return results via posted event */
- ODP_CRYPTO_ASYNC,
-} odp_crypto_op_mode_t;
-
-/**
- * Crypto API operation type
- */
-typedef enum {
- /** Encrypt and/or compute authentication ICV */
- ODP_CRYPTO_OP_ENCODE,
- /** Decrypt and/or verify authentication ICV */
- ODP_CRYPTO_OP_DECODE,
-} odp_crypto_op_t;
-
-/**
- * Crypto API cipher algorithm
- */
-typedef enum {
- /** No cipher algorithm specified */
- ODP_CIPHER_ALG_NULL,
-
- /** DES */
- ODP_CIPHER_ALG_DES,
-
- /** Triple DES with cipher block chaining */
- ODP_CIPHER_ALG_3DES_CBC,
-
- /** Triple DES with Electronic Codebook */
- ODP_CIPHER_ALG_3DES_ECB,
-
- /** AES with cipher block chaining */
- ODP_CIPHER_ALG_AES_CBC,
-
- /** AES with counter mode */
- ODP_CIPHER_ALG_AES_CTR,
-
- /** AES with electronic codebook */
- ODP_CIPHER_ALG_AES_ECB,
-
- /** AES with 128-bit cipher feedback */
- ODP_CIPHER_ALG_AES_CFB128,
-
- /** AES with XEX-based tweaked-codebook mode with ciphertext stealing
- * (XTS) */
- ODP_CIPHER_ALG_AES_XTS,
-
- /** AES-GCM
- *
- * AES in Galois/Counter Mode (GCM) algorithm. GCM provides both
- * authentication and ciphering of data (authenticated encryption)
- * in the same operation. Hence this algorithm must be paired always
- * with ODP_AUTH_ALG_AES_GCM authentication.
- */
- ODP_CIPHER_ALG_AES_GCM,
-
- /** AES-CCM
- *
- * AES in Counter with CBC-MAC (CCM) mode algorithm. CCM provides both
- * authentication and ciphering of data (authenticated encryption)
- * in the same operation. Hence this algorithm must be paired always
- * with ODP_AUTH_ALG_AES_CCM authentication.
- */
- ODP_CIPHER_ALG_AES_CCM,
-
- /** ChaCha20-Poly1305
- *
- * ChaCha20 with Poly1305 provide both authentication and ciphering of
- * data (authenticated encryption) in the same operation. Hence this
- * algorithm must be paired always with ODP_AUTH_ALG_CHACHA20_POLY1305
- * authentication.
- */
- ODP_CIPHER_ALG_CHACHA20_POLY1305,
-
- /** Confidentiality F8 algorithm (UEA1)
- *
- * KASUMI-based F8 algorithm (also known as UEA1).
- *
- * IV should be formatted according to the 3GPP TS 35.201:
- * COUNT || BEARER || DIRECTION || 0...0
- */
- ODP_CIPHER_ALG_KASUMI_F8,
-
- /** Confidentiality UEA2 algorithm (128-EEA1)
- *
- * SNOW 3G-based UEA2 algorithm (also known as 128-EEA1).
- *
- * IV (128 bit) should be formatted according to the ETSI/SAGE
- * UEA2 & UIA2 specification:
- * COUNT || BEARER || DIRECTION || 0...0 ||
- * COUNT || BEARER || DIRECTION || 0...0 ||
- */
- ODP_CIPHER_ALG_SNOW3G_UEA2,
-
- /** Confidentiality 128-EEA2 algorithm
- *
- * AES-CTR-based 128-EEA2 algorithm.
- *
- * IV (128 bit) should be formatted according to the ETSI/SAGE
- * 128-EA2 & 128-EIA2 specification:
- * COUNT || BEARER ||
- * DIRECTION || 0....0
- */
- ODP_CIPHER_ALG_AES_EEA2,
-
- /** ZUC based confidentiality algorithm
- *
- * 128-EEA3/128-NEA3 algorithm when key length is 128 bits.
- *
- * IV (128 bit) should be formatted according to the ETSI/SAGE
- * 128-EEA3 & 128-EIA3 specification:
- * COUNT || BEARER || DIRECTION || 0...0 ||
- * COUNT || BEARER || DIRECTION || 0...0 ||
- *
- * 256-bit key length support is experimental and subject to
- * change. The following variants may be supported:
- *
- * - ZUC-256 with 25 byte IV (of which 184 bits are variable)
- * as specified in "The ZUC-256 Stream Cipher".
- * - ZUC-256 with 16 byte IV as specified in
- * "An Addendum to the ZUC-256 Stream Cipher",
- * https://eprint.iacr.org/2021/1439
- */
- ODP_CIPHER_ALG_ZUC_EEA3,
-
-} odp_cipher_alg_t;
-
-/**
- * Crypto API authentication algorithm
- */
-typedef enum {
- /** No authentication algorithm specified */
- ODP_AUTH_ALG_NULL,
-
- /** HMAC-MD5
- *
- * MD5 algorithm in HMAC mode
- */
- ODP_AUTH_ALG_MD5_HMAC,
-
- /** HMAC-SHA-1
- *
- * SHA-1 algorithm in HMAC mode
- */
- ODP_AUTH_ALG_SHA1_HMAC,
-
- /** HMAC-SHA-224
- *
- * SHA-224 algorithm in HMAC mode
- */
- ODP_AUTH_ALG_SHA224_HMAC,
-
- /** HMAC-SHA-256
- *
- * SHA-256 algorithm in HMAC mode
- */
- ODP_AUTH_ALG_SHA256_HMAC,
-
- /** HMAC-SHA-384
- *
- * SHA-384 algorithm in HMAC mode
- */
- ODP_AUTH_ALG_SHA384_HMAC,
-
- /** HMAC-SHA-512
- *
- * SHA-512 algorithm in HMAC mode
- */
- ODP_AUTH_ALG_SHA512_HMAC,
-
- /** AES-GCM
- *
- * AES in Galois/Counter Mode (GCM) algorithm. GCM provides both
- * authentication and ciphering of data (authenticated encryption)
- * in the same operation. Hence this algorithm must be paired always
- * with ODP_CIPHER_ALG_AES_GCM cipher.
- */
- ODP_AUTH_ALG_AES_GCM,
-
- /** AES-GMAC
- *
- * AES Galois Message Authentication Code (GMAC) algorithm. AES-GMAC
- * is based on AES-GCM operation, but provides authentication only.
- * Hence this algorithm can be paired only with ODP_CIPHER_ALG_NULL
- * cipher.
- *
- * NIST and RFC specifications of GMAC refer to all data to be
- * authenticated as AAD. In ODP the data to be authenticated, i.e.
- * AAD, is ODP packet data and specified using the auth_range
- * parameter. The aad_length and aad_ptr parameters, which would
- * require the data to be contiguous in memory, are ignored with
- * AES-GMAC.
- *
- * GMAC needs an initialization vector, which can be passed via
- * session (auth_iv) or packet (auth_iv_ptr) level parameters.
- */
- ODP_AUTH_ALG_AES_GMAC,
-
- /** AES-CCM
- *
- * AES in Counter with CBC-MAC (CCM) mode algorithm. CCM provides both
- * authentication and ciphering of data (authenticated encryption)
- * in the same operation. Hence this algorithm must be paired always
- * with ODP_CIPHER_ALG_AES_CCM cipher.
- */
- ODP_AUTH_ALG_AES_CCM,
-
- /** AES-CMAC
- *
- * AES Cipher-based Message Authentication Code (CMAC) algorithm. CMAC
- * is a keyed hash function that is based on a symmetric key block
- * cipher, such as the AES.
- */
- ODP_AUTH_ALG_AES_CMAC,
-
- /** AES-XCBC-MAC
- *
- * AES CBC MAC for arbitrary-length messages (XCBC-MAC).
- *
- */
- ODP_AUTH_ALG_AES_XCBC_MAC,
-
- /** ChaCha20-Poly1305 AEAD
- *
- * ChaCha20 with Poly1305 provide both authentication and ciphering of
- * data (authenticated encryption) in the same operation. Hence this
- * algorithm must be paired always with
- * ODP_CIPHER_ALG_CHACHA20_POLY1305 cipher.
- */
- ODP_AUTH_ALG_CHACHA20_POLY1305,
-
- /** Integrity F9 algorithm (UIA1)
- *
- * KASUMI-based F9 algorithm (also known as UIA1).
- *
- * IV (9 bytes) is a concatenation of COUNT (32b), FRESH (32b) and
- * DIRECTION (LSB-aligned, 1b).
- * IV (8 bytes) is a concatenation of COUNT (32b) and FRESH (32b)
- * DIRECTION (1b) and padding should come at the end of message.
- */
- ODP_AUTH_ALG_KASUMI_F9,
-
- /** Integrity UIA2 algorithm (128-EIA1)
- *
- * SNOW 3G-based UIA2 algorithm (also known as 128-EIA1).
- * IV (128 bit) should be formatted according to the ETSI/SAGE
- * UEA2 & UIA2 specification:
- * COUNT || FRESH ||
- * DIRECTION XOR COUNT0 || COUNT1 .. COUNT31 ||
- * FRESH0 .. FRESH15 || FRESH16 XOR DIRECTION || FRESH17 .. FRESH31
- */
- ODP_AUTH_ALG_SNOW3G_UIA2,
-
- /** Integrity 128-EIA2 algorithm
- *
- * AES_CMAC-based 128-EIA2 algorithm.
- *
- * IV (128 bit) should be formatted according to the ETSI/SAGE
- * 128-EA2 & 128-EIA2 specification:
- * COUNT || BEARER ||
- * DIRECTION || 0....0
- */
- ODP_AUTH_ALG_AES_EIA2,
-
- /** ZUC-based integrity algorithm.
- *
- * 128-EIA3/128-NIA3 algorithm when key length is 128 bits.
- *
- * IV (128 bit) should be formatted according to the ETSI/SAGE
- * 128-EA3 & 128-EIA2 specification:
- * COUNT || BEARER ||
- * DIRECTION XOR COUNT0 || COUNT1 .. COUNT31 ||
- * BEARER || 0...0 || DIRECTION || 0...0
- *
- * 256-bit key length support is experimental and subject to
- * change. The following variants may be supported:
- *
- * - ZUC-256 with 25 byte IV (of which 184 bits are variable) and
- * 32/64/128 bit MAC as specified in "The ZUC-256 Stream Cipher".
- * - ZUC-256 with 16 byte IV and 32/64/128 bit MAC as specified in
- * "An Addendum to the ZUC-256 Stream Cipher",
- * https://eprint.iacr.org/2021/1439
- */
- ODP_AUTH_ALG_ZUC_EIA3,
-
- /** MD5 algorithm */
- ODP_AUTH_ALG_MD5,
-
- /** SHA1 algorithm */
- ODP_AUTH_ALG_SHA1,
-
- /** 224 bit SHA2 algorithm */
- ODP_AUTH_ALG_SHA224,
-
- /** 256 bit SHA2 algorithm */
- ODP_AUTH_ALG_SHA256,
-
- /** 384 bit SHA2 algorithm */
- ODP_AUTH_ALG_SHA384,
-
- /** 512 bit SHA2 algorithm */
- ODP_AUTH_ALG_SHA512,
-
-} odp_auth_alg_t;
-
-/**
- * Cipher algorithms in a bit field structure
- */
-typedef union odp_crypto_cipher_algos_t {
- /** Cipher algorithms */
- struct {
- /** ODP_CIPHER_ALG_NULL */
- uint32_t null : 1;
-
- /** ODP_CIPHER_ALG_DES */
- uint32_t des : 1;
-
- /** ODP_CIPHER_ALG_3DES_CBC */
- uint32_t trides_cbc : 1;
-
- /** ODP_CIPHER_ALG_3DES_ECB */
- uint32_t trides_ecb : 1;
-
- /** ODP_CIPHER_ALG_AES_CBC */
- uint32_t aes_cbc : 1;
-
- /** ODP_CIPHER_ALG_AES_CTR */
- uint32_t aes_ctr : 1;
-
- /** ODP_CIPHER_ALG_AES_ECB */
- uint32_t aes_ecb : 1;
-
- /** ODP_CIPHER_ALG_AES_CFB128 */
- uint32_t aes_cfb128 : 1;
-
- /** ODP_CIPHER_ALG_AES_XTS */
- uint32_t aes_xts : 1;
-
- /** ODP_CIPHER_ALG_AES_GCM */
- uint32_t aes_gcm : 1;
-
- /** ODP_CIPHER_ALG_AES_CCM */
- uint32_t aes_ccm : 1;
-
- /** ODP_CIPHER_ALG_CHACHA20_POLY1305 */
- uint32_t chacha20_poly1305 : 1;
-
- /** ODP_CIPHER_ALG_KASUMI_F8 */
- uint32_t kasumi_f8 : 1;
-
- /** ODP_CIPHER_ALG_SNOW3G_UEA2 */
- uint32_t snow3g_uea2 : 1;
-
- /** ODP_CIPHER_ALG_AES_EEA2 */
- uint32_t aes_eea2 : 1;
-
- /** ODP_CIPHER_ALG_ZUC_EEA3 */
- uint32_t zuc_eea3 : 1;
-
- } bit;
-
- /** All bits of the bit field structure
- *
- * This field can be used to set/clear all flags, or bitwise
- * operations over the entire structure. */
- uint32_t all_bits;
-} odp_crypto_cipher_algos_t;
-
-/**
- * Authentication algorithms in a bit field structure
- */
-typedef union odp_crypto_auth_algos_t {
- /** Authentication algorithms */
- struct {
- /** ODP_AUTH_ALG_NULL */
- uint32_t null : 1;
-
- /** ODP_AUTH_ALG_MD5_HMAC */
- uint32_t md5_hmac : 1;
-
- /** ODP_AUTH_ALG_SHA1_HMAC */
- uint32_t sha1_hmac : 1;
-
- /** ODP_AUTH_ALG_SHA224_HMAC */
- uint32_t sha224_hmac : 1;
-
- /** ODP_AUTH_ALG_SHA256_HMAC */
- uint32_t sha256_hmac : 1;
-
- /** ODP_AUTH_ALG_SHA384_HMAC */
- uint32_t sha384_hmac : 1;
-
- /** ODP_AUTH_ALG_SHA512_HMAC */
- uint32_t sha512_hmac : 1;
-
- /** ODP_AUTH_ALG_AES_GCM */
- uint32_t aes_gcm : 1;
-
- /** ODP_AUTH_ALG_AES_GMAC*/
- uint32_t aes_gmac : 1;
-
- /** ODP_AUTH_ALG_AES_CCM */
- uint32_t aes_ccm : 1;
-
- /** ODP_AUTH_ALG_AES_CMAC*/
- uint32_t aes_cmac : 1;
-
- /** ODP_AUTH_ALG_AES_XCBC_MAC*/
- uint32_t aes_xcbc_mac : 1;
-
- /** ODP_AUTH_ALG_CHACHA20_POLY1305 */
- uint32_t chacha20_poly1305 : 1;
-
- /** ODP_AUTH_ALG_KASUMI_F9 */
- uint32_t kasumi_f9 : 1;
-
- /** ODP_AUTH_ALG_SNOW3G_UIA2 */
- uint32_t snow3g_uia2 : 1;
-
- /** ODP_AUTH_ALG_AES_EIA2 */
- uint32_t aes_eia2 : 1;
-
- /** ODP_AUTH_ALG_ZUC_EIA3 */
- uint32_t zuc_eia3 : 1;
-
- /** ODP_AUTH_ALG_MD5 */
- uint32_t md5 : 1;
-
- /** ODP_AUTH_ALG_SHA1 */
- uint32_t sha1 : 1;
-
- /** ODP_AUTH_ALG_SHA224 */
- uint32_t sha224 : 1;
-
- /** ODP_AUTH_ALG_SHA256 */
- uint32_t sha256 : 1;
-
- /** ODP_AUTH_ALG_SHA384 */
- uint32_t sha384 : 1;
-
- /** ODP_AUTH_ALG_SHA512 */
- uint32_t sha512 : 1;
-
- } bit;
-
- /** All bits of the bit field structure
- *
- * This field can be used to set/clear all flags, or bitwise
- * operations over the entire structure. */
- uint32_t all_bits;
-} odp_crypto_auth_algos_t;
-
-/**
- * Crypto API key structure
- */
-typedef struct odp_crypto_key {
- /** Key data */
- uint8_t *data;
-
- /** Key length in bytes */
- uint32_t length;
-
-} odp_crypto_key_t;
-
-/**
- * Crypto API IV structure
- *
- * @deprecated Use per-packet IV in crypto operation parameters
- */
-typedef struct odp_crypto_iv {
- /** IV data
- *
- * Ignored when length is zero. Null value indicates that an
- * IV will be provided for each packet through the crypto
- * operation parameters. In that case the per-operation
- * IV parameter must always point to a valid IV.
- *
- * Default value is NULL.
- */
- uint8_t *data;
-
- /** IV length in bytes. Default value is zero. */
- uint32_t length;
-
-} ODP_DEPRECATE(odp_crypto_iv_t);
-
-/**
- * Crypto API session creation parameters
- */
-typedef struct odp_crypto_session_param_t {
- /** Encode vs. decode operation
- *
- * The default value is ODP_CRYPTO_OP_ENCODE.
- */
- odp_crypto_op_t op;
-
- /** Authenticate cipher vs. plain text
- *
- * Controls ordering of authentication and cipher operations,
- * and is relative to the operation (encode vs decode). When encoding,
- * TRUE indicates the authentication operation should be performed
- * after the cipher operation else before. When decoding, TRUE
- * indicates the reverse order of operation.
- *
- * The value is ignored with authenticated encryption algorithms
- * such as AES-GCM.
- *
- * true: Authenticate cipher text
- * false: Authenticate plain text
- *
- * The default value is false.
- */
- odp_bool_t auth_cipher_text;
-
- /** Hash result location may overlap authentication range
- *
- * This flag indicates that the hash result location may (but is
- * not required to) overlap authentication range. Setting this
- * flag may reduce performance.
- *
- * Default value is false.
- */
- odp_bool_t hash_result_in_auth_range;
-
- /** Preferred sync vs. async for odp_crypto_operation()
- *
- * The default value is ODP_CRYPTO_SYNC.
- *
- * @deprecated Used only with deprecated odp_crypto_operation()
- */
- odp_crypto_op_mode_t ODP_DEPRECATE(pref_mode);
-
- /** Operation mode when using packet interface: sync or async
- *
- * The default value is ODP_CRYPTO_SYNC.
- */
- odp_crypto_op_mode_t op_mode;
-
- /** Cipher algorithm
- *
- * Select cipher algorithm to be used. ODP_CIPHER_ALG_NULL indicates
- * that ciphering is disabled. Use odp_crypto_capability() for
- * supported algorithms. Note that some algorithms restrict choice of
- * the pairing authentication algorithm. When ciphering is enabled
- * cipher key and IV need to be set. The default value is
- * ODP_CIPHER_ALG_NULL.
- */
- odp_cipher_alg_t cipher_alg;
-
- /** Cipher key
- *
- * Use odp_crypto_cipher_capa() for supported key and IV lengths.
- */
- odp_crypto_key_t cipher_key;
-
- /** Cipher Initialization Vector (IV)
- *
- * Unless using the deprecated API, this specifies the length of
- * the IV only. The actual IV must then be provided in per-packet
- * parameters of crypto operations.
- */
- union {
-#if ODP_DEPRECATED_API
- /** @deprecated Cipher IV */
- odp_crypto_iv_t ODP_DEPRECATE(cipher_iv);
-#endif
- /** Cipher IV length */
- struct {
-#if ODP_DEPRECATED_API
- /** @cond
- * Unused padding field
- */
- uint8_t *dummy_padding_0;
- /** @endcond */
-#endif
- /** Length of cipher initialization vector.
- * Default value is zero.
- */
- uint32_t cipher_iv_len;
- };
- };
-
- /** Authentication algorithm
- *
- * Select authentication algorithm to be used. ODP_AUTH_ALG_NULL
- * indicates that authentication is disabled. Use
- * odp_crypto_capability() for supported algorithms. Note that some
- * algorithms restrict choice of the pairing cipher algorithm. When
- * single algorithm provides both ciphering and authentication
- * (i.e. Authenticated Encryption), authentication side key
- * (auth_key) and IV (auth_iv) are ignored, and cipher side values are
- * used instead. These algorithms ignore authentication side key
- * and IV: ODP_AUTH_ALG_AES_GCM, ODP_AUTH_ALG_AES_CCM and
- * ODP_AUTH_ALG_CHACHA20_POLY1305. Otherwise, all authentication side
- * parameters must be set when authentication is enabled. The default
- * value is ODP_AUTH_ALG_NULL.
- */
- odp_auth_alg_t auth_alg;
-
- /** Authentication key
- *
- * Use odp_crypto_auth_capability() for supported key lengths.
- */
- odp_crypto_key_t auth_key;
-
- /** Authentication Initialization Vector (IV)
- *
- * Unless using the deprecated API, this specifies the length of
- * the IV only. The actual IV must then be provided in per-packet
- * parameters of crypto operations.
- */
- union {
-#if ODP_DEPRECATED_API
- /** @deprecated Authentication IV */
- odp_crypto_iv_t ODP_DEPRECATE(auth_iv);
-#endif
- /** Authentication IV length */
- struct {
-#if ODP_DEPRECATED_API
- /** @cond
- * Unused padding field
- */
- uint8_t *dummy_padding_1;
- /** @endcond */
-#endif
- /** Length of authentication initialization vector.
- * Default value is zero.
- */
- uint32_t auth_iv_len;
- };
- };
-
- /** Authentication digest length in bytes
- *
- * Use odp_crypto_auth_capability() for supported digest lengths.
- */
- uint32_t auth_digest_len;
-
- /** Additional Authenticated Data (AAD) length in bytes
- *
- * AAD length is constant for all operations (packets) of the session.
- * Set to zero when AAD is not used. Use odp_crypto_auth_capability()
- * for supported AAD lengths. The default value is zero.
- */
- uint32_t auth_aad_len;
-
- /** Async mode completion event queue
- *
- * The completion queue is used to return completions from
- * odp_crypto_op_enq() (and the deprecated odp_crypto_operation())
- * to the application.
- */
- odp_queue_t compl_queue;
-
- /** Output pool
- *
- * When the output packet is not specified during the call to
- * crypto operation, the output packet will be allocated
- * from this pool.
- */
- odp_pool_t output_pool;
-
-} odp_crypto_session_param_t;
-
-/**
- * Crypto API per packet operation parameters
- *
- * @deprecated Use odp_crypto_packet_op_param_t instead.
- */
-typedef struct odp_crypto_op_param_t {
- /** Session handle from creation */
- odp_crypto_session_t session;
-
- /** User context */
- void *ctx;
-
- /** Input packet
- *
- * Specifies the input packet for the crypto operation. When the
- * 'out_pkt' variable is set to ODP_PACKET_INVALID (indicating a new
- * packet should be allocated for the resulting packet).
- */
- odp_packet_t pkt;
-
- /** Output packet
- *
- * Both "in place" (the original packet 'pkt' is modified) and
- * "copy" (the packet is replicated to a new packet which contains
- * the modified data) modes are supported. The "in place" mode of
- * operation is indicated by setting 'out_pkt' equal to 'pkt'.
- * For the copy mode of operation, setting 'out_pkt' to a valid packet
- * value indicates the caller wishes to specify the destination packet.
- * Setting 'out_pkt' to ODP_PACKET_INVALID indicates the caller wishes
- * the destination packet be allocated from the output pool specified
- * during session creation.
- */
- odp_packet_t out_pkt;
-
- /** IV pointer for cipher */
- uint8_t *cipher_iv_ptr;
-
- /** Authentication IV pointer */
- uint8_t *auth_iv_ptr;
-
- /** Offset from start of packet for hash result
- *
- * In case of decode sessions, the expected hash will be read from
- * this offset and compared with the calculated hash. After the
- * operation the hash bytes will have undefined values.
- *
- * In case of encode sessions the calculated hash will be stored in
- * this offset.
- *
- * If the hash_result_in_auth_range session parameter is true,
- * the hash result location may overlap auth_range. In that case
- * the result location will be zeroed in decode sessions before
- * hash calculation. Zeroing is not done in encode sessions.
- */
- uint32_t hash_result_offset;
-
- /** Pointer to AAD. AAD length is defined by 'auth_aad_len'
- * session parameter.
- */
- uint8_t *aad_ptr;
-
- /** Data range to be ciphered */
- odp_packet_data_range_t cipher_range;
-
- /** Data range to be authenticated
- *
- * The value is ignored with authenticated encryption algorithms,
- * such as AES-GCM, which authenticate data in the cipher range
- * and the AAD.
- *
- * As a special case AES-GMAC uses this field instead of aad_ptr
- * for the data bytes to be authenticated.
- */
- odp_packet_data_range_t auth_range;
-
-} ODP_DEPRECATE(odp_crypto_op_param_t);
-
-/**
- * Crypto packet API per packet operation parameters
- */
-typedef struct odp_crypto_packet_op_param_t {
- /** Session handle from creation */
- odp_crypto_session_t session;
-
- /** IV pointer for cipher */
- uint8_t *cipher_iv_ptr;
-
- /** IV pointer for authentication */
- uint8_t *auth_iv_ptr;
-
- /** Offset from start of packet for hash result
- *
- * In case of decode sessions, the expected hash will be read from
- * this offset and compared with the calculated hash. After the
- * operation the hash bytes will have undefined values.
- *
- * In case of encode sessions the calculated hash will be stored in
- * this offset.
- *
- * If the hash_result_in_auth_range session parameter is true,
- * the hash result location may overlap auth_range. In that case the
- * result location will be zeroed in decode sessions before hash
- * calculation. Zeroing is not done in encode sessions.
- */
- uint32_t hash_result_offset;
-
- /** Pointer to AAD. AAD length is defined by 'auth_aad_len'
- * session parameter.
- */
- uint8_t *aad_ptr;
-
- /** Data range to apply cipher */
- odp_packet_data_range_t cipher_range;
-
- /** Data range to authenticate */
- odp_packet_data_range_t auth_range;
-
-} odp_crypto_packet_op_param_t;
-
-/**
- * Crypto API session creation return code
- */
-typedef enum {
- /** Session created */
- ODP_CRYPTO_SES_ERR_NONE,
- /** Creation failed, no resources */
- ODP_CRYPTO_SES_ERR_ENOMEM,
- /** Creation failed, bad cipher params */
- ODP_CRYPTO_SES_ERR_CIPHER,
- /** Creation failed, bad auth params */
- ODP_CRYPTO_SES_ERR_AUTH,
-
- /** Unsupported combination of algorithms
- *
- * The combination of cipher and auth algorithms with their
- * specific parameters is not supported even if the algorithms
- * appear in capabilities and are supported in combination with
- * other algorithms or other algorithm specific parameters.
- */
- ODP_CRYPTO_SES_ERR_ALG_COMBO,
-
- /** Unsupported order of cipher and auth
- *
- * The requested mutual order of ciphering and authentication
- * is not supported with the chosen individual cipher and
- * authentication algorithms.
- */
- ODP_CRYPTO_SES_ERR_ALG_ORDER,
-
- /** Unsupported combination of session creation parameters
- *
- * The combination of provided session creation parameters is not
- * supported. This error can occur when there are limitations that
- * are not expressible through crypto capabilities or other error
- * status values.
- */
- ODP_CRYPTO_SES_ERR_PARAMS,
-} odp_crypto_ses_create_err_t;
-
-/** This synonym for backward compatibility will be deprecated later */
-#define ODP_CRYPTO_SES_CREATE_ERR_NONE ODP_CRYPTO_SES_ERR_NONE
-/** This synonym for backward compatibility will be deprecated later */
-#define ODP_CRYPTO_SES_CREATE_ERR_ENOMEM ODP_CRYPTO_SES_ERR_ENOMEM
-/** This synonym for backward compatibility will be deprecated later */
-#define ODP_CRYPTO_SES_CREATE_ERR_INV_CIPHER ODP_CRYPTO_SES_ERR_CIPHER
-/** This synonym for backward compatibility will be deprecated later */
-#define ODP_CRYPTO_SES_CREATE_ERR_INV_AUTH ODP_CRYPTO_SES_ERR_AUTH
-
-/**
- * Crypto API algorithm return code
- */
-typedef enum {
- /** Algorithm successful */
- ODP_CRYPTO_ALG_ERR_NONE,
- /** Invalid data block size */
- ODP_CRYPTO_ALG_ERR_DATA_SIZE,
- /** Key size invalid for algorithm */
- ODP_CRYPTO_ALG_ERR_KEY_SIZE,
- /** Computed ICV value mismatch */
- ODP_CRYPTO_ALG_ERR_ICV_CHECK,
- /** IV value not specified */
- ODP_CRYPTO_ALG_ERR_IV_INVALID,
-} odp_crypto_alg_err_t;
-
-/**
- * Crypto API hardware centric return code
- */
-typedef enum {
- /** Operation completed successfully */
- ODP_CRYPTO_HW_ERR_NONE,
- /** Error detected during DMA of data */
- ODP_CRYPTO_HW_ERR_DMA,
- /** Operation failed due to pool depletion */
- ODP_CRYPTO_HW_ERR_BP_DEPLETED,
-} odp_crypto_hw_err_t;
-
-/**
- * Cryto API per packet operation completion status
- */
-typedef struct odp_crypto_op_status {
- /** Algorithm specific return code */
- odp_crypto_alg_err_t alg_err;
-
- /** Hardware specific return code */
- odp_crypto_hw_err_t hw_err;
-
-} odp_crypto_op_status_t;
-
-/**
- * Crypto API operation result
- *
- * @deprecated Use odp_crypto_packet_result_t instead.
- */
-typedef struct odp_crypto_op_result {
- /** Request completed successfully */
- odp_bool_t ok;
-
- /** User context from request */
- void *ctx;
-
- /** Output packet */
- odp_packet_t pkt;
-
- /** Cipher status */
- odp_crypto_op_status_t cipher_status;
-
- /** Authentication status */
- odp_crypto_op_status_t auth_status;
-
-} ODP_DEPRECATE(odp_crypto_op_result_t);
-
-/**
- * Crypto packet API operation result
- */
-typedef struct odp_crypto_packet_result_t {
- /** Request completed successfully */
- odp_bool_t ok;
-
- /** Cipher status */
- odp_crypto_op_status_t cipher_status;
-
- /** Authentication status */
- odp_crypto_op_status_t auth_status;
-
-} odp_crypto_packet_result_t;
-
-/**
- * Crypto capabilities
- */
-typedef struct odp_crypto_capability_t {
- /** Maximum number of crypto sessions */
- uint32_t max_sessions;
-
- /** Supported packet operation in SYNC mode */
- odp_support_t sync_mode;
-
- /** Supported packet operation in ASYNC mode */
- odp_support_t async_mode;
-
- /** Supported cipher algorithms */
- odp_crypto_cipher_algos_t ciphers;
-
- /** Cipher algorithms implemented with HW offload */
- odp_crypto_cipher_algos_t hw_ciphers;
-
- /** Supported authentication algorithms */
- odp_crypto_auth_algos_t auths;
-
- /** Authentication algorithms implemented with HW offload */
- odp_crypto_auth_algos_t hw_auths;
-
- /**
- * Scheduled crypto completion queue support
- *
- * This defines whether scheduled queues are supported as crypto
- * compl_queue.
- * 0: Scheduled queues are not supported as crypto completion queues
- * 1: Scheduled queues are supported as crypto completion queues
- * @see odp_crypto_session_param_t
- */
- odp_bool_t queue_type_sched;
-
- /**
- * Plain crypto completion queue support
- *
- * This defines whether plain queues are supported as crypto
- * compl_queue.
- * 0: Plain queues are not supported as crypto completion queues
- * 1: Plain queues are supported as crypto completion queues
- * @see odp_crypto_session_param_t
- */
- odp_bool_t queue_type_plain;
-} odp_crypto_capability_t;
-
-/**
- * Cipher algorithm capabilities
- */
-typedef struct odp_crypto_cipher_capability_t {
- /** Key length in bytes */
- uint32_t key_len;
-
- /** IV length in bytes */
- uint32_t iv_len;
-
- /** Cipher is operating in bitwise mode
- *
- * This cipher works on series of bits, rather than sequences of bytes:
- * cipher_range in odp_crypto_op_param_t and
- * odp_crypto_packet_op_param_t will use bits, rather than bytes.
- *
- * Note: data buffer MUST start on the byte boundary, using offset
- * which is not divisible by 8 is unsupported and will result in
- * unspecified behaviour.
- *
- * Note2: If the data length is not a multiple of 8, the remaining
- * bits of the data in the last byte of the input/output will be the
- * most significant bits, i.e. the most significant bit is considered
- * to be the first bit of a byte for the purpose of input and output
- * data range. The output bits that fall out of the output range are
- * undefined.
- */
- odp_bool_t bit_mode;
-
-} odp_crypto_cipher_capability_t;
-
-/**
- * Authentication algorithm capabilities
- */
-typedef struct odp_crypto_auth_capability_t {
- /** Digest length in bytes */
- uint32_t digest_len;
-
- /** Key length in bytes */
- uint32_t key_len;
-
- /** IV length in bytes */
- uint32_t iv_len;
-
- /** Additional Authenticated Data (AAD) lengths */
- struct {
- /** Minimum AAD length in bytes */
- uint32_t min;
-
- /** Maximum AAD length in bytes */
- uint32_t max;
-
- /** Increment of supported lengths between min and max
- * (in bytes) */
- uint32_t inc;
- } aad_len;
-
- /** Auth is operating in bitstring mode
- *
- * This auth works on series of bits, rather than sequences of bytes:
- * auth_range in odp_crypto_op_param_t and
- * odp_crypto_packet_op_param_t will use bits, rather than bytes.
- *
- * Note: data buffer MUST start on the byte boundary, using offset
- * which is not divisible by 8 is unsupported and will result in
- * unpredictable behaviour.
- *
- * Note2: If the data length is not a multiple of 8, the remaining
- * bits of the data in the last byte of the input/output will be the
- * most significant bits, i.e. the most significant bit is considered
- * to be the first bit of a byte for the purpose of input and output
- * data range. The output bits that fall out of the output range are
- * undefined.
- */
- odp_bool_t bit_mode;
-
-} odp_crypto_auth_capability_t;
-
-/**
* Query crypto capabilities
*
* Outputs crypto capabilities on success.
@@ -1212,6 +156,9 @@ void odp_crypto_compl_free(odp_crypto_compl_t completion_event);
*
* @deprecated Use odp_crypto_op() or odp_crypto_op_enq() instead.
*
+ * This function may be called only for sessions configured with
+ * the ODP_CRYPTO_OP_TYPE_LEGACY operation type.
+ *
* Performs the cryptographic operations specified during session creation
* on the packet. If the operation is performed synchronously, "posted"
* will return FALSE and the result of the operation is immediately available.
@@ -1310,15 +257,15 @@ odp_packet_t odp_crypto_packet_from_event(odp_event_t ev);
odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt);
/**
- * Get crypto operation results from an crypto processed packet
+ * Get crypto operation results from a crypto processed packet
*
* Successful crypto operations of all types (SYNC and ASYNC) produce packets
* which contain crypto result metadata. This function copies the operation
- * results from an crypto processed packet. Event subtype of this kind of
+ * results from a crypto processed packet. Event subtype of this kind of
* packet is ODP_EVENT_PACKET_CRYPTO. Results are undefined if a non-crypto
* processed packet is passed as input.
*
- * @param packet An crypto processed packet (ODP_EVENT_PACKET_CRYPTO)
+ * @param packet A crypto processed packet (ODP_EVENT_PACKET_CRYPTO)
* @param[out] result Pointer to operation result for output
*
* @retval 0 On success
@@ -1331,16 +278,77 @@ int odp_crypto_result(odp_crypto_packet_result_t *result,
* Crypto packet operation
*
* Performs the SYNC cryptographic operations specified during session creation
- * on the packets. Caller should initialize pkt_out either with desired output
- * packet handles or with ODP_PACKET_INVALID to make ODP allocate new packets
- * from provided pool. All arrays should be of num_pkt size.
+ * on the packets. All arrays should be of num_pkt size.
+ *
+ * Use of the pkt_out parameter depends on the configured crypto operation
+ * type as described below.
+ *
+ * ODP_CRYPTO_OP_TYPE_LEGACY:
+ *
+ * Caller should initialize each element of pkt_out either with the desired
+ * output packet handle or with ODP_PACKET_INVALID to make ODP allocate a new
+ * packet from provided pool.
+ *
+ * All packet data and metadata are copied from the input packet to the output
+ * packet before the requested crypto operation is performed to the output
+ * packet. If an output packet is given to the operation, it must be at least
+ * as long as the input packet and, in encode operations, long enough for the
+ * hash result to be fully inside the packet data. Memory layout of the output
+ * packet may change during the crypto operation. If the output packet is
+ * longer than needed, it is not truncated and the extra data bytes retain
+ * their content.
+ *
+ * It is ok to pass the same packet handle as both the input packet and the
+ * output packet for the same crypto operation. In that case the input packet
+ * is consumed but returned as the output packet (with possibly different
+ * memory layout).
+ *
+ * ODP_CRYPTO_OP_TYPE_BASIC:
+ *
+ * ODP allocates the output packet from the pool from which the input
+ * packet was allocated. The processed input packet is consumed. All
+ * packet data and metadata are copied from the input packet to the output
+ * packet before the requested crypto operation is applied to the output
+ * packet. Memory layout (including packet data pointers, head and tail room,
+ * segmentation) of the output packet may differ from that of the input
+ * packet.
+ *
+ * The value of pktout[n] is ignored as pktout[n] is used purely as an
+ * output parameter that returns the handle of the newly allocated packet.
+ *
+ * ODP_CRYPTO_OP_TYPE_OOP:
+ *
+ * Writes the output bytes of the crypto operation in a caller provided
+ * output packet passed through pkt_out[n]. Input packets are not consumed
+ * nor modified. Memory layout (including packet data pointers, head and
+ * tail room, segmentation) of the output packet may change during the
+ * operation.
+ *
+ * Crypto output is the processed crypto_range, auth_range and
+ * MAC/digest (in encode sessions) of the input packet. The operation
+ * behaves as if crypto range and auth range were first copied from the
+ * input packet to the output packet and then the crypto operation
+ * was applied to the output packet.
+ *
+ * Crypto range and auth range of null cipher and auth algorithms are
+ * ignored, i.e. not copied in the output packet. Auth range of (AEAD)
+ * algorithms that ignore auth range is not copied.
+ *
+ * The offset of the crypto range and auth range in the output packet is
+ * the same as in the input packet, adjusted by dst_offset_shift operation
+ * parameter.
+ *
+ * pkt_out[n] must be a valid handle to a packet that is long enough to
+ * contain the shifted crypto range, auth range and, in encode sessions,
+ * the MAC/digest result. pkt_out[n] must not be the same as any input
+ * packet or any other output packet.
*
* @param pkt_in Packets to be processed
- * @param[in,out] pkt_out Packet handle array specifyint resulting packets
+ * @param[in,out] pkt_out Packet handle array for resulting packets
* @param param Operation parameters array
* @param num_pkt Number of packets to be processed
*
- * @return Number of input packets consumed (0 ... num_pkt)
+ * @return Number of input packets processed (0 ... num_pkt)
* @retval <0 on failure
*/
int odp_crypto_op(const odp_packet_t pkt_in[],
@@ -1351,14 +359,24 @@ int odp_crypto_op(const odp_packet_t pkt_in[],
/**
* Crypto packet operation
*
- * Performs the ASYNC cryptographic operations specified during session creation
- * on the packets. Caller should initialize pkt_out either with desired output
- * packet handles or with ODP_PACKET_INVALID to make ODP allocate new packets
- * from provided pool. All arrays should be of num_pkt size. Resulting packets
- * are returned through events.
+ * Performs the ASYNC cryptographic operations specified during session
+ * creation on the packets. Behaves otherwise like odp_crypto_op() but
+ * returns output packets through events.
+ *
+ * With operation types other than ODP_CRYPTO_OP_TYPE_LEGACY, packet
+ * data of processed packets may not be valid before odp_crypto_result()
+ * has been called.
+ *
+ * With ODP_CRYPTO_OP_TYPE_OOP, an enqueued input packet is consumed but
+ * returned back unmodified after the crypto operation is complete. The
+ * caller may not access the input packet until getting the handle back
+ * through odp_crypto_result().
+ *
+ * All arrays should be of num_pkt size, except that pkt_out parameter
+ * is ignored when the crypto operation type is ODP_CRYPTO_OP_TYPE_BASIC.
*
* @param pkt_in Packets to be processed
- * @param pkt_out Packet handle array specifying resulting packets
+ * @param pkt_out Packet handle array for resulting packets
* @param param Operation parameters array
* @param num_pkt Number of packets to be processed
*
diff --git a/include/odp/api/spec/crypto_types.h b/include/odp/api/spec/crypto_types.h
new file mode 100644
index 000000000..574f56445
--- /dev/null
+++ b/include/odp/api/spec/crypto_types.h
@@ -0,0 +1,1102 @@
+/* Copyright (c) 2014-2018, Linaro Limited
+ * Copyright (c) 2021-2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODP crypto types */
+
+#ifndef ODP_API_SPEC_CRYPTO_TYPES_H_
+#define ODP_API_SPEC_CRYPTO_TYPES_H_
+#include <odp/visibility_begin.h>
+
+#include <odp/api/deprecated.h>
+#include <odp/api/packet_types.h>
+#include <odp/api/pool_types.h>
+#include <odp/api/std_types.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** @addtogroup odp_crypto
+ * @{
+ */
+
+/**
+ * @def ODP_CRYPTO_SESSION_INVALID
+ * Invalid session handle
+ */
+
+/**
+ * @typedef odp_crypto_session_t
+ * Crypto API opaque session handle
+ */
+
+/**
+ * @typedef odp_crypto_compl_t
+* @deprecated Crypto API completion event (platform dependent).
+*/
+
+/**
+ * Crypto API operation mode
+ */
+typedef enum {
+ /** Synchronous, return results immediately */
+ ODP_CRYPTO_SYNC,
+ /** Asynchronous, return results via posted event */
+ ODP_CRYPTO_ASYNC,
+} odp_crypto_op_mode_t;
+
+/**
+ * Crypto API operation
+ */
+typedef enum {
+ /** Encrypt and/or compute authentication ICV */
+ ODP_CRYPTO_OP_ENCODE,
+ /** Decrypt and/or verify authentication ICV */
+ ODP_CRYPTO_OP_DECODE,
+} odp_crypto_op_t;
+
+/**
+ * Crypto API cipher algorithm
+ */
+typedef enum {
+ /** No cipher algorithm specified */
+ ODP_CIPHER_ALG_NULL,
+
+ /** DES */
+ ODP_CIPHER_ALG_DES,
+
+ /** Triple DES with cipher block chaining */
+ ODP_CIPHER_ALG_3DES_CBC,
+
+ /** Triple DES with Electronic Codebook */
+ ODP_CIPHER_ALG_3DES_ECB,
+
+ /** AES with cipher block chaining */
+ ODP_CIPHER_ALG_AES_CBC,
+
+ /** AES with counter mode */
+ ODP_CIPHER_ALG_AES_CTR,
+
+ /** AES with electronic codebook */
+ ODP_CIPHER_ALG_AES_ECB,
+
+ /** AES with 128-bit cipher feedback */
+ ODP_CIPHER_ALG_AES_CFB128,
+
+ /** AES with XEX-based tweaked-codebook mode with ciphertext stealing
+ * (XTS) */
+ ODP_CIPHER_ALG_AES_XTS,
+
+ /** AES-GCM
+ *
+ * AES in Galois/Counter Mode (GCM) algorithm. GCM provides both
+ * authentication and ciphering of data (authenticated encryption)
+ * in the same operation. Hence this algorithm must be paired always
+ * with ODP_AUTH_ALG_AES_GCM authentication.
+ */
+ ODP_CIPHER_ALG_AES_GCM,
+
+ /** AES-CCM
+ *
+ * AES in Counter with CBC-MAC (CCM) mode algorithm. CCM provides both
+ * authentication and ciphering of data (authenticated encryption)
+ * in the same operation. Hence this algorithm must be paired always
+ * with ODP_AUTH_ALG_AES_CCM authentication.
+ */
+ ODP_CIPHER_ALG_AES_CCM,
+
+ /** ChaCha20-Poly1305
+ *
+ * ChaCha20 with Poly1305 provide both authentication and ciphering of
+ * data (authenticated encryption) in the same operation. Hence this
+ * algorithm must be paired always with ODP_AUTH_ALG_CHACHA20_POLY1305
+ * authentication.
+ */
+ ODP_CIPHER_ALG_CHACHA20_POLY1305,
+
+ /** Confidentiality F8 algorithm (UEA1)
+ *
+ * KASUMI-based F8 algorithm (also known as UEA1).
+ *
+ * IV should be formatted according to the 3GPP TS 35.201:
+ * COUNT || BEARER || DIRECTION || 0...0
+ */
+ ODP_CIPHER_ALG_KASUMI_F8,
+
+ /** Confidentiality UEA2 algorithm (128-EEA1)
+ *
+ * SNOW 3G-based UEA2 algorithm (also known as 128-EEA1).
+ *
+ * IV (128 bits) should be formatted according to the ETSI/SAGE
+ * UEA2 & UIA2 specification:
+ * COUNT || BEARER || DIRECTION || 0...0 ||
+ * COUNT || BEARER || DIRECTION || 0...0 ||
+ */
+ ODP_CIPHER_ALG_SNOW3G_UEA2,
+
+ /** Confidentiality 128-EEA2 algorithm
+ *
+ * AES-CTR-based 128-EEA2 algorithm.
+ *
+ * IV (128 bits) should be formatted according to the ETSI/SAGE
+ * 128-EAA2 & 128-EIA2 specification:
+ * COUNT || BEARER || DIRECTION || 0....0
+ */
+ ODP_CIPHER_ALG_AES_EEA2,
+
+ /** ZUC based confidentiality algorithm
+ *
+ * 128-EEA3/128-NEA3 algorithm when key length is 128 bits.
+ *
+ * IV (128 bits) should be formatted according to the ETSI/SAGE
+ * 128-EEA3 & 128-EIA3 specification:
+ * COUNT || BEARER || DIRECTION || 0...0 ||
+ * COUNT || BEARER || DIRECTION || 0...0 ||
+ *
+ * 256-bit key length support is experimental and subject to
+ * change. The following variants may be supported:
+ *
+ * - ZUC-256 with 25 byte IV (of which 184 bits are variable)
+ * as specified in "The ZUC-256 Stream Cipher".
+ * - ZUC-256 with 16 byte IV as specified in
+ * "An Addendum to the ZUC-256 Stream Cipher",
+ * https://eprint.iacr.org/2021/1439
+ */
+ ODP_CIPHER_ALG_ZUC_EEA3,
+
+} odp_cipher_alg_t;
+
+/**
+ * Crypto API authentication algorithm
+ */
+typedef enum {
+ /** No authentication algorithm specified */
+ ODP_AUTH_ALG_NULL,
+
+ /** HMAC-MD5
+ *
+ * MD5 algorithm in HMAC mode
+ */
+ ODP_AUTH_ALG_MD5_HMAC,
+
+ /** HMAC-SHA-1
+ *
+ * SHA-1 algorithm in HMAC mode
+ */
+ ODP_AUTH_ALG_SHA1_HMAC,
+
+ /** HMAC-SHA-224
+ *
+ * SHA-224 algorithm in HMAC mode
+ */
+ ODP_AUTH_ALG_SHA224_HMAC,
+
+ /** HMAC-SHA-256
+ *
+ * SHA-256 algorithm in HMAC mode
+ */
+ ODP_AUTH_ALG_SHA256_HMAC,
+
+ /** HMAC-SHA-384
+ *
+ * SHA-384 algorithm in HMAC mode
+ */
+ ODP_AUTH_ALG_SHA384_HMAC,
+
+ /** HMAC-SHA-512
+ *
+ * SHA-512 algorithm in HMAC mode
+ */
+ ODP_AUTH_ALG_SHA512_HMAC,
+
+ /** AES-GCM
+ *
+ * AES in Galois/Counter Mode (GCM) algorithm. GCM provides both
+ * authentication and ciphering of data (authenticated encryption)
+ * in the same operation. Hence this algorithm must be paired always
+ * with ODP_CIPHER_ALG_AES_GCM cipher.
+ */
+ ODP_AUTH_ALG_AES_GCM,
+
+ /** AES-GMAC
+ *
+ * AES Galois Message Authentication Code (GMAC) algorithm. AES-GMAC
+ * is based on AES-GCM operation, but provides authentication only.
+ * Hence this algorithm can be paired only with ODP_CIPHER_ALG_NULL
+ * cipher.
+ *
+ * NIST and RFC specifications of GMAC refer to all data to be
+ * authenticated as AAD. In ODP the data to be authenticated, i.e.
+ * AAD, is ODP packet data and specified using the auth_range
+ * parameter. The aad_length and aad_ptr parameters, which would
+ * require the data to be contiguous in memory, are ignored with
+ * AES-GMAC.
+ *
+ * GMAC needs an initialization vector, which must be passed via
+ * operation parameters (auth_iv_ptr).
+ */
+ ODP_AUTH_ALG_AES_GMAC,
+
+ /** AES-CCM
+ *
+ * AES in Counter with CBC-MAC (CCM) mode algorithm. CCM provides both
+ * authentication and ciphering of data (authenticated encryption)
+ * in the same operation. Hence this algorithm must be paired always
+ * with ODP_CIPHER_ALG_AES_CCM cipher.
+ */
+ ODP_AUTH_ALG_AES_CCM,
+
+ /** AES-CMAC
+ *
+ * AES Cipher-based Message Authentication Code (CMAC) algorithm. CMAC
+ * is a keyed hash function that is based on a symmetric key block
+ * cipher, such as the AES.
+ */
+ ODP_AUTH_ALG_AES_CMAC,
+
+ /** AES-XCBC-MAC
+ *
+ * AES CBC MAC for arbitrary-length messages (XCBC-MAC).
+ *
+ */
+ ODP_AUTH_ALG_AES_XCBC_MAC,
+
+ /** ChaCha20-Poly1305 AEAD
+ *
+ * ChaCha20 with Poly1305 provide both authentication and ciphering of
+ * data (authenticated encryption) in the same operation. Hence this
+ * algorithm must be paired always with
+ * ODP_CIPHER_ALG_CHACHA20_POLY1305 cipher.
+ */
+ ODP_AUTH_ALG_CHACHA20_POLY1305,
+
+ /** Integrity F9 algorithm (UIA1)
+ *
+ * KASUMI-based F9 algorithm (also known as UIA1).
+ *
+ * IV (9 bytes) is a concatenation of COUNT (32b), FRESH (32b) and
+ * DIRECTION (LSB-aligned, 1b).
+ * IV (8 bytes) is a concatenation of COUNT (32b) and FRESH (32b)
+ * DIRECTION (1b) and padding should come at the end of message.
+ */
+ ODP_AUTH_ALG_KASUMI_F9,
+
+ /** Integrity UIA2 algorithm (128-EIA1)
+ *
+ * SNOW 3G-based UIA2 algorithm (also known as 128-EIA1).
+ * IV (128 bits) should be formatted according to the ETSI/SAGE
+ * UEA2 & UIA2 specification:
+ * COUNT || FRESH ||
+ * DIRECTION XOR COUNT0 || COUNT1 .. COUNT31 ||
+ * FRESH0 .. FRESH15 || FRESH16 XOR DIRECTION || FRESH17 .. FRESH31
+ */
+ ODP_AUTH_ALG_SNOW3G_UIA2,
+
+ /** Integrity 128-EIA2 algorithm
+ *
+ * AES_CMAC-based 128-EIA2 algorithm.
+ *
+ * IV (64 bits) should be formatted according to the ETSI/SAGE
+ * 128-EEA2 & 128-EIA2 specification:
+ * COUNT || BEARER || DIRECTION || 0....0
+ */
+ ODP_AUTH_ALG_AES_EIA2,
+
+ /** ZUC-based integrity algorithm.
+ *
+ * 128-EIA3/128-NIA3 algorithm when key length is 128 bits.
+ *
+ * IV (128 bits) should be formatted according to the ETSI/SAGE
+ * 128-EEA3 & 128-EIA2 specification:
+ * COUNT || BEARER ||
+ * DIRECTION XOR COUNT0 || COUNT1 .. COUNT31 ||
+ * BEARER || 0...0 || DIRECTION || 0...0
+ *
+ * 256-bit key length support is experimental and subject to
+ * change. The following variants may be supported:
+ *
+ * - ZUC-256 with 25 byte IV (of which 184 bits are variable) and
+ * 32/64/128 bit MAC as specified in "The ZUC-256 Stream Cipher".
+ * - ZUC-256 with 16 byte IV and 32/64/128 bit MAC as specified in
+ * "An Addendum to the ZUC-256 Stream Cipher",
+ * https://eprint.iacr.org/2021/1439
+ */
+ ODP_AUTH_ALG_ZUC_EIA3,
+
+ /** MD5 algorithm */
+ ODP_AUTH_ALG_MD5,
+
+ /** SHA1 algorithm */
+ ODP_AUTH_ALG_SHA1,
+
+ /** 224 bit SHA2 algorithm */
+ ODP_AUTH_ALG_SHA224,
+
+ /** 256 bit SHA2 algorithm */
+ ODP_AUTH_ALG_SHA256,
+
+ /** 384 bit SHA2 algorithm */
+ ODP_AUTH_ALG_SHA384,
+
+ /** 512 bit SHA2 algorithm */
+ ODP_AUTH_ALG_SHA512,
+
+} odp_auth_alg_t;
+
+/**
+ * Cipher algorithms in a bit field structure
+ */
+typedef union odp_crypto_cipher_algos_t {
+ /** Cipher algorithms */
+ struct {
+ /** ODP_CIPHER_ALG_NULL */
+ uint32_t null : 1;
+
+ /** ODP_CIPHER_ALG_DES */
+ uint32_t des : 1;
+
+ /** ODP_CIPHER_ALG_3DES_CBC */
+ uint32_t trides_cbc : 1;
+
+ /** ODP_CIPHER_ALG_3DES_ECB */
+ uint32_t trides_ecb : 1;
+
+ /** ODP_CIPHER_ALG_AES_CBC */
+ uint32_t aes_cbc : 1;
+
+ /** ODP_CIPHER_ALG_AES_CTR */
+ uint32_t aes_ctr : 1;
+
+ /** ODP_CIPHER_ALG_AES_ECB */
+ uint32_t aes_ecb : 1;
+
+ /** ODP_CIPHER_ALG_AES_CFB128 */
+ uint32_t aes_cfb128 : 1;
+
+ /** ODP_CIPHER_ALG_AES_XTS */
+ uint32_t aes_xts : 1;
+
+ /** ODP_CIPHER_ALG_AES_GCM */
+ uint32_t aes_gcm : 1;
+
+ /** ODP_CIPHER_ALG_AES_CCM */
+ uint32_t aes_ccm : 1;
+
+ /** ODP_CIPHER_ALG_CHACHA20_POLY1305 */
+ uint32_t chacha20_poly1305 : 1;
+
+ /** ODP_CIPHER_ALG_KASUMI_F8 */
+ uint32_t kasumi_f8 : 1;
+
+ /** ODP_CIPHER_ALG_SNOW3G_UEA2 */
+ uint32_t snow3g_uea2 : 1;
+
+ /** ODP_CIPHER_ALG_AES_EEA2 */
+ uint32_t aes_eea2 : 1;
+
+ /** ODP_CIPHER_ALG_ZUC_EEA3 */
+ uint32_t zuc_eea3 : 1;
+
+ } bit;
+
+ /** All bits of the bit field structure
+ *
+ * This field can be used to set/clear all flags, or bitwise
+ * operations over the entire structure. */
+ uint32_t all_bits;
+} odp_crypto_cipher_algos_t;
+
+/**
+ * Authentication algorithms in a bit field structure
+ */
+typedef union odp_crypto_auth_algos_t {
+ /** Authentication algorithms */
+ struct {
+ /** ODP_AUTH_ALG_NULL */
+ uint32_t null : 1;
+
+ /** ODP_AUTH_ALG_MD5_HMAC */
+ uint32_t md5_hmac : 1;
+
+ /** ODP_AUTH_ALG_SHA1_HMAC */
+ uint32_t sha1_hmac : 1;
+
+ /** ODP_AUTH_ALG_SHA224_HMAC */
+ uint32_t sha224_hmac : 1;
+
+ /** ODP_AUTH_ALG_SHA256_HMAC */
+ uint32_t sha256_hmac : 1;
+
+ /** ODP_AUTH_ALG_SHA384_HMAC */
+ uint32_t sha384_hmac : 1;
+
+ /** ODP_AUTH_ALG_SHA512_HMAC */
+ uint32_t sha512_hmac : 1;
+
+ /** ODP_AUTH_ALG_AES_GCM */
+ uint32_t aes_gcm : 1;
+
+ /** ODP_AUTH_ALG_AES_GMAC*/
+ uint32_t aes_gmac : 1;
+
+ /** ODP_AUTH_ALG_AES_CCM */
+ uint32_t aes_ccm : 1;
+
+ /** ODP_AUTH_ALG_AES_CMAC*/
+ uint32_t aes_cmac : 1;
+
+ /** ODP_AUTH_ALG_AES_XCBC_MAC*/
+ uint32_t aes_xcbc_mac : 1;
+
+ /** ODP_AUTH_ALG_CHACHA20_POLY1305 */
+ uint32_t chacha20_poly1305 : 1;
+
+ /** ODP_AUTH_ALG_KASUMI_F9 */
+ uint32_t kasumi_f9 : 1;
+
+ /** ODP_AUTH_ALG_SNOW3G_UIA2 */
+ uint32_t snow3g_uia2 : 1;
+
+ /** ODP_AUTH_ALG_AES_EIA2 */
+ uint32_t aes_eia2 : 1;
+
+ /** ODP_AUTH_ALG_ZUC_EIA3 */
+ uint32_t zuc_eia3 : 1;
+
+ /** ODP_AUTH_ALG_MD5 */
+ uint32_t md5 : 1;
+
+ /** ODP_AUTH_ALG_SHA1 */
+ uint32_t sha1 : 1;
+
+ /** ODP_AUTH_ALG_SHA224 */
+ uint32_t sha224 : 1;
+
+ /** ODP_AUTH_ALG_SHA256 */
+ uint32_t sha256 : 1;
+
+ /** ODP_AUTH_ALG_SHA384 */
+ uint32_t sha384 : 1;
+
+ /** ODP_AUTH_ALG_SHA512 */
+ uint32_t sha512 : 1;
+
+ } bit;
+
+ /** All bits of the bit field structure
+ *
+ * This field can be used to set/clear all flags, or bitwise
+ * operations over the entire structure. */
+ uint32_t all_bits;
+} odp_crypto_auth_algos_t;
+
+/**
+ * Crypto API key structure
+ */
+typedef struct odp_crypto_key {
+ /** Key data */
+ uint8_t *data;
+
+ /** Key length in bytes */
+ uint32_t length;
+
+} odp_crypto_key_t;
+
+/**
+ * Type of odp_crypto_op()/odp_crypto_op_enq() calls.
+ */
+typedef enum odp_crypto_op_type_t {
+ /**
+ * Input packet data and metadata are copied in the output packet
+ * and then processed. Output packet is allocated by the caller
+ * or by ODP. odp_crypto_op(), odp_crypto_op_enq() and
+ * odp_crypto_operation() can be used.
+ *
+ * This is the default value but will be deprecated in the future.
+ */
+ ODP_CRYPTO_OP_TYPE_LEGACY,
+
+ /**
+ * Input packet data and metadata are copied in the output packet
+ * and then processed. Output packet is allocated by ODP.
+ * odp_crypto_op() and odp_crypto_op_enq() can be used.
+ */
+ ODP_CRYPTO_OP_TYPE_BASIC,
+
+ /**
+ * Out-of-place crypto operation. Output packet is provided by
+ * the caller and the input packet is not consumed nor modified.
+ *
+ * Output of the crypto operation is written in the caller provided
+ * output packet without affecting other data and metadata of the
+ * output packet. Memory layout of the output packet may change
+ * during the operation.
+ *
+ * Crypto output is the processed crypto_range, auth_range and
+ * MAC/digest (in encode sessions) of the input packet.
+ *
+ * odp_crypto_op() and odp_crypto_op_enq() can be used.
+ */
+ ODP_CRYPTO_OP_TYPE_OOP,
+} odp_crypto_op_type_t;
+
+/**
+ * Crypto API session creation parameters
+ */
+typedef struct odp_crypto_session_param_t {
+ /** Encode vs. decode operation
+ *
+ * The default value is ODP_CRYPTO_OP_ENCODE.
+ */
+ odp_crypto_op_t op;
+
+ /** Crypto operation type
+ *
+ * This field defines how the crypto operation functions are
+ * to be called and what they return. In particular, this field
+ * specifies the interpretation of the output packet parameter,
+ * how output packets are allocated and what data and metadata
+ * they contain.
+ *
+ * The default value is ODP_CRYPTO_OP_TYPE_LEGACY.
+ */
+ odp_crypto_op_type_t op_type;
+
+ /** Authenticate cipher vs. plain text
+ *
+ * Controls ordering of authentication and cipher operations,
+ * and is relative to the operation (encode vs decode). When encoding,
+ * TRUE indicates the authentication operation should be performed
+ * after the cipher operation else before. When decoding, TRUE
+ * indicates the reverse order of operation.
+ *
+ * The value is ignored with authenticated encryption algorithms
+ * such as AES-GCM.
+ *
+ * true: Authenticate cipher text
+ * false: Authenticate plain text
+ *
+ * The default value is false.
+ */
+ odp_bool_t auth_cipher_text;
+
+ /** Hash result location may overlap authentication range
+ *
+ * This flag indicates that the hash result location may (but is
+ * not required to) overlap authentication range. Setting this
+ * flag may reduce performance.
+ *
+ * Default value is false.
+ */
+ odp_bool_t hash_result_in_auth_range;
+
+ /** Preferred sync vs. async for odp_crypto_operation()
+ *
+ * The default value is ODP_CRYPTO_SYNC.
+ *
+ * @deprecated Used only with deprecated odp_crypto_operation()
+ */
+ odp_crypto_op_mode_t ODP_DEPRECATE(pref_mode);
+
+ /** Operation mode when using packet interface: sync or async
+ *
+ * The default value is ODP_CRYPTO_SYNC.
+ */
+ odp_crypto_op_mode_t op_mode;
+
+ /** Cipher algorithm
+ *
+ * Select cipher algorithm to be used. ODP_CIPHER_ALG_NULL indicates
+ * that ciphering is disabled. Use odp_crypto_capability() for
+ * supported algorithms. Note that some algorithms restrict choice of
+ * the pairing authentication algorithm. When ciphering is enabled
+ * cipher key and IV need to be set. The default value is
+ * ODP_CIPHER_ALG_NULL.
+ */
+ odp_cipher_alg_t cipher_alg;
+
+ /** Cipher key
+ *
+ * Use odp_crypto_cipher_capa() for supported key and IV lengths.
+ */
+ odp_crypto_key_t cipher_key;
+
+ /** Cipher IV length. The default value is zero. */
+ uint32_t cipher_iv_len;
+
+ /** Authentication algorithm
+ *
+ * Select authentication algorithm to be used. ODP_AUTH_ALG_NULL
+ * indicates that authentication is disabled. Use
+ * odp_crypto_capability() for supported algorithms. Note that some
+ * algorithms restrict choice of the pairing cipher algorithm. When
+ * single algorithm provides both ciphering and authentication
+ * (i.e. Authenticated Encryption), authentication side key
+ * (auth_key) and IV (auth_iv) are ignored, and cipher side values are
+ * used instead. These algorithms ignore authentication side key
+ * and IV: ODP_AUTH_ALG_AES_GCM, ODP_AUTH_ALG_AES_CCM and
+ * ODP_AUTH_ALG_CHACHA20_POLY1305. Otherwise, all authentication side
+ * parameters must be set when authentication is enabled. The default
+ * value is ODP_AUTH_ALG_NULL.
+ */
+ odp_auth_alg_t auth_alg;
+
+ /** Authentication key
+ *
+ * Use odp_crypto_auth_capability() for supported key lengths.
+ */
+ odp_crypto_key_t auth_key;
+
+ /** Authentication IV length. The default value is zero. */
+ uint32_t auth_iv_len;
+
+ /** Authentication digest length in bytes
+ *
+ * Use odp_crypto_auth_capability() for supported digest lengths.
+ */
+ uint32_t auth_digest_len;
+
+ /** Additional Authenticated Data (AAD) length in bytes
+ *
+ * AAD length is constant for all operations (packets) of the session.
+ * Set to zero when AAD is not used. Use odp_crypto_auth_capability()
+ * for supported AAD lengths. The default value is zero.
+ */
+ uint32_t auth_aad_len;
+
+ /** Async mode completion event queue
+ *
+ * The completion queue is used to return completions from
+ * odp_crypto_op_enq() (and the deprecated odp_crypto_operation())
+ * to the application.
+ */
+ odp_queue_t compl_queue;
+
+ /** Output pool
+ *
+ * When the output packet is not specified during the call to
+ * crypto operation in the legacy operation type, the output
+ * packet will be allocated from this pool.
+ *
+ * In ODP_CRYPTO_OP_TYPE_BASIC and ODP_CRYPTO_OP_TYPE_OOP
+ * operation types this must be set to ODP_POOL_INVALID.
+ */
+ odp_pool_t output_pool;
+
+} odp_crypto_session_param_t;
+
+/**
+ * Crypto API per packet operation parameters
+ *
+ * @deprecated Use odp_crypto_packet_op_param_t instead.
+ */
+typedef struct odp_crypto_op_param_t {
+ /** Session handle from creation */
+ odp_crypto_session_t session;
+
+ /** User context */
+ void *ctx;
+
+ /** Input packet
+ *
+ * Specifies the input packet for the crypto operation. When the
+ * 'out_pkt' variable is set to ODP_PACKET_INVALID (indicating a new
+ * packet should be allocated for the resulting packet).
+ */
+ odp_packet_t pkt;
+
+ /** Output packet
+ *
+ * Both "in place" (the original packet 'pkt' is modified) and
+ * "copy" (the packet is replicated to a new packet which contains
+ * the modified data) modes are supported. The "in place" mode of
+ * operation is indicated by setting 'out_pkt' equal to 'pkt'.
+ * For the copy mode of operation, setting 'out_pkt' to a valid packet
+ * value indicates the caller wishes to specify the destination packet.
+ * Setting 'out_pkt' to ODP_PACKET_INVALID indicates the caller wishes
+ * the destination packet be allocated from the output pool specified
+ * during session creation.
+ */
+ odp_packet_t out_pkt;
+
+ /** IV pointer for cipher */
+ uint8_t *cipher_iv_ptr;
+
+ /** Authentication IV pointer */
+ uint8_t *auth_iv_ptr;
+
+ /** Offset from start of packet for hash result
+ *
+ * In case of decode sessions, the expected hash will be read from
+ * this offset and compared with the calculated hash. After the
+ * operation the hash bytes will have undefined values.
+ *
+ * In case of encode sessions the calculated hash will be stored in
+ * this offset.
+ *
+ * If the hash_result_in_auth_range session parameter is true,
+ * the hash result location may overlap auth_range. In that case
+ * the result location will be zeroed in decode sessions before
+ * hash calculation. Zeroing is not done in encode sessions.
+ */
+ uint32_t hash_result_offset;
+
+ /** Pointer to AAD. AAD length is defined by 'auth_aad_len'
+ * session parameter.
+ */
+ uint8_t *aad_ptr;
+
+ /** Data range to be ciphered */
+ odp_packet_data_range_t cipher_range;
+
+ /** Data range to be authenticated
+ *
+ * The value is ignored with authenticated encryption algorithms,
+ * such as AES-GCM, which authenticate data in the cipher range
+ * and the AAD.
+ *
+ * As a special case AES-GMAC uses this field instead of aad_ptr
+ * for the data bytes to be authenticated.
+ */
+ odp_packet_data_range_t auth_range;
+
+} ODP_DEPRECATE(odp_crypto_op_param_t);
+
+/**
+ * Crypto packet API per packet operation parameters
+ */
+typedef struct odp_crypto_packet_op_param_t {
+ /** Session handle from creation */
+ odp_crypto_session_t session;
+
+ /** IV pointer for cipher */
+ uint8_t *cipher_iv_ptr;
+
+ /** IV pointer for authentication */
+ uint8_t *auth_iv_ptr;
+
+ /** Offset from start of packet for hash result
+ *
+ * In case of decode sessions, the expected hash will be read from
+ * this offset from the input packet and compared with the calculated
+ * hash. After the operation the hash bytes will have undefined
+ * values except with out-of-place sessions (ODP_CRYPTO_OP_TYPE_OOP
+ * operation type).
+ *
+ * With out-of-place decode sessions the input packet is not modified
+ * but if the hash location overlaps the cipher range or the auth
+ * range, then the corresponding location in the output packet will
+ * have undefined content.
+ *
+ * In case of encode sessions the calculated hash will be stored in
+ * this offset in the output packet.
+ *
+ * If the hash_result_in_auth_range session parameter is true,
+ * the hash result location may overlap auth_range. In that case the
+ * result location will be treated as containing zero bytes for the
+ * purpose of hash calculation in decode sessions.
+ */
+ uint32_t hash_result_offset;
+
+ /** Pointer to AAD. AAD length is defined by 'auth_aad_len'
+ * session parameter.
+ */
+ uint8_t *aad_ptr;
+
+ /** Data range to apply cipher */
+ odp_packet_data_range_t cipher_range;
+
+ /** Data range to authenticate */
+ odp_packet_data_range_t auth_range;
+
+ /** Shift of the output offsets with ODP_CRYPTO_OP_TYPE_OOP
+ *
+ * The processed crypto range and auth range of the input packet
+ * will be written in the output packet at the offset specified
+ * in the ranges (i.e. the same as in the input packet), shifted
+ * by this many bytes. This allows directing the output to
+ * a different packet offset than the offset of the input data.
+ *
+ * This is ignored if the crypto operation type is not
+ * ODP_CRYPTO_OP_TYPE_OOP.
+ */
+ int32_t dst_offset_shift;
+
+} odp_crypto_packet_op_param_t;
+
+/**
+ * Crypto API session creation return code
+ */
+typedef enum {
+ /** Session created */
+ ODP_CRYPTO_SES_ERR_NONE,
+ /** Creation failed, no resources */
+ ODP_CRYPTO_SES_ERR_ENOMEM,
+ /** Creation failed, bad cipher params */
+ ODP_CRYPTO_SES_ERR_CIPHER,
+ /** Creation failed, bad auth params */
+ ODP_CRYPTO_SES_ERR_AUTH,
+
+ /** Unsupported combination of algorithms
+ *
+ * The combination of cipher and auth algorithms with their
+ * specific parameters is not supported even if the algorithms
+ * appear in capabilities and are supported in combination with
+ * other algorithms or other algorithm specific parameters.
+ */
+ ODP_CRYPTO_SES_ERR_ALG_COMBO,
+
+ /** Unsupported order of cipher and auth
+ *
+ * The requested mutual order of ciphering and authentication
+ * is not supported with the chosen individual cipher and
+ * authentication algorithms.
+ */
+ ODP_CRYPTO_SES_ERR_ALG_ORDER,
+
+ /** Unsupported combination of session creation parameters
+ *
+ * The combination of provided session creation parameters is not
+ * supported. This error can occur when there are limitations that
+ * are not expressible through crypto capabilities or other error
+ * status values.
+ */
+ ODP_CRYPTO_SES_ERR_PARAMS,
+} odp_crypto_ses_create_err_t;
+
+/** This synonym for backward compatibility will be deprecated later */
+#define ODP_CRYPTO_SES_CREATE_ERR_NONE ODP_CRYPTO_SES_ERR_NONE
+/** This synonym for backward compatibility will be deprecated later */
+#define ODP_CRYPTO_SES_CREATE_ERR_ENOMEM ODP_CRYPTO_SES_ERR_ENOMEM
+/** This synonym for backward compatibility will be deprecated later */
+#define ODP_CRYPTO_SES_CREATE_ERR_INV_CIPHER ODP_CRYPTO_SES_ERR_CIPHER
+/** This synonym for backward compatibility will be deprecated later */
+#define ODP_CRYPTO_SES_CREATE_ERR_INV_AUTH ODP_CRYPTO_SES_ERR_AUTH
+
+/**
+ * Crypto API algorithm return code
+ */
+typedef enum {
+ /** Algorithm successful */
+ ODP_CRYPTO_ALG_ERR_NONE,
+ /** Invalid data block size */
+ ODP_CRYPTO_ALG_ERR_DATA_SIZE,
+ /** Key size invalid for algorithm */
+ ODP_CRYPTO_ALG_ERR_KEY_SIZE,
+ /** Computed ICV value mismatch */
+ ODP_CRYPTO_ALG_ERR_ICV_CHECK,
+ /** IV value not specified */
+ ODP_CRYPTO_ALG_ERR_IV_INVALID,
+} odp_crypto_alg_err_t;
+
+/**
+ * Crypto API hardware centric return code
+ */
+typedef enum {
+ /** Operation completed successfully */
+ ODP_CRYPTO_HW_ERR_NONE,
+ /** Error detected during DMA of data */
+ ODP_CRYPTO_HW_ERR_DMA,
+ /** Operation failed due to pool depletion */
+ ODP_CRYPTO_HW_ERR_BP_DEPLETED,
+} odp_crypto_hw_err_t;
+
+/**
+ * Cryto API per packet operation completion status
+ */
+typedef struct odp_crypto_op_status {
+ /** Algorithm specific return code */
+ odp_crypto_alg_err_t alg_err;
+
+ /** Hardware specific return code */
+ odp_crypto_hw_err_t hw_err;
+
+} odp_crypto_op_status_t;
+
+/**
+ * Crypto API operation result
+ *
+ * @deprecated Use odp_crypto_packet_result_t instead.
+ */
+typedef struct odp_crypto_op_result {
+ /** Request completed successfully */
+ odp_bool_t ok;
+
+ /** User context from request */
+ void *ctx;
+
+ /** Output packet */
+ odp_packet_t pkt;
+
+ /** Cipher status */
+ odp_crypto_op_status_t cipher_status;
+
+ /** Authentication status */
+ odp_crypto_op_status_t auth_status;
+
+} ODP_DEPRECATE(odp_crypto_op_result_t);
+
+/**
+ * Crypto packet API operation result
+ */
+typedef struct odp_crypto_packet_result_t {
+ /** Request completed successfully */
+ odp_bool_t ok;
+
+ /** Input packet passed to odp_crypo_op_enq() when the operation
+ * type of the session is ODP_CRYPTO_OP_TYPE_OOP. In other cases
+ * this field does not have a valid value.
+ */
+ odp_packet_t pkt_in;
+
+ /** Cipher status */
+ odp_crypto_op_status_t cipher_status;
+
+ /** Authentication status */
+ odp_crypto_op_status_t auth_status;
+
+} odp_crypto_packet_result_t;
+
+/**
+ * Crypto capabilities
+ */
+typedef struct odp_crypto_capability_t {
+ /** Maximum number of crypto sessions */
+ uint32_t max_sessions;
+
+ /** Supported packet operation in SYNC mode */
+ odp_support_t sync_mode;
+
+ /** Supported packet operation in ASYNC mode */
+ odp_support_t async_mode;
+
+ /** Supported cipher algorithms */
+ odp_crypto_cipher_algos_t ciphers;
+
+ /** Cipher algorithms implemented with HW offload */
+ odp_crypto_cipher_algos_t hw_ciphers;
+
+ /** Supported authentication algorithms */
+ odp_crypto_auth_algos_t auths;
+
+ /** Authentication algorithms implemented with HW offload */
+ odp_crypto_auth_algos_t hw_auths;
+
+ /**
+ * Scheduled crypto completion queue support
+ *
+ * This defines whether scheduled queues are supported as crypto
+ * compl_queue.
+ * 0: Scheduled queues are not supported as crypto completion queues
+ * 1: Scheduled queues are supported as crypto completion queues
+ * @see odp_crypto_session_param_t
+ */
+ odp_bool_t queue_type_sched;
+
+ /**
+ * Plain crypto completion queue support
+ *
+ * This defines whether plain queues are supported as crypto
+ * compl_queue.
+ * 0: Plain queues are not supported as crypto completion queues
+ * 1: Plain queues are supported as crypto completion queues
+ * @see odp_crypto_session_param_t
+ */
+ odp_bool_t queue_type_plain;
+} odp_crypto_capability_t;
+
+/**
+ * Cipher algorithm capabilities
+ */
+typedef struct odp_crypto_cipher_capability_t {
+ /** Key length in bytes */
+ uint32_t key_len;
+
+ /** IV length in bytes */
+ uint32_t iv_len;
+
+ /** Cipher is operating in bitwise mode
+ *
+ * This cipher works on series of bits, rather than sequences of bytes:
+ * cipher_range in odp_crypto_op_param_t and
+ * odp_crypto_packet_op_param_t will use bits, rather than bytes.
+ *
+ * Note: data buffer MUST start on the byte boundary, using offset
+ * which is not divisible by 8 is unsupported and will result in
+ * unspecified behaviour.
+ *
+ * Note2: If the data length is not a multiple of 8, the remaining
+ * bits of the data in the last byte of the input/output will be the
+ * most significant bits, i.e. the most significant bit is considered
+ * to be the first bit of a byte for the purpose of input and output
+ * data range. The output bits that fall out of the output range are
+ * undefined.
+ */
+ odp_bool_t bit_mode;
+
+} odp_crypto_cipher_capability_t;
+
+/**
+ * Authentication algorithm capabilities
+ */
+typedef struct odp_crypto_auth_capability_t {
+ /** Digest length in bytes */
+ uint32_t digest_len;
+
+ /** Key length in bytes */
+ uint32_t key_len;
+
+ /** IV length in bytes */
+ uint32_t iv_len;
+
+ /** Additional Authenticated Data (AAD) lengths */
+ struct {
+ /** Minimum AAD length in bytes */
+ uint32_t min;
+
+ /** Maximum AAD length in bytes */
+ uint32_t max;
+
+ /** Increment of supported lengths between min and max
+ * (in bytes) */
+ uint32_t inc;
+ } aad_len;
+
+ /** Auth is operating in bitstring mode
+ *
+ * This auth works on series of bits, rather than sequences of bytes:
+ * auth_range in odp_crypto_op_param_t and
+ * odp_crypto_packet_op_param_t will use bits, rather than bytes.
+ *
+ * Note: data buffer MUST start on the byte boundary, using offset
+ * which is not divisible by 8 is unsupported and will result in
+ * unpredictable behaviour.
+ *
+ * Note2: If the data length is not a multiple of 8, the remaining
+ * bits of the data in the last byte of the input/output will be the
+ * most significant bits, i.e. the most significant bit is considered
+ * to be the first bit of a byte for the purpose of input and output
+ * data range. The output bits that fall out of the output range are
+ * undefined.
+ */
+ odp_bool_t bit_mode;
+
+} odp_crypto_auth_capability_t;
+
+/**
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <odp/visibility_end.h>
+#endif
diff --git a/include/odp/api/spec/ipsec.h b/include/odp/api/spec/ipsec.h
index b091961cc..603bf3aef 100644
--- a/include/odp/api/spec/ipsec.h
+++ b/include/odp/api/spec/ipsec.h
@@ -19,7 +19,7 @@
extern "C" {
#endif
-#include <odp/api/crypto.h>
+#include <odp/api/crypto_types.h>
#include <odp/api/event_types.h>
#include <odp/api/ipsec_types.h>
#include <odp/api/packet_types.h>
diff --git a/include/odp/api/spec/ipsec_types.h b/include/odp/api/spec/ipsec_types.h
index b74d26d4f..b34fb5982 100644
--- a/include/odp/api/spec/ipsec_types.h
+++ b/include/odp/api/spec/ipsec_types.h
@@ -20,7 +20,7 @@ extern "C" {
#endif
#include <odp/api/classification.h>
-#include <odp/api/crypto.h>
+#include <odp/api/crypto_types.h>
#include <odp/api/packet_io_types.h>
#include <odp/api/protocols.h>
#include <odp/api/std_types.h>
diff --git a/include/odp/api/spec/packet_flags.h b/include/odp/api/spec/packet_flags.h
index 98bf2b7bf..c62e50885 100644
--- a/include/odp/api/spec/packet_flags.h
+++ b/include/odp/api/spec/packet_flags.h
@@ -1,4 +1,5 @@
/* Copyright (c) 2014-2018, Linaro Limited
+ * Copyright (c) 2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -22,7 +23,13 @@ extern "C" {
#include <odp/api/packet_types.h>
/** @addtogroup odp_packet
- * Operations on packet metadata flags.
+ * @par Operations on packet metadata flags
+ *
+ * If user sets multiple conflicting packet metadata flags
+ * using odp_packet_has_XX_set() functions, only the last set flag value is
+ * guaranteed to hold. The values of other conflicting flags are implementation
+ * specific. The conflicting flag combinations are defined in function
+ * documentations.
* @{
*/
@@ -172,6 +179,8 @@ int odp_packet_has_jumbo(odp_packet_t pkt);
/**
* Check for VLAN
*
+ * Check if packet contains normal or QinQ VLAN header.
+ *
* @param pkt Packet handle
*
* @retval non-zero Packet contains a VLAN header
@@ -182,6 +191,8 @@ int odp_packet_has_vlan(odp_packet_t pkt);
/**
* Check for VLAN QinQ (stacked VLAN)
*
+ * Check if packet contains QinQ VLAN header.
+ *
* @param pkt Packet handle
*
* @retval non-zero Packet contains a VLAN QinQ header
@@ -401,6 +412,9 @@ void odp_packet_has_jumbo_set(odp_packet_t pkt, int val);
/**
* Set flag for VLAN
*
+ * Set when packet contains normal VLAN header. Only one VLAN flag
+ * (VLAN/VLAN QinQ) can be set simultaneously.
+ *
* @param pkt Packet handle
* @param val Value
*/
@@ -409,6 +423,9 @@ void odp_packet_has_vlan_set(odp_packet_t pkt, int val);
/**
* Set flag for VLAN QinQ (stacked VLAN)
*
+ * Set when packet contains QinQ VLAN header. Only one VLAN flag
+ * (VLAN/VLAN QinQ) can be set simultaneously.
+ *
* @param pkt Packet handle
* @param val Value
*/
@@ -417,6 +434,8 @@ void odp_packet_has_vlan_qinq_set(odp_packet_t pkt, int val);
/**
* Set flag for ARP
*
+ * Only one of ARP/IPv4/IPv6 flags can be set simultaneously.
+ *
* @param pkt Packet handle
* @param val Value
*/
@@ -425,6 +444,8 @@ void odp_packet_has_arp_set(odp_packet_t pkt, int val);
/**
* Set flag for IPv4
*
+ * Only one of ARP/IPv4/IPv6 flags can be set simultaneously.
+ *
* @param pkt Packet handle
* @param val Value
*/
@@ -433,6 +454,8 @@ void odp_packet_has_ipv4_set(odp_packet_t pkt, int val);
/**
* Set flag for IPv6
*
+ * Only one of ARP/IPv4/IPv6 flags can be set simultaneously.
+ *
* @param pkt Packet handle
* @param val Value
*/
@@ -441,6 +464,8 @@ void odp_packet_has_ipv6_set(odp_packet_t pkt, int val);
/**
* Set flag for IP broadcast address
*
+ * Only one of IP broadcast/multicast flags can be set simultaneously.
+ *
* @param pkt Packet handle
* @param val Value
*/
@@ -449,6 +474,8 @@ void odp_packet_has_ip_bcast_set(odp_packet_t pkt, int val);
/**
* Set flag for IP multicast address
*
+ * Only one of IP broadcast/multicast flags can be set simultaneously.
+ *
* @param pkt Packet handle
* @param val Value
*/
@@ -481,6 +508,8 @@ void odp_packet_has_ipsec_set(odp_packet_t pkt, int val);
/**
* Set flag for UDP
*
+ * Only one of TCP/UDP/SCTP/ICMP flags can be set simultaneously.
+ *
* @param pkt Packet handle
* @param val Value
*/
@@ -489,6 +518,8 @@ void odp_packet_has_udp_set(odp_packet_t pkt, int val);
/**
* Set flag for TCP
*
+ * Only one of TCP/UDP/SCTP/ICMP flags can be set simultaneously.
+ *
* @param pkt Packet handle
* @param val Value
*/
@@ -497,6 +528,8 @@ void odp_packet_has_tcp_set(odp_packet_t pkt, int val);
/**
* Set flag for SCTP
*
+ * Only one of TCP/UDP/SCTP/ICMP flags can be set simultaneously.
+ *
* @param pkt Packet handle
* @param val Value
*/
@@ -505,6 +538,8 @@ void odp_packet_has_sctp_set(odp_packet_t pkt, int val);
/**
* Set flag for ICMP
*
+ * Only one of TCP/UDP/SCTP/ICMP flags can be set simultaneously.
+ *
* @param pkt Packet handle
* @param val Value
*/
diff --git a/include/odp/api/spec/packet_io.h b/include/odp/api/spec/packet_io.h
index e8f0cd0b9..9a6117cee 100644
--- a/include/odp/api/spec/packet_io.h
+++ b/include/odp/api/spec/packet_io.h
@@ -20,7 +20,6 @@ extern "C" {
#endif
#include <odp/api/classification.h>
-#include <odp/api/deprecated.h>
#include <odp/api/packet_types.h>
#include <odp/api/packet_io_stats.h>
#include <odp/api/packet_io_types.h>
@@ -833,38 +832,6 @@ int odp_pktio_info(odp_pktio_t pktio, odp_pktio_info_t *info);
int odp_pktio_link_info(odp_pktio_t pktio, odp_pktio_link_info_t *info);
/**
- * Packet input timestamp resolution in hertz
- *
- * @deprecated Use odp_pktio_ts_res() instead, which returns resolution for
- * both packet input and output timestamps.
- *
- * This is the resolution of packet input timestamps. Returns zero on a failure
- * or when timestamping is disabled.
- *
- * @param pktio Packet IO handle
- *
- * @return Packet input timestamp resolution in hertz
- * @retval 0 on failure
- */
-uint64_t ODP_DEPRECATE(odp_pktin_ts_res)(odp_pktio_t pktio);
-
-/**
- * Convert nanoseconds to packet input time
- *
- * @deprecated Use odp_pktio_ts_from_ns() instead, which can be used with both
- * packet input and output timestamps.
- *
- * Packet input time source is used for timestamping incoming packets.
- * This function is used convert nanosecond time to packet input timestamp time.
- *
- * @param pktio Packet IO handle
- * @param ns Time in nanoseconds
- *
- * @return Packet input timestamp
- */
-odp_time_t ODP_DEPRECATE(odp_pktin_ts_from_ns)(odp_pktio_t pktio, uint64_t ns);
-
-/**
* Packet IO timestamp resolution in hertz
*
* This is the resolution of packet input and output timestamps using a packet
diff --git a/include/odp/api/spec/packet_io_stats.h b/include/odp/api/spec/packet_io_stats.h
index c6b782d5a..156a3a95e 100644
--- a/include/odp/api/spec/packet_io_stats.h
+++ b/include/odp/api/spec/packet_io_stats.h
@@ -19,7 +19,6 @@
extern "C" {
#endif
-#include <odp/api/deprecated.h>
#include <odp/api/queue_types.h>
/** @addtogroup odp_packet_io
@@ -68,20 +67,6 @@ typedef struct odp_pktio_stats_t {
* configuration, packets with errors may be dropped or not. */
uint64_t in_errors;
- /**
- * For packet-oriented interfaces, the number of packets received via
- * the interface which were discarded because of an unknown or
- * unsupported protocol. For character-oriented or fixed-length
- * interfaces that support protocol multiplexing the number of
- * transmission units received via the interface which were discarded
- * because of an unknown or unsupported protocol. For any interface
- * that does not support protocol multiplexing, this counter will always
- * be 0. See ifInUnknownProtos in RFC 2863, RFC 3635.
- *
- * @deprecated This counter has been deprecated.
- */
- uint64_t ODP_DEPRECATE(in_unknown_protos);
-
/** Number of octets in successfully transmitted packets. In case of
* Ethernet, packet size includes MAC header. */
uint64_t out_octets;
diff --git a/include/odp/api/spec/packet_io_types.h b/include/odp/api/spec/packet_io_types.h
index fe86f6f12..a46668c30 100644
--- a/include/odp/api/spec/packet_io_types.h
+++ b/include/odp/api/spec/packet_io_types.h
@@ -19,7 +19,6 @@
extern "C" {
#endif
-#include <odp/api/deprecated.h>
#include <odp/api/packet_types.h>
#include <odp/api/packet_io_stats.h>
#include <odp/api/pool_types.h>
diff --git a/include/odp/api/spec/shared_memory.h b/include/odp/api/spec/shared_memory.h
index 36b38782f..eebf288c9 100644
--- a/include/odp/api/spec/shared_memory.h
+++ b/include/odp/api/spec/shared_memory.h
@@ -51,19 +51,6 @@ extern "C" {
*/
/**
- * Application SW only, no HW access
- *
- * @deprecated When set, application will not share the reserved memory with HW
- * accelerators. However, leaving this flag to zero does not guarantee that
- * the reserved memory can be accessed from HW, and thus usage of this flag is
- * considered deprecated. If HW accessible memory is required, set
- * ODP_SHM_HW_ACCESS instead.
- *
- * This flag must not be combined with ODP_SHM_HW_ACCESS.
- */
-#define ODP_SHM_SW_ONLY 0x1
-
-/**
* Share with external processes
*/
#define ODP_SHM_PROC 0x2
diff --git a/include/odp/api/spec/timer.h b/include/odp/api/spec/timer.h
index edfa423e7..0e1f07f09 100644
--- a/include/odp/api/spec/timer.h
+++ b/include/odp/api/spec/timer.h
@@ -20,6 +20,7 @@
extern "C" {
#endif
+#include <odp/api/deprecated.h>
#include <odp/api/timer_types.h>
#include <odp/api/event_types.h>
#include <odp/api/pool_types.h>
@@ -387,7 +388,7 @@ int odp_timer_periodic_cancel(odp_timer_t timer);
*
* @deprecated Use odp_timer_start() or odp_timer_restart() instead
*/
-int odp_timer_set_abs(odp_timer_t timer, uint64_t abs_tick, odp_event_t *tmo_ev);
+int ODP_DEPRECATE(odp_timer_set_abs)(odp_timer_t timer, uint64_t abs_tick, odp_event_t *tmo_ev);
/**
* Set (or reset) a timer with relative expiration time
@@ -416,7 +417,7 @@ int odp_timer_set_abs(odp_timer_t timer, uint64_t abs_tick, odp_event_t *tmo_ev)
*
* @deprecated Use odp_timer_start() or odp_timer_restart() instead
*/
-int odp_timer_set_rel(odp_timer_t timer, uint64_t rel_tick, odp_event_t *tmo_ev);
+int ODP_DEPRECATE(odp_timer_set_rel)(odp_timer_t timer, uint64_t rel_tick, odp_event_t *tmo_ev);
/**
* Cancel a timer
diff --git a/include/odp/api/spec/timer_types.h b/include/odp/api/spec/timer_types.h
index 754c831b2..d61992fba 100644
--- a/include/odp/api/spec/timer_types.h
+++ b/include/odp/api/spec/timer_types.h
@@ -20,8 +20,9 @@
extern "C" {
#endif
-#include <odp/api/std_types.h>
+#include <odp/api/deprecated.h>
#include <odp/api/event_types.h>
+#include <odp/api/std_types.h>
/** @defgroup odp_timer ODP TIMER
* Timer generating timeout events.
@@ -245,13 +246,21 @@ typedef enum {
/** The default clock source */
#define ODP_CLOCK_DEFAULT ODP_CLOCK_SRC_0
-/** For backwards compatibility, ODP_CLOCK_CPU is synonym of ODP_CLOCK_DEFAULT.
- * This will be deprecated in the future. */
+#if ODP_DEPRECATED_API
+/**
+ * For backwards compatibility, ODP_CLOCK_CPU is synonym of ODP_CLOCK_DEFAULT.
+ *
+ * @deprecated Use #ODP_CLOCK_DEFAULT instead.
+ */
#define ODP_CLOCK_CPU ODP_CLOCK_DEFAULT
-/** For backwards compatibility, ODP_CLOCK_EXT is synonym of ODP_CLOCK_SRC_1.
- * This will be deprecated in the future. */
+/**
+ * For backwards compatibility, ODP_CLOCK_EXT is synonym of ODP_CLOCK_SRC_1.
+ *
+ * @deprecated Use #ODP_CLOCK_SRC_1 instead.
+ */
#define ODP_CLOCK_EXT ODP_CLOCK_SRC_1
+#endif
/**
* Timer expiration mode
@@ -514,17 +523,28 @@ typedef enum {
} odp_timer_set_t;
-/** For backwards compatibility, ODP_TIMER_TOOEARLY is synonym of ODP_TIMER_TOO_NEAR.
- * This will be deprecated in the future. */
+#if ODP_DEPRECATED_API
+/**
+ * For backwards compatibility, ODP_TIMER_TOOEARLY is synonym of ODP_TIMER_TOO_NEAR.
+ *
+ * @deprecated Use #ODP_TIMER_TOO_NEAR instead.
+ */
#define ODP_TIMER_TOOEARLY ODP_TIMER_TOO_NEAR
-/** For backwards compatibility, ODP_TIMER_TOOLATE is synonym of ODP_TIMER_TOO_FAR.
- * This will be deprecated in the future. */
+/**
+ * For backwards compatibility, ODP_TIMER_TOOLATE is synonym of ODP_TIMER_TOO_FAR.
+ *
+ * @deprecated Use #ODP_TIMER_TOO_FAR instead.
+ */
#define ODP_TIMER_TOOLATE ODP_TIMER_TOO_FAR
-/** For backwards compatibility, ODP_TIMER_NOEVENT is synonym of ODP_TIMER_FAIL.
- * This will be deprecated in the future. */
+/**
+ * For backwards compatibility, ODP_TIMER_NOEVENT is synonym of ODP_TIMER_FAIL.
+ *
+ * @deprecated Use #ODP_TIMER_FAIL instead.
+ */
#define ODP_TIMER_NOEVENT ODP_TIMER_FAIL
+#endif
/**
* Timer tick information
diff --git a/include/odp/api/spec/traffic_mngr.h b/include/odp/api/spec/traffic_mngr.h
index f634cfb44..9bd735a7a 100644
--- a/include/odp/api/spec/traffic_mngr.h
+++ b/include/odp/api/spec/traffic_mngr.h
@@ -745,39 +745,6 @@ void odp_tm_requirements_init(odp_tm_requirements_t *requirements);
*/
void odp_tm_egress_init(odp_tm_egress_t *egress);
-/** Query All TM Capabilities
- *
- * @deprecated Use odp_tm_egress_capabilities() instead that also additionally
- * takes egress as input to provide capabilities specific to a given egress.
- *
- * This function returns the set of TM capabilities that are common for all
- * egresses. The reason that this returns a SET of capabilities and not just
- * one, is because it is expected that many HW based implementations may have
- * one set of limits for the HW and also support a SW TM implementation with a
- * (presumably larger) different set of limits. There are also cases where
- * there could be more than one SW implementation (one supporting say tens of
- * thousands of tm_queues and a variant supporting tens of millions of
- * tm_queues). It returns capabilities that are valid for all egresses.
- * The caller passes in an array of odp_tm_capabilities_t records and the
- * maximum number of such records to output. If number of such records
- * implementation supports is larger than caller requested number, then
- * only caller requested number of records are written and return value is
- * max number of records implementation supports.
- * Caller then may again call with larger number of records to be returned.
- *
- * @param[out] capabilities An array of odp_tm_capabilities_t records to
- * be filled in.
- * @param capabilities_size The number of odp_tm_capabilities_t records
- * in the capabilities array.
- * @return Returns < 0 upon failure. Returns N > 0,
- * where N is the maximum number of different
- * odp_tm_capabilities_t records that the
- * implementations supports. *NOTE* that this
- * number can be > capabilities_size!
- */
-int ODP_DEPRECATE(odp_tm_capabilities)(odp_tm_capabilities_t capabilities[],
- uint32_t capabilities_size);
-
/** Query TM Capabilities specific to an egress
*
* The function returns the set of TM limits supported by this implementation
@@ -1094,22 +1061,14 @@ typedef struct {
* for this integer is in bits per second when packet_mode is
* not TRUE while packets per second when packet mode is TRUE.
*/
- union {
- /** @deprecated Use commit_rate instead */
- uint64_t ODP_DEPRECATE(commit_bps);
- uint64_t commit_rate; /**< Commit information rate */
- };
+ uint64_t commit_rate;
/** The peak information rate for this shaper profile. The units for
* this integer is in bits per second when packet_mode is
* not TRUE while in packets per second when packet mode is TRUE.
* This field is ignored when dual_rate is FALSE.
*/
- union {
- /** @deprecated Use peak_rate instead */
- uint64_t ODP_DEPRECATE(peak_bps);
- uint64_t peak_rate; /**< Peak information rate */
- };
+ uint64_t peak_rate;
/** The commit burst tolerance for this shaper profile. The units for
* this field is bits when packet_mode is not TRUE and packets when
diff --git a/include/odp/arch/arm32-linux/odp/api/abi/buffer_types.h b/include/odp/arch/arm32-linux/odp/api/abi/buffer_types.h
new file mode 100644
index 000000000..8dd78321f
--- /dev/null
+++ b/include/odp/arch/arm32-linux/odp/api/abi/buffer_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/buffer_types.h>
diff --git a/include/odp/arch/arm32-linux/odp/api/abi/crypto_types.h b/include/odp/arch/arm32-linux/odp/api/abi/crypto_types.h
new file mode 100644
index 000000000..ddadec1c1
--- /dev/null
+++ b/include/odp/arch/arm32-linux/odp/api/abi/crypto_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/crypto_types.h>
diff --git a/include/odp/arch/arm64-linux/odp/api/abi/buffer_types.h b/include/odp/arch/arm64-linux/odp/api/abi/buffer_types.h
new file mode 100644
index 000000000..8dd78321f
--- /dev/null
+++ b/include/odp/arch/arm64-linux/odp/api/abi/buffer_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/buffer_types.h>
diff --git a/include/odp/arch/arm64-linux/odp/api/abi/crypto_types.h b/include/odp/arch/arm64-linux/odp/api/abi/crypto_types.h
new file mode 100644
index 000000000..ddadec1c1
--- /dev/null
+++ b/include/odp/arch/arm64-linux/odp/api/abi/crypto_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/crypto_types.h>
diff --git a/include/odp/arch/default-linux/odp/api/abi/buffer_types.h b/include/odp/arch/default-linux/odp/api/abi/buffer_types.h
new file mode 100644
index 000000000..8dd78321f
--- /dev/null
+++ b/include/odp/arch/default-linux/odp/api/abi/buffer_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/buffer_types.h>
diff --git a/include/odp/arch/default-linux/odp/api/abi/crypto_types.h b/include/odp/arch/default-linux/odp/api/abi/crypto_types.h
new file mode 100644
index 000000000..ddadec1c1
--- /dev/null
+++ b/include/odp/arch/default-linux/odp/api/abi/crypto_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/crypto_types.h>
diff --git a/include/odp/arch/power64-linux/odp/api/abi/buffer_types.h b/include/odp/arch/power64-linux/odp/api/abi/buffer_types.h
new file mode 100644
index 000000000..8dd78321f
--- /dev/null
+++ b/include/odp/arch/power64-linux/odp/api/abi/buffer_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/buffer_types.h>
diff --git a/include/odp/arch/power64-linux/odp/api/abi/crypto_types.h b/include/odp/arch/power64-linux/odp/api/abi/crypto_types.h
new file mode 100644
index 000000000..ddadec1c1
--- /dev/null
+++ b/include/odp/arch/power64-linux/odp/api/abi/crypto_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/crypto_types.h>
diff --git a/include/odp/arch/x86_32-linux/odp/api/abi/buffer_types.h b/include/odp/arch/x86_32-linux/odp/api/abi/buffer_types.h
new file mode 100644
index 000000000..8dd78321f
--- /dev/null
+++ b/include/odp/arch/x86_32-linux/odp/api/abi/buffer_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/buffer_types.h>
diff --git a/include/odp/arch/x86_32-linux/odp/api/abi/crypto_types.h b/include/odp/arch/x86_32-linux/odp/api/abi/crypto_types.h
new file mode 100644
index 000000000..ddadec1c1
--- /dev/null
+++ b/include/odp/arch/x86_32-linux/odp/api/abi/crypto_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/crypto_types.h>
diff --git a/include/odp/arch/x86_64-linux/odp/api/abi/buffer_types.h b/include/odp/arch/x86_64-linux/odp/api/abi/buffer_types.h
new file mode 100644
index 000000000..8dd78321f
--- /dev/null
+++ b/include/odp/arch/x86_64-linux/odp/api/abi/buffer_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/buffer_types.h>
diff --git a/include/odp/arch/x86_64-linux/odp/api/abi/crypto_types.h b/include/odp/arch/x86_64-linux/odp/api/abi/crypto_types.h
new file mode 100644
index 000000000..ddadec1c1
--- /dev/null
+++ b/include/odp/arch/x86_64-linux/odp/api/abi/crypto_types.h
@@ -0,0 +1,7 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/abi-default/crypto_types.h>
diff --git a/include/odp/autoheader_external.h.in b/include/odp/autoheader_external.h.in
index dd60fa0b2..74662b2e0 100644
--- a/include/odp/autoheader_external.h.in
+++ b/include/odp/autoheader_external.h.in
@@ -11,4 +11,7 @@
/* Define cache line size */
#undef _ODP_CACHE_LINE_SIZE
+/* Define to 1 or 2 to enable event validation */
+#undef _ODP_EVENT_VALIDATION
+
#endif
diff --git a/include/odp/autoheader_internal.h.in b/include/odp/autoheader_internal.h.in
index 33d9f280f..a42b34ad0 100644
--- a/include/odp/autoheader_internal.h.in
+++ b/include/odp/autoheader_internal.h.in
@@ -35,4 +35,7 @@
/* Define to 1 to enable XDP support */
#undef _ODP_PKTIO_XDP
+/* Define to 1 to enable IPSec MB crypto support */
+#undef _ODP_IPSECMB
+
#endif
diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am
index 3bd599bcd..76e205212 100644
--- a/platform/linux-dpdk/Makefile.am
+++ b/platform/linux-dpdk/Makefile.am
@@ -32,9 +32,11 @@ odpapiplatinclude_HEADERS = \
include/odp/api/plat/buffer_inline_types.h \
include/odp/api/plat/byteorder_inlines.h \
include/odp/api/plat/cpu_inlines.h \
+ include/odp/api/plat/crypto_inlines.h \
include/odp/api/plat/debug_inlines.h \
include/odp/api/plat/event_inlines.h \
include/odp/api/plat/event_inline_types.h \
+ include/odp/api/plat/event_validation_external.h \
include/odp/api/plat/event_vector_inline_types.h \
include/odp/api/plat/hash_inlines.h \
include/odp/api/plat/ipsec_inlines.h \
@@ -68,11 +70,13 @@ odpapiabiarchinclude_HEADERS += \
include-abi/odp/api/abi/atomic.h \
include-abi/odp/api/abi/barrier.h \
include-abi/odp/api/abi/buffer.h \
+ include-abi/odp/api/abi/buffer_types.h \
include-abi/odp/api/abi/byteorder.h \
include-abi/odp/api/abi/classification.h \
include-abi/odp/api/abi/comp.h \
include-abi/odp/api/abi/cpumask.h \
include-abi/odp/api/abi/crypto.h \
+ include-abi/odp/api/abi/crypto_types.h \
include-abi/odp/api/abi/debug.h \
include-abi/odp/api/abi/dma_types.h \
include-abi/odp/api/abi/errno.h \
@@ -141,6 +145,7 @@ noinst_HEADERS = \
${top_srcdir}/platform/linux-generic/include/odp_print_internal.h \
include/odp_errno_define.h \
include/odp_event_internal.h \
+ ${top_srcdir}/platform/linux-generic/include/odp_event_validation_internal.h \
${top_srcdir}/platform/linux-generic/include/odp_packet_dpdk.h \
${top_srcdir}/platform/linux-generic/include/odp_pcapng.h \
${top_srcdir}/platform/linux-generic/include/odp_pkt_queue_internal.h \
@@ -154,6 +159,8 @@ noinst_HEADERS = \
${top_srcdir}/platform/linux-generic/include/odp_ring_common.h \
${top_srcdir}/platform/linux-generic/include/odp_ring_internal.h \
${top_srcdir}/platform/linux-generic/include/odp_ring_mpmc_internal.h \
+ ${top_srcdir}/platform/linux-generic/include/odp_ring_mpmc_u32_internal.h \
+ ${top_srcdir}/platform/linux-generic/include/odp_ring_mpmc_u64_internal.h \
${top_srcdir}/platform/linux-generic/include/odp_ring_ptr_internal.h \
include/odp_ptr_ring_mpmc_internal.h \
include/odp_ptr_ring_spsc_internal.h \
@@ -197,6 +204,7 @@ __LIB__libodp_dpdk_la_SOURCES = \
odp_crypto.c \
odp_errno.c \
odp_event.c \
+ ../linux-generic/odp_event_validation.c \
../linux-generic/odp_hash_crc_gen.c \
odp_init.c \
../linux-generic/odp_impl.c \
@@ -249,6 +257,7 @@ __LIB__libodp_dpdk_la_SOURCES += \
../linux-generic/odp_buffer_api.c \
../linux-generic/odp_byteorder_api.c \
../linux-generic/odp_cpu_api.c \
+ ../linux-generic/odp_crypto_api.c \
../linux-generic/odp_event_api.c \
../linux-generic/odp_hash_api.c \
../linux-generic/odp_ipsec_api.c \
diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/buffer_types.h b/platform/linux-dpdk/include-abi/odp/api/abi/buffer_types.h
new file mode 120000
index 000000000..49e88bd2e
--- /dev/null
+++ b/platform/linux-dpdk/include-abi/odp/api/abi/buffer_types.h
@@ -0,0 +1 @@
+../../../../../linux-generic/include-abi/odp/api/abi/buffer_types.h \ No newline at end of file
diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/crypto_types.h b/platform/linux-dpdk/include-abi/odp/api/abi/crypto_types.h
new file mode 120000
index 000000000..ce973bb2d
--- /dev/null
+++ b/platform/linux-dpdk/include-abi/odp/api/abi/crypto_types.h
@@ -0,0 +1 @@
+../../../../../linux-generic/include-abi/odp/api/abi/crypto_types.h \ No newline at end of file
diff --git a/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h b/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h
index 6265914b4..396d78559 100644
--- a/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h
+++ b/platform/linux-dpdk/include/odp/api/plat/buffer_inlines.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2019-2022, Nokia
+/* Copyright (c) 2019-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -7,15 +7,15 @@
#ifndef ODP_PLAT_BUFFER_INLINES_H_
#define ODP_PLAT_BUFFER_INLINES_H_
+#include <odp/api/buffer_types.h>
#include <odp/api/event.h>
#include <odp/api/hints.h>
#include <odp/api/pool_types.h>
-#include <odp/api/abi/buffer.h>
-
#include <odp/api/plat/buffer_inline_types.h>
#include <odp/api/plat/debug_inlines.h>
#include <odp/api/plat/event_inline_types.h>
+#include <odp/api/plat/event_validation_external.h>
#include <rte_mbuf.h>
#include <rte_mempool.h>
@@ -95,6 +95,8 @@ _ODP_INLINE void odp_buffer_free(odp_buffer_t buf)
{
struct rte_mbuf *mbuf = (struct rte_mbuf *)buf;
+ _odp_buffer_validate(buf, _ODP_EV_BUFFER_FREE);
+
rte_mempool_put(mbuf->pool, mbuf);
}
@@ -107,6 +109,8 @@ _ODP_INLINE void odp_buffer_free_multi(const odp_buffer_t buf[], int num)
if (odp_unlikely(num <= 0))
return;
+ _odp_buffer_validate_multi(buf, num, _ODP_EV_BUFFER_FREE_MULTI);
+
mbuf_tbl[0] = (struct rte_mbuf *)buf[0];
mp_pending = mbuf_tbl[0]->pool;
num_pending = 1;
@@ -134,6 +138,9 @@ _ODP_INLINE int odp_buffer_is_valid(odp_buffer_t buf)
if (odp_event_type(odp_buffer_to_event(buf)) != ODP_EVENT_BUFFER)
return 0;
+ if (odp_unlikely(_odp_buffer_validate(buf, _ODP_EV_BUFFER_IS_VALID)))
+ return 0;
+
return 1;
}
diff --git a/platform/linux-dpdk/include/odp/api/plat/crypto_inlines.h b/platform/linux-dpdk/include/odp/api/plat/crypto_inlines.h
new file mode 120000
index 000000000..0c1935696
--- /dev/null
+++ b/platform/linux-dpdk/include/odp/api/plat/crypto_inlines.h
@@ -0,0 +1 @@
+../../../../../linux-generic/include/odp/api/plat/crypto_inlines.h \ No newline at end of file
diff --git a/platform/linux-dpdk/include/odp/api/plat/event_validation_external.h b/platform/linux-dpdk/include/odp/api/plat/event_validation_external.h
new file mode 120000
index 000000000..001662d8b
--- /dev/null
+++ b/platform/linux-dpdk/include/odp/api/plat/event_validation_external.h
@@ -0,0 +1 @@
+../../../../../linux-generic/include/odp/api/plat/event_validation_external.h \ No newline at end of file
diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h b/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h
index 9a72a70ef..c6ede5d3a 100644
--- a/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h
+++ b/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h
@@ -1,5 +1,5 @@
/* Copyright (c) 2018, Linaro Limited
- * Copyright (c) 2022, Nokia
+ * Copyright (c) 2022-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -44,6 +44,7 @@ typedef struct _odp_packet_inline_offset_t {
uint16_t subtype;
uint16_t cls_mark;
uint16_t ipsec_ctx;
+ uint16_t crypto_op;
uint16_t buf_addr;
uint16_t data;
uint16_t pkt_len;
diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h
index 4347afb88..4ae53c7b3 100644
--- a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h
+++ b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h
@@ -1,5 +1,5 @@
/* Copyright (c) 2016-2018, Linaro Limited
- * Copyright (c) 2019-2022, Nokia
+ * Copyright (c) 2019-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -25,6 +25,7 @@ extern "C" {
#include <odp/api/time.h>
#include <odp/api/plat/debug_inlines.h>
+#include <odp/api/plat/event_validation_external.h>
#include <odp/api/plat/packet_io_inlines.h>
#include <odp/api/plat/packet_inline_types.h>
#include <odp/api/plat/pool_inline_types.h>
@@ -663,17 +664,23 @@ _ODP_INLINE odp_event_t odp_packet_tx_compl_to_event(odp_packet_tx_compl_t tx_co
_ODP_INLINE void odp_packet_free(odp_packet_t pkt)
{
+ _odp_packet_validate(pkt, _ODP_EV_PACKET_FREE);
+
rte_pktmbuf_free((struct rte_mbuf *)pkt);
}
_ODP_INLINE void odp_packet_free_multi(const odp_packet_t pkt[], int num)
{
+ _odp_packet_validate_multi(pkt, num, _ODP_EV_PACKET_FREE_MULTI);
+
rte_pktmbuf_free_bulk((struct rte_mbuf **)(uintptr_t)pkt, (unsigned int)num);
}
_ODP_INLINE void odp_packet_free_sp(const odp_packet_t pkt[], int num)
{
- odp_packet_free_multi(pkt, num);
+ _odp_packet_validate_multi(pkt, num, _ODP_EV_PACKET_FREE_SP);
+
+ rte_pktmbuf_free_bulk((struct rte_mbuf **)(uintptr_t)pkt, (unsigned int)num);
}
_ODP_INLINE void *odp_packet_seg_data(odp_packet_t pkt ODP_UNUSED,
diff --git a/platform/linux-dpdk/include/odp_config_internal.h b/platform/linux-dpdk/include/odp_config_internal.h
index 18754184e..e104b9d24 100644
--- a/platform/linux-dpdk/include/odp_config_internal.h
+++ b/platform/linux-dpdk/include/odp_config_internal.h
@@ -1,5 +1,5 @@
/* Copyright (c) 2016-2018, Linaro Limited
- * Copyright (c) 2020-2021, Nokia
+ * Copyright (c) 2020-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -70,7 +70,7 @@ extern "C" {
/*
* Maximum number of stashes
*/
-#define CONFIG_MAX_STASHES 128
+#define CONFIG_MAX_STASHES 2048
/*
* Maximum number of packet IO resources
@@ -133,10 +133,10 @@ extern "C" {
/*
* Number of shared memory blocks reserved for implementation internal use.
*
- * Each stash and packet pool requires one SHM block, and 20 blocks are
- * reserved for per ODP module global data.
+ * Each packet pool requires one SHM block and 20 blocks are reserved for
+ * ODP module global data.
*/
-#define CONFIG_INTERNAL_SHM_BLOCKS (CONFIG_MAX_STASHES + ODP_CONFIG_POOLS + 20)
+#define CONFIG_INTERNAL_SHM_BLOCKS (ODP_CONFIG_POOLS + 20)
/*
* Maximum number of shared memory blocks.
diff --git a/platform/linux-dpdk/include/odp_event_internal.h b/platform/linux-dpdk/include/odp_event_internal.h
index 320f42146..5ccf5a3ed 100644
--- a/platform/linux-dpdk/include/odp_event_internal.h
+++ b/platform/linux-dpdk/include/odp_event_internal.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2021-2022, Nokia
+/* Copyright (c) 2021-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -80,6 +80,13 @@ static inline void _odp_event_type_set(odp_event_t event, int ev)
_odp_event_hdr(event)->event_type = ev;
}
+static inline uint64_t *_odp_event_endmark_get_ptr(odp_event_t event)
+{
+ struct rte_mbuf *mbuf = _odp_event_to_mbuf(event);
+
+ return (uint64_t *)((uint8_t *)mbuf->buf_addr + mbuf->buf_len);
+}
+
#ifdef __cplusplus
}
#endif
diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h b/platform/linux-dpdk/include/odp_packet_io_internal.h
index 78b1f343a..2efa52b67 100644
--- a/platform/linux-dpdk/include/odp_packet_io_internal.h
+++ b/platform/linux-dpdk/include/odp_packet_io_internal.h
@@ -1,5 +1,5 @@
/* Copyright (c) 2013-2018, Linaro Limited
- * Copyright (c) 2019-2022, Nokia
+ * Copyright (c) 2019-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -53,11 +53,7 @@ ODP_STATIC_ASSERT(PKTIO_LSO_PROFILES < UINT8_MAX, "PKTIO_LSO_PROFILES_ERROR");
/* Forward declaration */
struct pktio_if_ops;
-#if ODP_CACHE_LINE_SIZE == 128
-#define PKTIO_PRIVATE_SIZE 1536
-#else
-#define PKTIO_PRIVATE_SIZE 1344
-#endif
+#define PKTIO_PRIVATE_SIZE 9216
typedef struct ODP_ALIGNED_CACHE {
const struct pktio_if_ops *ops; /**< Implementation specific methods */
diff --git a/platform/linux-dpdk/include/odp_pool_internal.h b/platform/linux-dpdk/include/odp_pool_internal.h
index a5ffcf517..b82324a77 100644
--- a/platform/linux-dpdk/include/odp_pool_internal.h
+++ b/platform/linux-dpdk/include/odp_pool_internal.h
@@ -1,5 +1,5 @@
/* Copyright (c) 2013-2018, Linaro Limited
- * Copyright (c) 2021-2022, Nokia
+ * Copyright (c) 2021-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -72,7 +72,7 @@ typedef struct ODP_ALIGNED_CACHE {
/* Everything under this mark is memset() to zero on pool create */
uint8_t memset_mark;
struct rte_mempool *rte_mempool;
- uint32_t seg_len;
+ uint32_t seg_len; /* Initial packet segment length (excludes endmark) */
uint32_t ext_head_offset;
uint32_t num;
uint32_t num_populated;
@@ -84,6 +84,7 @@ typedef struct ODP_ALIGNED_CACHE {
odp_shm_t uarea_shm;
uint64_t uarea_shm_size;
uint32_t uarea_size;
+ uint32_t trailer_size; /* Endmark size */
uint8_t *uarea_base_addr;
char name[ODP_POOL_NAME_LEN];
diff --git a/platform/linux-dpdk/m4/configure.m4 b/platform/linux-dpdk/m4/configure.m4
index 90d55b6f6..70295400f 100644
--- a/platform/linux-dpdk/m4/configure.m4
+++ b/platform/linux-dpdk/m4/configure.m4
@@ -5,11 +5,13 @@ ODP_VISIBILITY
ODP_ATOMIC
m4_include([platform/linux-dpdk/m4/odp_cpu.m4])
+m4_include([platform/linux-dpdk/m4/odp_event_validation.m4])
m4_include([platform/linux-dpdk/m4/odp_libconfig.m4])
m4_include([platform/linux-dpdk/m4/odp_openssl.m4])
m4_include([platform/linux-dpdk/m4/odp_pcapng.m4])
m4_include([platform/linux-dpdk/m4/odp_scheduler.m4])
+ODP_EVENT_VALIDATION
ODP_PTHREAD
ODP_SCHEDULER
ODP_TIMER
@@ -68,6 +70,7 @@ AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS
# Add text to the end of configure with platform specific settings.
# Make sure it's aligned same as other lines in configure.ac.
AS_VAR_APPEND([PLAT_CFG_TEXT], ["
+ event_validation: ${enable_event_validation}
openssl: ${with_openssl}
openssl_rand: ${openssl_rand}
pcap: ${have_pmd_pcap}
diff --git a/platform/linux-dpdk/m4/odp_event_validation.m4 b/platform/linux-dpdk/m4/odp_event_validation.m4
new file mode 120000
index 000000000..0d457c6ff
--- /dev/null
+++ b/platform/linux-dpdk/m4/odp_event_validation.m4
@@ -0,0 +1 @@
+../../linux-generic/m4/odp_event_validation.m4 \ No newline at end of file
diff --git a/platform/linux-dpdk/m4/odp_libconfig.m4 b/platform/linux-dpdk/m4/odp_libconfig.m4
index 84d331b8a..1d56f0a5c 100644
--- a/platform/linux-dpdk/m4/odp_libconfig.m4
+++ b/platform/linux-dpdk/m4/odp_libconfig.m4
@@ -3,7 +3,7 @@
##########################################################################
m4_define([_odp_config_version_generation], [0])
m4_define([_odp_config_version_major], [1])
-m4_define([_odp_config_version_minor], [18])
+m4_define([_odp_config_version_minor], [21])
m4_define([_odp_config_version],
[_odp_config_version_generation._odp_config_version_major._odp_config_version_minor])
diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c
index 076de1b0e..713419a14 100644
--- a/platform/linux-dpdk/odp_crypto.c
+++ b/platform/linux-dpdk/odp_crypto.c
@@ -1,5 +1,5 @@
/* Copyright (c) 2017-2018, Linaro Limited
- * Copyright (c) 2018-2022, Nokia
+ * Copyright (c) 2018-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -83,10 +83,7 @@ typedef struct crypto_session_entry_s {
unsigned int chained_bufs_ok:1;
} flags;
uint8_t cdev_id;
-#if ODP_DEPRECATED_API
- uint8_t cipher_iv_data[MAX_IV_LENGTH];
- uint8_t auth_iv_data[MAX_IV_LENGTH];
-#endif
+
} crypto_session_entry_t;
typedef struct crypto_global_s {
@@ -1395,6 +1392,13 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param,
return -1;
}
+ /* ODP_CRYPTO_OP_TYPE_OOP not supported */
+ if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) {
+ *status = ODP_CRYPTO_SES_ERR_PARAMS;
+ *session_out = ODP_CRYPTO_SESSION_INVALID;
+ return -1;
+ }
+
if (rte_cryptodev_count() == 0) {
_ODP_ERR("No crypto devices available\n");
*status = ODP_CRYPTO_SES_ERR_ENOMEM;
@@ -1496,16 +1500,7 @@ int odp_crypto_session_create(const odp_crypto_session_param_t *param,
out_null:
session->rte_session = rte_session;
session->cdev_id = cdev_id;
-#if ODP_DEPRECATED_API
- if (param->cipher_iv.data)
- memcpy(session->cipher_iv_data,
- param->cipher_iv.data,
- param->cipher_iv.length);
- if (param->auth_iv.data)
- memcpy(session->auth_iv_data,
- param->auth_iv.data,
- param->auth_iv.length);
-#endif
+
/* We're happy */
*session_out = (intptr_t)session;
*status = ODP_CRYPTO_SES_ERR_NONE;
@@ -1623,42 +1618,6 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl)
return (uint64_t)hdl;
}
-odp_packet_t odp_crypto_packet_from_event(odp_event_t ev)
-{
- /* This check not mandated by the API specification */
- _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET);
- _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO);
-
- return odp_packet_from_event(ev);
-}
-
-odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt)
-{
- return odp_packet_to_event(pkt);
-}
-
-static
-odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt)
-{
- odp_packet_hdr_t *hdr = packet_hdr(pkt);
-
- return &hdr->crypto_op_result;
-}
-
-int odp_crypto_result(odp_crypto_packet_result_t *result,
- odp_packet_t packet)
-{
- odp_crypto_packet_result_t *op_result;
-
- _ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) == ODP_EVENT_PACKET_CRYPTO);
-
- op_result = get_op_result_from_packet(packet);
-
- memcpy(result, op_result, sizeof(*result));
-
- return 0;
-}
-
static uint8_t *crypto_prepare_digest(const crypto_session_entry_t *session,
odp_packet_t pkt,
const odp_crypto_packet_op_param_t *param,
@@ -1718,17 +1677,8 @@ static void crypto_fill_aead_param(const crypto_session_entry_t *session,
iv_ptr++;
}
-#if ODP_DEPRECATED_API
- if (param->cipher_iv_ptr)
- memcpy(iv_ptr, param->cipher_iv_ptr, iv_len);
- else if (session->p.cipher_iv.data)
- memcpy(iv_ptr, session->cipher_iv_data, iv_len);
- else
- _ODP_ASSERT(iv_len == 0);
-#else
_ODP_ASSERT(iv_len == 0 || param->cipher_iv_ptr != NULL);
memcpy(iv_ptr, param->cipher_iv_ptr, iv_len);
-#endif
op->sym->aead.data.offset = param->cipher_range.offset;
op->sym->aead.data.length = param->cipher_range.length;
@@ -1752,29 +1702,6 @@ static void crypto_fill_sym_param(const crypto_session_entry_t *session,
&op->sym->auth.digest.phys_addr);
}
-#if ODP_DEPRECATED_API
- if (param->cipher_iv_ptr) {
- iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET);
- memcpy(iv_ptr, param->cipher_iv_ptr, cipher_iv_len);
- } else if (session->p.cipher_iv.data) {
- iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET);
- memcpy(iv_ptr, session->cipher_iv_data, cipher_iv_len);
- } else {
- _ODP_ASSERT(cipher_iv_len == 0);
- }
-
- if (param->auth_iv_ptr) {
- iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
- IV_OFFSET + MAX_IV_LENGTH);
- memcpy(iv_ptr, param->auth_iv_ptr, auth_iv_len);
- } else if (session->p.auth_iv.data) {
- iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
- IV_OFFSET + MAX_IV_LENGTH);
- memcpy(iv_ptr, session->auth_iv_data, auth_iv_len);
- } else {
- _ODP_ASSERT(auth_iv_len == 0);
- }
-#else
_ODP_ASSERT(cipher_iv_len == 0 || param->cipher_iv_ptr != NULL);
_ODP_ASSERT(auth_iv_len == 0 || param->auth_iv_ptr != NULL);
iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET);
@@ -1784,7 +1711,6 @@ static void crypto_fill_sym_param(const crypto_session_entry_t *session,
iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET + MAX_IV_LENGTH);
memcpy(iv_ptr, param->auth_iv_ptr, auth_iv_len);
}
-#endif
op->sym->cipher.data.offset = param->cipher_range.offset;
op->sym->cipher.data.length = param->cipher_range.length;
@@ -1865,6 +1791,9 @@ static odp_packet_t get_output_packet(const crypto_session_entry_t *session,
{
int rc;
+ if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC))
+ return pkt_in;
+
if (odp_likely(pkt_in == pkt_out))
return pkt_out;
@@ -2145,7 +2074,7 @@ static void op_finish(crypto_op_t *op)
/* Fill in result */
packet_subtype_set(pkt, ODP_EVENT_PACKET_CRYPTO);
- op_result = get_op_result_from_packet(pkt);
+ op_result = &packet_hdr(pkt)->crypto_op_result;
op_result->cipher_status.alg_err = rc_cipher;
op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
op_result->auth_status.alg_err = rc_auth;
@@ -2188,6 +2117,10 @@ int odp_crypto_operation(odp_crypto_op_param_t *param,
odp_crypto_op_result_t local_result;
int rc;
+ if (((crypto_session_entry_t *)(intptr_t)param->session)->p.op_type !=
+ ODP_CRYPTO_OP_TYPE_LEGACY)
+ return -1;
+
packet_param.session = param->session;
packet_param.cipher_iv_ptr = param->cipher_iv_ptr;
packet_param.auth_iv_ptr = param->auth_iv_ptr;
@@ -2268,7 +2201,8 @@ int odp_crypto_op_enq(const odp_packet_t pkt_in[],
_ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode);
_ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue);
- out_pkts[i] = pkt_out[i];
+ if (session->p.op_type != ODP_CRYPTO_OP_TYPE_BASIC)
+ out_pkts[i] = pkt_out[i];
}
num_pkt = odp_crypto_int(pkt_in, out_pkts, param, num_pkt);
diff --git a/platform/linux-dpdk/odp_event.c b/platform/linux-dpdk/odp_event.c
index 24853f83e..625b6eefe 100644
--- a/platform/linux-dpdk/odp_event.c
+++ b/platform/linux-dpdk/odp_event.c
@@ -1,5 +1,5 @@
/* Copyright (c) 2015-2018, Linaro Limited
- * Copyright (c) 2020-2022, Nokia
+ * Copyright (c) 2020-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -18,6 +18,7 @@
#include <odp_debug_internal.h>
#include <odp_packet_internal.h>
#include <odp_event_internal.h>
+#include <odp_event_validation_internal.h>
#include <odp_event_vector_internal.h>
/* Inlined API functions */
@@ -42,13 +43,15 @@ _odp_event_inline_offset ODP_ALIGNED_CACHE = {
#include <odp/visibility_end.h>
-void odp_event_free(odp_event_t event)
+static inline void event_free(odp_event_t event, _odp_ev_id_t id)
{
switch (odp_event_type(event)) {
case ODP_EVENT_BUFFER:
+ _odp_buffer_validate(odp_buffer_from_event(event), id);
odp_buffer_free(odp_buffer_from_event(event));
break;
case ODP_EVENT_PACKET:
+ _odp_packet_validate(odp_packet_from_event(event), id);
odp_packet_free(odp_packet_from_event(event));
break;
case ODP_EVENT_PACKET_VECTOR:
@@ -76,17 +79,21 @@ void odp_event_free(odp_event_t event)
}
}
-void odp_event_free_multi(const odp_event_t event[], int num)
+void odp_event_free(odp_event_t event)
{
- int i;
+ event_free(event, _ODP_EV_EVENT_FREE);
+}
- for (i = 0; i < num; i++)
- odp_event_free(event[i]);
+void odp_event_free_multi(const odp_event_t event[], int num)
+{
+ for (int i = 0; i < num; i++)
+ event_free(event[i], _ODP_EV_EVENT_FREE_MULTI);
}
void odp_event_free_sp(const odp_event_t event[], int num)
{
- odp_event_free_multi(event, num);
+ for (int i = 0; i < num; i++)
+ event_free(event[i], _ODP_EV_EVENT_FREE_SP);
}
uint64_t odp_event_to_u64(odp_event_t hdl)
@@ -104,9 +111,9 @@ int odp_event_is_valid(odp_event_t event)
switch (odp_event_type(event)) {
case ODP_EVENT_BUFFER:
- /* Fall through */
+ return !_odp_buffer_validate(odp_buffer_from_event(event), _ODP_EV_EVENT_IS_VALID);
case ODP_EVENT_PACKET:
- /* Fall through */
+ return !_odp_packet_validate(odp_packet_from_event(event), _ODP_EV_EVENT_IS_VALID);
case ODP_EVENT_TIMEOUT:
/* Fall through */
#if ODP_DEPRECATED_API
diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c
index 329252f9c..b144835ea 100644
--- a/platform/linux-dpdk/odp_init.c
+++ b/platform/linux-dpdk/odp_init.c
@@ -1,5 +1,5 @@
/* Copyright (c) 2013-2018, Linaro Limited
- * Copyright (c) 2019-2021, Nokia
+ * Copyright (c) 2019-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -41,6 +41,7 @@ enum init_stage {
HASH_INIT,
THREAD_INIT,
POOL_INIT,
+ EVENT_VALIDATION_INIT,
STASH_INIT,
QUEUE_INIT,
SCHED_INIT,
@@ -426,6 +427,13 @@ static int term_global(enum init_stage stage)
}
/* Fall through */
+ case EVENT_VALIDATION_INIT:
+ if (_odp_event_validation_term_global()) {
+ _ODP_ERR("ODP event validation term failed.\n");
+ rc = -1;
+ }
+ /* Fall through */
+
case POOL_INIT:
if (_odp_pool_term_global()) {
_ODP_ERR("ODP buffer pool term failed.\n");
@@ -590,6 +598,12 @@ int odp_init_global(odp_instance_t *instance,
}
stage = POOL_INIT;
+ if (_odp_event_validation_init_global()) {
+ _ODP_ERR("ODP event validation init failed.\n");
+ goto init_failed;
+ }
+ stage = EVENT_VALIDATION_INIT;
+
if (_odp_stash_init_global()) {
_ODP_ERR("ODP stash init failed.\n");
goto init_failed;
diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c
index 16b940f2f..9b8755ce9 100644
--- a/platform/linux-dpdk/odp_packet.c
+++ b/platform/linux-dpdk/odp_packet.c
@@ -1,5 +1,5 @@
/* Copyright (c) 2013-2018, Linaro Limited
- * Copyright (c) 2019-2022, Nokia
+ * Copyright (c) 2019-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -24,6 +24,7 @@
#include <odp_chksum_internal.h>
#include <odp_debug_internal.h>
#include <odp_event_internal.h>
+#include <odp_event_validation_internal.h>
#include <odp_macros_internal.h>
#include <odp_packet_internal.h>
#include <odp_packet_io_internal.h>
@@ -66,6 +67,7 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = {
.subtype = offsetof(odp_packet_hdr_t, subtype),
.cls_mark = offsetof(odp_packet_hdr_t, cls_mark),
.ipsec_ctx = offsetof(odp_packet_hdr_t, ipsec_ctx),
+ .crypto_op = offsetof(odp_packet_hdr_t, crypto_op_result),
.buf_addr = offsetof(odp_packet_hdr_t, event_hdr.mb.buf_addr),
.data = offsetof(odp_packet_hdr_t, event_hdr.mb.data_off),
.pkt_len = offsetof(odp_packet_hdr_t, event_hdr.mb.pkt_len),
@@ -1144,6 +1146,9 @@ int odp_packet_is_valid(odp_packet_t pkt)
if (odp_event_type(ev) != ODP_EVENT_PACKET)
return 0;
+ if (odp_unlikely(_odp_packet_validate(pkt, _ODP_EV_PACKET_IS_VALID)))
+ return 0;
+
switch (odp_event_subtype(ev)) {
case ODP_EVENT_PACKET_BASIC:
/* Fall through */
diff --git a/platform/linux-dpdk/odp_packet_flags.c b/platform/linux-dpdk/odp_packet_flags.c
index 728df157a..345ac8488 100644
--- a/platform/linux-dpdk/odp_packet_flags.c
+++ b/platform/linux-dpdk/odp_packet_flags.c
@@ -72,10 +72,12 @@ void odp_packet_has_jumbo_set(odp_packet_t pkt, int val)
void odp_packet_has_vlan_set(odp_packet_t pkt, int val)
{
setflag(pkt, input_flags.vlan, val);
+ setflag(pkt, input_flags.vlan_qinq, 0);
}
void odp_packet_has_vlan_qinq_set(odp_packet_t pkt, int val)
{
+ setflag(pkt, input_flags.vlan, val);
setflag(pkt, input_flags.vlan_qinq, val);
}
diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c
index b0df7997a..d36f197c0 100644
--- a/platform/linux-dpdk/odp_pool.c
+++ b/platform/linux-dpdk/odp_pool.c
@@ -1,5 +1,5 @@
/* Copyright (c) 2013-2018, Linaro Limited
- * Copyright (c) 2019-2022, Nokia
+ * Copyright (c) 2019-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -16,6 +16,8 @@
#include <odp_buffer_internal.h>
#include <odp_config_internal.h>
#include <odp_debug_internal.h>
+#include <odp_event_internal.h>
+#include <odp_event_validation_internal.h>
#include <odp_event_vector_internal.h>
#include <odp_init_internal.h>
#include <odp_libconfig_internal.h>
@@ -380,6 +382,12 @@ odp_dpdk_mbuf_ctor(struct rte_mempool *mp,
pkt_hdr->uarea_addr = uarea;
}
+ /* Initialize data endmark */
+ if (mb_ctor_arg->type == ODP_POOL_BUFFER || mb_ctor_arg->type == ODP_POOL_PACKET) {
+ mb->buf_len -= _ODP_EV_ENDMARK_SIZE;
+ _odp_event_endmark_set(_odp_event_from_mbuf(mb));
+ }
+
/* Initialize event vector metadata */
if (mb_ctor_arg->type == ODP_POOL_VECTOR) {
odp_event_vector_hdr_t *vect_hdr = (void *)raw_mbuf;
@@ -679,6 +687,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
uint32_t buf_align, blk_size, headroom, tailroom, min_seg_len;
uint32_t max_len, min_align;
uint32_t uarea_size = 0;
+ uint32_t trailer_size = 0;
int8_t event_type;
char pool_name[ODP_POOL_NAME_LEN];
char rte_name[RTE_MEMPOOL_NAMESIZE];
@@ -709,7 +718,8 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
switch (type) {
case ODP_POOL_BUFFER:
buf_align = params->buf.align;
- blk_size = params->buf.size;
+ trailer_size = _ODP_EV_ENDMARK_SIZE;
+ blk_size = params->buf.size + trailer_size;
cache_size = params->buf.cache_size;
uarea_size = params->buf.uarea_size;
@@ -734,6 +744,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
case ODP_POOL_PACKET:
headroom = RTE_PKTMBUF_HEADROOM;
tailroom = CONFIG_PACKET_TAILROOM;
+ trailer_size = _ODP_EV_ENDMARK_SIZE;
min_seg_len = CONFIG_PACKET_SEG_LEN_MIN;
min_align = ODP_CONFIG_BUFFER_ALIGN_MIN;
cache_size = params->pkt.cache_size;
@@ -743,16 +754,16 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
blk_size = params->pkt.seg_len;
if (params->pkt.len > blk_size)
blk_size = params->pkt.len;
- /* Make sure at least one max len packet fits in the
- * pool.
- */
+
+ /* Make sure at least one max len packet fits in the pool */
max_len = 0;
if (params->pkt.max_len != 0)
max_len = params->pkt.max_len;
if ((max_len + blk_size) / blk_size > params->pkt.num)
blk_size = (max_len + params->pkt.num) /
params->pkt.num;
- blk_size = _ODP_ROUNDUP_ALIGN(headroom + blk_size + tailroom, min_align);
+ blk_size = _ODP_ROUNDUP_ALIGN(headroom + blk_size + tailroom + trailer_size,
+ min_align);
/* Segment size minus headroom might be rounded down by the driver (e.g.
* ixgbe) to the nearest multiple of 1024. Round it up here to make sure the
@@ -827,7 +838,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
mp = rte_pktmbuf_pool_create(rte_name, num, cache_size,
priv_size, data_room_size,
rte_socket_id());
- pool->seg_len = data_room_size;
+ pool->seg_len = data_room_size - trailer_size;
} else {
unsigned int priv_size;
@@ -855,6 +866,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
pool->type = type;
pool->type_2 = type_2;
pool->params = *params;
+ pool->trailer_size = trailer_size;
if (reserve_uarea(pool, uarea_size, num)) {
_ODP_ERR("User area SHM reserve failed\n");
@@ -1157,7 +1169,7 @@ int odp_pool_ext_capability(odp_pool_type_t type,
capa->pkt.max_num_buf = _odp_pool_glb->config.pkt_max_num;
capa->pkt.max_buf_size = MAX_SIZE;
capa->pkt.odp_header_size = SIZEOF_OBJHDR + sizeof(odp_packet_hdr_t);
- capa->pkt.odp_trailer_size = 0;
+ capa->pkt.odp_trailer_size = _ODP_EV_ENDMARK_SIZE;
capa->pkt.min_mem_align = ODP_CACHE_LINE_SIZE;
capa->pkt.min_buf_align = ODP_CACHE_LINE_SIZE;
capa->pkt.min_head_align = EXT_MIN_HEAD_ALIGN;
@@ -1326,12 +1338,13 @@ odp_pool_t odp_pool_ext_create(const char *name,
pool->ext_param = *params;
pool->ext_head_offset = hdr_size;
+ pool->trailer_size = _ODP_EV_ENDMARK_SIZE;
pool->num = num;
pool->num_populated = 0;
pool->params.pkt.uarea_size = params->pkt.uarea_size;
pool->params.type = params->type;
pool->pool_ext = 1;
- pool->seg_len = blk_size;
+ pool->seg_len = blk_size - pool->trailer_size;
pool->type = params->type;
strcpy(pool->name, pool_name);
@@ -1449,7 +1462,7 @@ int odp_pool_ext_populate(odp_pool_t pool_hdl, void *buf[], uint32_t buf_size,
mb_ctor_arg.seg_buf_offset = sizeof(odp_packet_hdr_t) +
params->pkt.app_header_size;
- mb_ctor_arg.seg_buf_size = pool->seg_len;
+ mb_ctor_arg.seg_buf_size = pool->seg_len + pool->trailer_size;
mb_ctor_arg.type = params->type;
mb_ctor_arg.event_type = pool->type;
mb_ctor_arg.pool = pool;
diff --git a/platform/linux-dpdk/odp_queue_basic.c b/platform/linux-dpdk/odp_queue_basic.c
index a83164e2f..ebe86ef1b 100644
--- a/platform/linux-dpdk/odp_queue_basic.c
+++ b/platform/linux-dpdk/odp_queue_basic.c
@@ -1,43 +1,46 @@
/* Copyright (c) 2013-2018, Linaro Limited
- * Copyright (c) 2021, Nokia
+ * Copyright (c) 2021-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
-#include <odp/api/queue.h>
-#include <odp_queue_basic_internal.h>
-#include <odp_queue_if.h>
-#include <odp/api/std_types.h>
#include <odp/api/align.h>
-#include <odp_pool_internal.h>
-#include <odp_init_internal.h>
-#include <odp/api/shared_memory.h>
-#include <odp/api/schedule.h>
-#include <odp_schedule_if.h>
-#include <odp_config_internal.h>
-#include <odp_packet_io_internal.h>
-#include <odp_debug_internal.h>
-#include <odp_macros_internal.h>
#include <odp/api/hints.h>
+#include <odp/api/packet_io.h>
+#include <odp/api/queue.h>
+#include <odp/api/schedule.h>
+#include <odp/api/shared_memory.h>
+#include <odp/api/std_types.h>
#include <odp/api/sync.h>
-#include <odp/api/plat/sync_inlines.h>
+#include <odp/api/ticketlock.h>
#include <odp/api/traffic_mngr.h>
-#include <odp_libconfig_internal.h>
-#include <odp_timer_internal.h>
+
#include <odp/api/plat/queue_inline_types.h>
+#include <odp/api/plat/sync_inlines.h>
+#include <odp/api/plat/ticketlock_inlines.h>
+
+#include <odp_config_internal.h>
+#include <odp_debug_internal.h>
+#include <odp_event_internal.h>
#include <odp_global_data.h>
+#include <odp_init_internal.h>
+#include <odp_libconfig_internal.h>
+#include <odp_macros_internal.h>
+#include <odp_packet_io_internal.h>
+#include <odp_pool_internal.h>
#include <odp_queue_basic_internal.h>
-#include <odp_event_internal.h>
+#include <odp_queue_if.h>
+#include <odp_schedule_if.h>
+#include <odp_timer_internal.h>
+
+#include <inttypes.h>
+#include <string.h>
-#include <odp/api/plat/ticketlock_inlines.h>
#define LOCK(queue_ptr) odp_ticketlock_lock(&((queue_ptr)->lock))
#define UNLOCK(queue_ptr) odp_ticketlock_unlock(&((queue_ptr)->lock))
#define LOCK_INIT(queue_ptr) odp_ticketlock_init(&((queue_ptr)->lock))
-#include <string.h>
-#include <inttypes.h>
-
#define MIN_QUEUE_SIZE (32 - 1)
#define MAX_QUEUE_SIZE ((1 * 1024 * 1024) - 1)
diff --git a/platform/linux-dpdk/odp_shared_memory.c b/platform/linux-dpdk/odp_shared_memory.c
index 470b78eb5..871a22ed7 100644
--- a/platform/linux-dpdk/odp_shared_memory.c
+++ b/platform/linux-dpdk/odp_shared_memory.c
@@ -1,5 +1,5 @@
/* Copyright (c) 2017-2018, Linaro Limited
- * Copyright (c) 2021-2022, Nokia
+ * Copyright (c) 2021-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -8,7 +8,6 @@
#include <odp_posix_extensions.h>
#include <odp/api/debug.h>
-#include <odp/api/deprecated.h>
#include <odp/api/shared_memory.h>
#include <odp/api/spinlock.h>
@@ -32,14 +31,7 @@
#include <rte_memzone.h>
/* Supported ODP_SHM_* flags */
-#if ODP_DEPRECATED_API
- #define DEPRECATED_SHM_FLAGS (ODP_SHM_SW_ONLY)
-#else
- #define DEPRECATED_SHM_FLAGS 0
-#endif
-
-#define SUPPORTED_SHM_FLAGS (ODP_SHM_EXPORT | ODP_SHM_HP | \
- ODP_SHM_SINGLE_VA | DEPRECATED_SHM_FLAGS)
+#define SUPPORTED_SHM_FLAGS (ODP_SHM_EXPORT | ODP_SHM_HP | ODP_SHM_SINGLE_VA)
#define SHM_MAX_ALIGN (0x80000000)
#define SHM_BLOCK_NAME "%" PRIu64 "-%d-%s"
@@ -299,8 +291,7 @@ odp_shm_t odp_shm_reserve(const char *name, uint64_t size, uint64_t align,
block->type = SHM_TYPE_LOCAL;
block->size = size;
- /* Note: ODP_SHM_SW_ONLY/ODP_SHM_PROC/ODP_SHM_SINGLE_VA flags are
- * currently ignored. */
+ /* Note: ODP_SHM_PROC/ODP_SHM_SINGLE_VA flags are currently ignored. */
shm_zone(mz)->flags = flags;
odp_spinlock_unlock(&shm_tbl->lock);
diff --git a/platform/linux-dpdk/odp_timer.c b/platform/linux-dpdk/odp_timer.c
index 5296a9d3c..1021a96cd 100644
--- a/platform/linux-dpdk/odp_timer.c
+++ b/platform/linux-dpdk/odp_timer.c
@@ -1,5 +1,5 @@
/* Copyright (c) 2018, Linaro Limited
- * Copyright (c) 2019-2022, Nokia
+ * Copyright (c) 2019-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -7,6 +7,7 @@
#include <odp_posix_extensions.h>
+#include <odp/api/deprecated.h>
#include <odp/api/queue.h>
#include <odp/api/shared_memory.h>
#include <odp/api/std.h>
@@ -1015,14 +1016,14 @@ retry:
return ODP_TIMER_SUCCESS;
}
-int odp_timer_set_abs(odp_timer_t timer_hdl, uint64_t abs_tick,
- odp_event_t *tmo_ev)
+int ODP_DEPRECATE(odp_timer_set_abs)(odp_timer_t timer_hdl, uint64_t abs_tick,
+ odp_event_t *tmo_ev)
{
return timer_set(timer_hdl, abs_tick, tmo_ev, 1);
}
-int odp_timer_set_rel(odp_timer_t timer_hdl, uint64_t rel_tick,
- odp_event_t *tmo_ev)
+int ODP_DEPRECATE(odp_timer_set_rel)(odp_timer_t timer_hdl, uint64_t rel_tick,
+ odp_event_t *tmo_ev)
{
return timer_set(timer_hdl, rel_tick, tmo_ev, 0);
}
diff --git a/platform/linux-dpdk/test/alternate-timer.conf b/platform/linux-dpdk/test/alternate-timer.conf
index 0b326f259..d976d05fe 100644
--- a/platform/linux-dpdk/test/alternate-timer.conf
+++ b/platform/linux-dpdk/test/alternate-timer.conf
@@ -1,6 +1,6 @@
# Mandatory fields
odp_implementation = "linux-dpdk"
-config_file_version = "0.1.18"
+config_file_version = "0.1.21"
timer: {
# Enable alternate DPDK timer implementation
diff --git a/platform/linux-dpdk/test/crypto.conf b/platform/linux-dpdk/test/crypto.conf
index f3d642963..1fcaf658e 100644
--- a/platform/linux-dpdk/test/crypto.conf
+++ b/platform/linux-dpdk/test/crypto.conf
@@ -1,6 +1,6 @@
# Mandatory fields
odp_implementation = "linux-dpdk"
-config_file_version = "0.1.18"
+config_file_version = "0.1.21"
system: {
# One crypto queue pair is required per thread for lockless operation
diff --git a/platform/linux-dpdk/test/process-mode.conf b/platform/linux-dpdk/test/process-mode.conf
index b95f50ea2..7a98cef10 100644
--- a/platform/linux-dpdk/test/process-mode.conf
+++ b/platform/linux-dpdk/test/process-mode.conf
@@ -1,6 +1,6 @@
# Mandatory fields
odp_implementation = "linux-dpdk"
-config_file_version = "0.1.18"
+config_file_version = "0.1.21"
dpdk: {
process_mode_memory_mb = 1024
diff --git a/platform/linux-dpdk/test/sched-basic.conf b/platform/linux-dpdk/test/sched-basic.conf
index 181136c33..e3d9dd304 100644
--- a/platform/linux-dpdk/test/sched-basic.conf
+++ b/platform/linux-dpdk/test/sched-basic.conf
@@ -1,9 +1,10 @@
# Mandatory fields
odp_implementation = "linux-dpdk"
-config_file_version = "0.1.18"
+config_file_version = "0.1.21"
# Test scheduler with an odd spread value and without dynamic load balance
sched_basic: {
prio_spread = 3
load_balance = 0
+ order_stash_size = 0
}
diff --git a/platform/linux-dpdk/test/stash-custom.conf b/platform/linux-dpdk/test/stash-custom.conf
new file mode 100644
index 000000000..5e3ea5cdf
--- /dev/null
+++ b/platform/linux-dpdk/test/stash-custom.conf
@@ -0,0 +1,8 @@
+# Mandatory fields
+odp_implementation = "linux-dpdk"
+config_file_version = "0.1.21"
+
+# Test overflow safe stash variant
+stash: {
+ strict_size = 0
+}
diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am
index e762148aa..d49a2138b 100644
--- a/platform/linux-generic/Makefile.am
+++ b/platform/linux-generic/Makefile.am
@@ -34,9 +34,11 @@ odpapiplatinclude_HEADERS = \
include/odp/api/plat/buffer_inline_types.h \
include/odp/api/plat/byteorder_inlines.h \
include/odp/api/plat/cpu_inlines.h \
+ include/odp/api/plat/crypto_inlines.h \
include/odp/api/plat/debug_inlines.h \
include/odp/api/plat/event_inlines.h \
include/odp/api/plat/event_inline_types.h \
+ include/odp/api/plat/event_validation_external.h \
include/odp/api/plat/event_vector_inline_types.h \
include/odp/api/plat/hash_inlines.h \
include/odp/api/plat/ipsec_inlines.h \
@@ -70,11 +72,13 @@ odpapiabiarchinclude_HEADERS += \
include-abi/odp/api/abi/atomic.h \
include-abi/odp/api/abi/barrier.h \
include-abi/odp/api/abi/buffer.h \
+ include-abi/odp/api/abi/buffer_types.h \
include-abi/odp/api/abi/byteorder.h \
include-abi/odp/api/abi/classification.h \
include-abi/odp/api/abi/comp.h \
include-abi/odp/api/abi/cpumask.h \
include-abi/odp/api/abi/crypto.h \
+ include-abi/odp/api/abi/crypto_types.h \
include-abi/odp/api/abi/debug.h \
include-abi/odp/api/abi/dma_types.h \
include-abi/odp/api/abi/errno.h \
@@ -129,6 +133,7 @@ noinst_HEADERS = \
include/odp_debug_internal.h \
include/odp_errno_define.h \
include/odp_event_internal.h \
+ include/odp_event_validation_internal.h \
include/odp_fdserver_internal.h \
include/odp_forward_typedefs_internal.h \
include/odp_global_data.h \
@@ -164,6 +169,8 @@ noinst_HEADERS = \
include/odp_ring_common.h \
include/odp_ring_internal.h \
include/odp_ring_mpmc_internal.h \
+ include/odp_ring_mpmc_u32_internal.h \
+ include/odp_ring_mpmc_u64_internal.h \
include/odp_ring_ptr_internal.h \
include/odp_ring_spsc_internal.h \
include/odp_ring_st_internal.h \
@@ -205,6 +212,7 @@ __LIB__libodp_linux_la_SOURCES = \
odp_dma.c \
odp_errno.c \
odp_event.c \
+ odp_event_validation.c \
odp_fdserver.c \
odp_hash_crc_gen.c \
odp_impl.c \
@@ -278,16 +286,22 @@ if WITH_ARMV8_CRYPTO
__LIB__libodp_linux_la_SOURCES += \
arch/aarch64/odp_crypto_armv8.c
else
+if WITH_IPSECMB_CRYPTO
+__LIB__libodp_linux_la_SOURCES += \
+ odp_crypto_ipsecmb.c
+else
__LIB__libodp_linux_la_SOURCES += \
odp_crypto_null.c
endif
endif
+endif
if ODP_ABI_COMPAT
__LIB__libodp_linux_la_SOURCES += \
odp_atomic_api.c \
odp_buffer_api.c \
odp_byteorder_api.c \
odp_cpu_api.c \
+ odp_crypto_api.c \
odp_event_api.c \
odp_hash_api.c \
odp_ipsec_api.c \
@@ -438,6 +452,7 @@ __LIB__libodp_linux_la_LIBADD += $(DPDK_LIBS_LIBODP)
__LIB__libodp_linux_la_LIBADD += $(PTHREAD_LIBS)
__LIB__libodp_linux_la_LIBADD += $(TIMER_LIBS)
__LIB__libodp_linux_la_LIBADD += $(LIBXDP_LIBS)
+__LIB__libodp_linux_la_LIBADD += $(IPSEC_MB_LIBS)
if ODP_PKTIO_PCAP
__LIB__libodp_linux_la_LIBADD += $(PCAP_LIBS)
diff --git a/platform/linux-generic/README b/platform/linux-generic/README
index 8f41d1d45..138e6040c 100644
--- a/platform/linux-generic/README
+++ b/platform/linux-generic/README
@@ -1,5 +1,5 @@
Copyright (c) 2014-2018, Linaro Limited
-Copyright (c) 2019, Nokia
+Copyright (c) 2019-2023, Nokia
All rights reserved.
SPDX-License-Identifier: BSD-3-Clause
@@ -52,7 +52,29 @@ SPDX-License-Identifier: BSD-3-Clause
Note that there may be issues with the quality or security of rdrand and
rdseed. [2]
-6. References
+6. Event validation
+ ODP linux-generic implementation supports additional fast path event
+ validity checks which are disabled by default to minimize overhead. These
+ checks can be enabled with --enable-event-validation [abort/warn] or
+ --enabled-debug=full configuration options.
+
+ Event validation adds additional endmark data to ODP buffers and packets,
+ which is used to detect data writes outside allowed areas. Endmarks are
+ checked by the implementation each time application calls one the following
+ API functions:
+ - odp_buffer_free() / odp_buffer_free_multi()
+ - odp_buffer_is_valid()
+ - odp_event_free() / odp_event_free_multi() / odp_event_free_sp()
+ - odp_event_is_valid()
+ - odp_packet_free() / odp_packet_free_multi() / odp_packet_free_sp()
+ - odp_packet_is_valid()
+ - odp_queue_enq() / odp_queue_enq_multi()
+
+ Event validation can function in two modes: abort (default) and warn. In
+ abort mode the application is terminated immediately if an event validity
+ check fails. In warn mode only an error log message is printed.
+
+7. References
[1] Intel Digital Random Number Generator (DRNG) Software Implementation
Guide. John P Mechalas, 17 October 2018.
https://www.intel.com/content/www/us/en/developer/articles/guide/intel-digital-random-number-generator-drng-software-implementation-guide.html
diff --git a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c
index 11fadc971..67ae6a389 100644
--- a/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c
+++ b/platform/linux-generic/arch/aarch64/odp_crypto_armv8.c
@@ -1,6 +1,6 @@
/* Copyright (c) 2014-2018, Linaro Limited
* Copyright (c) 2021, ARM Limited
- * Copyright (c) 2022, Nokia
+ * Copyright (c) 2022-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -38,6 +38,10 @@
#define ARM_CRYPTO_MAX_DATA_LENGTH 65536
#define ARM_CRYPTO_MAX_DIGEST_LENGTH 16
+#define AES_GCM_IV_LEN 12
+ODP_STATIC_ASSERT(AES_GCM_IV_LEN <= ARM_CRYPTO_MAX_IV_LENGTH,
+ "AES_GCM_IV_LEN exceeds ARM_CRYPTO_MAX_IV_LENGTH");
+
/*
* ARM crypto library may read up to 15 bytes past the end of input
* data and AAD and write up to 15 bytes past the end of output data.
@@ -70,9 +74,9 @@ static const odp_crypto_cipher_capability_t cipher_capa_null[] = {
#ifdef __ARM_FEATURE_AES
static const odp_crypto_cipher_capability_t cipher_capa_aes_gcm[] = {
-{.key_len = 16, .iv_len = 12},
-{.key_len = 24, .iv_len = 12},
-{.key_len = 32, .iv_len = 12} };
+{.key_len = 16, .iv_len = AES_GCM_IV_LEN},
+{.key_len = 24, .iv_len = AES_GCM_IV_LEN},
+{.key_len = 32, .iv_len = AES_GCM_IV_LEN} };
#endif
/*
@@ -111,18 +115,11 @@ struct odp_crypto_generic_session_t {
odp_crypto_session_param_t p;
struct {
-#if ODP_DEPRECATED_API
- /* Copy of session IV data */
- uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH];
-#endif
uint8_t key_data[ARM_CRYPTO_MAX_CIPHER_KEY_LENGTH];
} cipher;
struct {
uint8_t key[ARM_CRYPTO_MAX_AUTH_KEY_LENGTH];
-#if ODP_DEPRECATED_API
- uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH];
-#endif
} auth;
crypto_func_t func;
@@ -176,21 +173,13 @@ void free_session(odp_crypto_generic_session_t *session)
odp_spinlock_unlock(&global->lock);
}
-static
-odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt)
-{
- odp_packet_hdr_t *hdr = packet_hdr(pkt);
-
- return &hdr->crypto_op_result;
-}
-
static inline void set_crypto_op_result(odp_packet_t pkt,
odp_crypto_alg_err_t cipher_err,
odp_crypto_alg_err_t auth_err)
{
odp_crypto_packet_result_t *op_result;
- op_result = get_op_result_from_packet(pkt);
+ op_result = &packet_hdr(pkt)->crypto_op_result;
op_result->cipher_status.alg_err = cipher_err;
op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
op_result->auth_status.alg_err = auth_err;
@@ -235,8 +224,8 @@ void aes_gcm_encrypt(odp_packet_t pkt,
.d = {0, 0}
}
};
- uint8_t *iv_ptr;
- uint64_t iv_bit_length = session->p.cipher_iv_len * 8;
+ uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH];
+ uint64_t iv_bit_length = AES_GCM_IV_LEN * 8;
uint64_t plaintext_bit_length = param->cipher_range.length * 8;
uint64_t aad_bit_length = session->p.auth_aad_len * 8;
uint32_t in_pos = param->cipher_range.offset;
@@ -252,21 +241,13 @@ void aes_gcm_encrypt(odp_packet_t pkt,
goto err;
}
-#if ODP_DEPRECATED_API
- if (param->cipher_iv_ptr)
- iv_ptr = param->cipher_iv_ptr;
- else if (session->p.cipher_iv.data)
- iv_ptr = session->cipher.iv_data;
- else
- goto err;
-#else
- iv_ptr = param->cipher_iv_ptr;
- _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL);
-#endif
+ /* The crypto lib may read 16 bytes. Copy to a big enough buffer */
+ _ODP_ASSERT(param->cipher_iv_ptr != NULL);
+ memcpy(iv_data, param->cipher_iv_ptr, AES_GCM_IV_LEN);
cs.constants = &session->cc;
- rc = armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs);
+ rc = armv8_aes_gcm_set_counter(iv_data, iv_bit_length, &cs);
if (odp_unlikely(rc)) {
_ODP_DBG("ARM Crypto: Failure while setting nonce\n");
goto err;
@@ -335,9 +316,9 @@ void aes_gcm_decrypt(odp_packet_t pkt,
.d = {0, 0}
}
};
- uint8_t *iv_ptr;
+ uint8_t iv_data[ARM_CRYPTO_MAX_IV_LENGTH];
uint8_t tag[AES_GCM_TAG_LEN];
- uint64_t iv_bit_length = session->p.cipher_iv_len * 8;
+ uint64_t iv_bit_length = AES_GCM_IV_LEN * 8;
uint64_t plaintext_bit_length = param->cipher_range.length * 8;
uint64_t aad_bit_length = session->p.auth_aad_len * 8;
uint32_t in_pos = param->cipher_range.offset;
@@ -352,21 +333,13 @@ void aes_gcm_decrypt(odp_packet_t pkt,
goto err;
}
-#if ODP_DEPRECATED_API
- if (param->cipher_iv_ptr)
- iv_ptr = param->cipher_iv_ptr;
- else if (session->p.cipher_iv.data)
- iv_ptr = session->cipher.iv_data;
- else
- goto err;
-#else
- iv_ptr = param->cipher_iv_ptr;
- _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL);
-#endif
+ /* The crypto lib may read 16 bytes. Copy to a big enough buffer */
+ _ODP_ASSERT(param->cipher_iv_ptr != NULL);
+ memcpy(iv_data, param->cipher_iv_ptr, AES_GCM_IV_LEN);
cs.constants = &session->cc;
- rc = armv8_aes_gcm_set_counter(iv_ptr, iv_bit_length, &cs);
+ rc = armv8_aes_gcm_set_counter(iv_data, iv_bit_length, &cs);
if (odp_unlikely(rc)) {
_ODP_DBG("ARM Crypto: Failure while setting nonce\n");
goto err;
@@ -433,7 +406,7 @@ static int process_aes_gcm_param(odp_crypto_generic_session_t *session)
return -1;
/* Verify IV len is correct */
- if (12 != session->p.cipher_iv_len)
+ if (session->p.cipher_iv_len != AES_GCM_IV_LEN)
return -1;
if (ARM_CRYPTO_MAX_CIPHER_KEY_LENGTH < session->p.cipher_key.length)
@@ -555,6 +528,12 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param,
return -1;
}
+ if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) {
+ *status = ODP_CRYPTO_SES_ERR_PARAMS;
+ *session_out = ODP_CRYPTO_SESSION_INVALID;
+ return -1;
+ }
+
/* Allocate memory for this session */
session = alloc_session();
if (NULL == session) {
@@ -577,17 +556,6 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param,
goto err;
}
-#if ODP_DEPRECATED_API
- /* Copy IV data */
- if (session->p.cipher_iv.data)
- memcpy(session->cipher.iv_data, session->p.cipher_iv.data,
- session->p.cipher_iv.length);
-
- if (session->p.auth_iv.data)
- memcpy(session->auth.iv_data, session->p.auth_iv.data,
- session->p.auth_iv.length);
-#endif
-
/* Process based on cipher */
switch (param->cipher_alg) {
case ODP_CIPHER_ALG_NULL:
@@ -705,6 +673,10 @@ odp_crypto_operation(odp_crypto_op_param_t *param,
odp_crypto_op_result_t local_result;
int rc;
+ if (((odp_crypto_generic_session_t *)(intptr_t)param->session)->p.op_type !=
+ ODP_CRYPTO_OP_TYPE_LEGACY)
+ return -1;
+
packet_param.session = param->session;
packet_param.cipher_iv_ptr = param->cipher_iv_ptr;
packet_param.auth_iv_ptr = param->auth_iv_ptr;
@@ -877,35 +849,6 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl)
return (uint64_t)hdl;
}
-odp_packet_t odp_crypto_packet_from_event(odp_event_t ev)
-{
- /* This check not mandated by the API specification */
- _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET);
- _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO);
-
- return odp_packet_from_event(ev);
-}
-
-odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt)
-{
- return odp_packet_to_event(pkt);
-}
-
-int odp_crypto_result(odp_crypto_packet_result_t *result,
- odp_packet_t packet)
-{
- odp_crypto_packet_result_t *op_result;
-
- _ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) ==
- ODP_EVENT_PACKET_CRYPTO);
-
- op_result = get_op_result_from_packet(packet);
-
- memcpy(result, op_result, sizeof(*result));
-
- return 0;
-}
-
static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src)
{
int md_copy;
@@ -934,6 +877,9 @@ static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *sessio
{
int rc;
+ if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC))
+ return pkt_in;
+
if (odp_likely(pkt_in == pkt_out))
return pkt_out;
@@ -1018,7 +964,9 @@ int odp_crypto_op_enq(const odp_packet_t pkt_in[],
_ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode);
_ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue);
- pkt = pkt_out[i];
+ if (session->p.op_type != ODP_CRYPTO_OP_TYPE_BASIC)
+ pkt = pkt_out[i];
+
rc = crypto_int(pkt_in[i], &pkt, &param[i]);
if (rc < 0)
break;
diff --git a/platform/linux-generic/include-abi/odp/api/abi/buffer.h b/platform/linux-generic/include-abi/odp/api/abi/buffer.h
index 8239e15da..a6309fe39 100644
--- a/platform/linux-generic/include-abi/odp/api/abi/buffer.h
+++ b/platform/linux-generic/include-abi/odp/api/abi/buffer.h
@@ -1,4 +1,5 @@
/* Copyright (c) 2015-2018, Linaro Limited
+ * Copyright (c) 2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -7,7 +8,7 @@
/**
* @file
*
- * ODP buffer descriptor
+ * ODP buffer
*/
#ifndef ODP_API_ABI_BUFFER_H_
@@ -17,24 +18,9 @@
extern "C" {
#endif
-#include <odp/api/std_types.h>
-#include <odp/api/plat/strong_types.h>
-
-/** @ingroup odp_buffer
- * @{
- */
-
-typedef ODP_HANDLE_T(odp_buffer_t);
-
-#define ODP_BUFFER_INVALID _odp_cast_scalar(odp_buffer_t, 0)
-
-/* Inlined functions for non-ABI compat mode */
+/* Inlined API functions */
#include <odp/api/plat/buffer_inlines.h>
-/**
- * @}
- */
-
#ifdef __cplusplus
}
#endif
diff --git a/platform/linux-generic/include-abi/odp/api/abi/buffer_types.h b/platform/linux-generic/include-abi/odp/api/abi/buffer_types.h
new file mode 100644
index 000000000..1d54bab07
--- /dev/null
+++ b/platform/linux-generic/include-abi/odp/api/abi/buffer_types.h
@@ -0,0 +1,40 @@
+/* Copyright (c) 2015-2018, Linaro Limited
+ * Copyright (c) 2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODP buffer types
+ */
+
+#ifndef ODP_API_ABI_BUFFER_TYPES_H_
+#define ODP_API_ABI_BUFFER_TYPES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp/api/std_types.h>
+#include <odp/api/plat/strong_types.h>
+
+/** @ingroup odp_buffer
+ * @{
+ */
+
+typedef ODP_HANDLE_T(odp_buffer_t);
+
+#define ODP_BUFFER_INVALID _odp_cast_scalar(odp_buffer_t, 0)
+
+/**
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/include-abi/odp/api/abi/crypto.h b/platform/linux-generic/include-abi/odp/api/abi/crypto.h
index b57667e76..bef725c28 100644
--- a/platform/linux-generic/include-abi/odp/api/abi/crypto.h
+++ b/platform/linux-generic/include-abi/odp/api/abi/crypto.h
@@ -1,4 +1,5 @@
/* Copyright (c) 2015-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -18,18 +19,8 @@
extern "C" {
#endif
-/** @ingroup odp_crypto
- * @{
- */
-
-#define ODP_CRYPTO_SESSION_INVALID (0xffffffffffffffffULL)
-
-typedef uint64_t odp_crypto_session_t;
-typedef ODP_HANDLE_T(odp_crypto_compl_t);
-
-/**
- * @}
- */
+/* Inlined API functions */
+#include <odp/api/plat/crypto_inlines.h>
#ifdef __cplusplus
}
diff --git a/platform/linux-generic/include-abi/odp/api/abi/crypto_types.h b/platform/linux-generic/include-abi/odp/api/abi/crypto_types.h
new file mode 100644
index 000000000..a5cb43c5d
--- /dev/null
+++ b/platform/linux-generic/include-abi/odp/api/abi/crypto_types.h
@@ -0,0 +1,42 @@
+/* Copyright (c) 2015-2018, Linaro Limited
+ * Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODP crypto
+ */
+
+#ifndef ODP_API_ABI_CRYPTO_TYPES_H_
+#define ODP_API_ABI_CRYPTO_TYPES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp/api/std_types.h>
+
+#include <odp/api/plat/strong_types.h>
+
+/** @ingroup odp_crypto
+ * @{
+ */
+
+#define ODP_CRYPTO_SESSION_INVALID (0xffffffffffffffffULL)
+
+typedef uint64_t odp_crypto_session_t;
+typedef ODP_HANDLE_T(odp_crypto_compl_t);
+
+/**
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h
index 34d4b5675..75ef36cf3 100644
--- a/platform/linux-generic/include/odp/api/plat/buffer_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/buffer_inlines.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2019-2022, Nokia
+/* Copyright (c) 2019-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -7,11 +7,10 @@
#ifndef ODP_PLAT_BUFFER_INLINES_H_
#define ODP_PLAT_BUFFER_INLINES_H_
+#include <odp/api/buffer_types.h>
#include <odp/api/event.h>
#include <odp/api/pool_types.h>
-#include <odp/api/abi/buffer.h>
-
#include <odp/api/plat/buffer_inline_types.h>
#include <odp/api/plat/debug_inlines.h>
#include <odp/api/plat/event_inline_types.h>
diff --git a/platform/linux-generic/include/odp/api/plat/crypto_inlines.h b/platform/linux-generic/include/odp/api/plat/crypto_inlines.h
new file mode 100644
index 000000000..8e98d8580
--- /dev/null
+++ b/platform/linux-generic/include/odp/api/plat/crypto_inlines.h
@@ -0,0 +1,65 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_PLAT_CRYPTO_INLINES_H_
+#define ODP_PLAT_CRYPTO_INLINES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp/api/crypto_types.h>
+#include <odp/api/event.h>
+#include <odp/api/packet.h>
+
+#include <odp/api/plat/debug_inlines.h>
+#include <odp/api/plat/packet_inline_types.h>
+
+/** @cond _ODP_HIDE_FROM_DOXYGEN_ */
+
+#ifndef _ODP_NO_INLINE
+ /* Inline functions by default */
+ #define _ODP_INLINE static inline
+ #define odp_crypto_packet_from_event __odp_crypto_packet_from_event
+ #define odp_crypto_packet_to_event __odp_crypto_packet_to_event
+ #define odp_crypto_result __odp_crypto_result
+#else
+ #define _ODP_INLINE
+#endif
+
+_ODP_INLINE odp_packet_t odp_crypto_packet_from_event(odp_event_t ev)
+{
+ _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET);
+ _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO);
+
+ return odp_packet_from_event(ev);
+}
+
+_ODP_INLINE odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt)
+{
+ return odp_packet_to_event(pkt);
+}
+
+_ODP_INLINE int odp_crypto_result(odp_crypto_packet_result_t *result, odp_packet_t pkt)
+{
+ odp_crypto_packet_result_t *op_result;
+
+ _ODP_ASSERT(odp_packet_subtype(pkt) == ODP_EVENT_PACKET_CRYPTO);
+
+ op_result = _odp_pkt_get_ptr(pkt, odp_crypto_packet_result_t, crypto_op);
+
+ *result = *op_result;
+
+ return 0;
+}
+
+/** @endcond */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/include/odp/api/plat/event_validation_external.h b/platform/linux-generic/include/odp/api/plat/event_validation_external.h
new file mode 100644
index 000000000..7f5c0364f
--- /dev/null
+++ b/platform/linux-generic/include/odp/api/plat/event_validation_external.h
@@ -0,0 +1,111 @@
+/* Copyright (c) 2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODP event validation
+ *
+ * @warning These definitions are not part of ODP API, they are for
+ * implementation internal use only.
+ */
+
+#ifndef ODP_EVENT_VALIDATION_EXTERNAL_H_
+#define ODP_EVENT_VALIDATION_EXTERNAL_H_
+
+#include <odp/autoheader_external.h>
+
+#include <odp/api/buffer_types.h>
+#include <odp/api/event_types.h>
+#include <odp/api/hints.h>
+#include <odp/api/packet_types.h>
+
+/** @cond _ODP_HIDE_FROM_DOXYGEN_ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Enumerations for identifying ODP API functions */
+typedef enum {
+ _ODP_EV_BUFFER_FREE = 0,
+ _ODP_EV_BUFFER_FREE_MULTI,
+ _ODP_EV_BUFFER_IS_VALID,
+ _ODP_EV_EVENT_FREE,
+ _ODP_EV_EVENT_FREE_MULTI,
+ _ODP_EV_EVENT_FREE_SP,
+ _ODP_EV_EVENT_IS_VALID,
+ _ODP_EV_PACKET_FREE,
+ _ODP_EV_PACKET_FREE_MULTI,
+ _ODP_EV_PACKET_FREE_SP,
+ _ODP_EV_PACKET_IS_VALID,
+ _ODP_EV_QUEUE_ENQ,
+ _ODP_EV_QUEUE_ENQ_MULTI,
+ _ODP_EV_MAX
+} _odp_ev_id_t;
+
+/* Implementation internal event validation functions */
+#if _ODP_EVENT_VALIDATION
+
+int _odp_event_validate(odp_event_t event, _odp_ev_id_t id);
+
+int _odp_event_validate_multi(const odp_event_t event[], int num, _odp_ev_id_t id);
+
+int _odp_buffer_validate(odp_buffer_t buf, _odp_ev_id_t ev_id);
+
+int _odp_buffer_validate_multi(const odp_buffer_t buf[], int num, _odp_ev_id_t ev_id);
+
+int _odp_packet_validate(odp_packet_t pkt, _odp_ev_id_t ev_id);
+
+int _odp_packet_validate_multi(const odp_packet_t pkt[], int num, _odp_ev_id_t ev_id);
+
+#else
+
+static inline int _odp_event_validate(odp_event_t event ODP_UNUSED, _odp_ev_id_t ev_id ODP_UNUSED)
+{
+ return 0;
+}
+
+static inline int _odp_event_validate_multi(const odp_event_t event[] ODP_UNUSED,
+ int num ODP_UNUSED,
+ _odp_ev_id_t ev_id ODP_UNUSED)
+{
+ return 0;
+}
+
+static inline int _odp_buffer_validate(odp_buffer_t buf ODP_UNUSED, _odp_ev_id_t ev_id ODP_UNUSED)
+{
+ return 0;
+}
+
+static inline int _odp_buffer_validate_multi(const odp_buffer_t buf[] ODP_UNUSED,
+ int num ODP_UNUSED,
+ _odp_ev_id_t ev_id ODP_UNUSED)
+{
+ return 0;
+}
+
+static inline int _odp_packet_validate(odp_packet_t pkt ODP_UNUSED, _odp_ev_id_t ev_id ODP_UNUSED)
+{
+ return 0;
+}
+
+static inline int _odp_packet_validate_multi(const odp_packet_t pkt[] ODP_UNUSED,
+ int num ODP_UNUSED,
+ _odp_ev_id_t ev_id ODP_UNUSED)
+{
+ return 0;
+}
+
+#endif /* _ODP_EVENT_VALIDATION */
+
+#ifdef __cplusplus
+}
+#endif
+
+/** @endcond */
+
+#endif
diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h
index ae03457f9..6773b73ad 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h
@@ -53,6 +53,7 @@ typedef struct _odp_packet_inline_offset_t {
uint16_t subtype;
uint16_t cls_mark;
uint16_t ipsec_ctx;
+ uint16_t crypto_op;
} _odp_packet_inline_offset_t;
diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h
index 01d47d837..93e95e21c 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h
@@ -632,7 +632,8 @@ _ODP_INLINE uint32_t odp_packet_buf_size(odp_packet_buf_t pkt_buf)
odp_pool_t pool = _odp_pkt_get(pkt_buf, odp_pool_t, pool);
return _odp_pool_get(pool, uint32_t, ext_pkt_buf_size) -
- _odp_pool_get(pool, uint32_t, ext_head_offset);
+ _odp_pool_get(pool, uint32_t, ext_head_offset) -
+ _odp_pool_get(pool, uint32_t, trailer_size);
}
_ODP_INLINE void *odp_packet_buf_head(odp_packet_buf_t pkt_buf)
diff --git a/platform/linux-generic/include/odp/api/plat/pool_inline_types.h b/platform/linux-generic/include/odp/api/plat/pool_inline_types.h
index 02f59f982..fbff7eda7 100644
--- a/platform/linux-generic/include/odp/api/plat/pool_inline_types.h
+++ b/platform/linux-generic/include/odp/api/plat/pool_inline_types.h
@@ -30,6 +30,7 @@ typedef struct _odp_pool_inline_offset_t {
uint16_t index;
uint16_t seg_len;
uint16_t uarea_size;
+ uint16_t trailer_size;
uint16_t ext_head_offset;
uint16_t ext_pkt_buf_size;
diff --git a/platform/linux-generic/include/odp/api/plat/queue_inlines.h b/platform/linux-generic/include/odp/api/plat/queue_inlines.h
index 22673a887..609c0c9e4 100644
--- a/platform/linux-generic/include/odp/api/plat/queue_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/queue_inlines.h
@@ -1,4 +1,5 @@
/* Copyright (c) 2018, Linaro Limited
+ * Copyright (c) 2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -7,6 +8,9 @@
#ifndef ODP_PLAT_QUEUE_INLINES_H_
#define ODP_PLAT_QUEUE_INLINES_H_
+#include <odp/api/hints.h>
+
+#include <odp/api/plat/event_validation_external.h>
#include <odp/api/plat/queue_inline_types.h>
/** @cond _ODP_HIDE_FROM_DOXYGEN_ */
@@ -37,12 +41,18 @@ _ODP_INLINE void *odp_queue_context(odp_queue_t handle)
_ODP_INLINE int odp_queue_enq(odp_queue_t queue, odp_event_t ev)
{
+ if (odp_unlikely(_odp_event_validate(ev, _ODP_EV_QUEUE_ENQ)))
+ return -1;
+
return _odp_queue_api->queue_enq(queue, ev);
}
_ODP_INLINE int odp_queue_enq_multi(odp_queue_t queue,
const odp_event_t events[], int num)
{
+ if (odp_unlikely(_odp_event_validate_multi(events, num, _ODP_EV_QUEUE_ENQ_MULTI)))
+ return -1;
+
return _odp_queue_api->queue_enq_multi(queue, events, num);
}
diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h
index d3d09abf4..e4f8d6d6d 100644
--- a/platform/linux-generic/include/odp_config_internal.h
+++ b/platform/linux-generic/include/odp_config_internal.h
@@ -1,5 +1,5 @@
/* Copyright (c) 2016-2018, Linaro Limited
- * Copyright (c) 2019-2021, Nokia
+ * Copyright (c) 2019-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -66,7 +66,7 @@ extern "C" {
/*
* Maximum number of stashes
*/
-#define CONFIG_MAX_STASHES 128
+#define CONFIG_MAX_STASHES 2048
/*
* Maximum number of packet IO resources
@@ -134,10 +134,10 @@ extern "C" {
/*
* Number of shared memory blocks reserved for implementation internal use.
*
- * Each stash requires one SHM block, each pool requires three blocks (buffers,
- * ring, user area), and 20 blocks are reserved for per ODP module global data.
+ * Each pool requires three blocks (buffers, ring, user area), and 20 blocks
+ * are reserved for per ODP module global data.
*/
-#define CONFIG_INTERNAL_SHM_BLOCKS (CONFIG_MAX_STASHES + (ODP_CONFIG_POOLS * 3) + 20)
+#define CONFIG_INTERNAL_SHM_BLOCKS ((ODP_CONFIG_POOLS * 3) + 20)
/*
* Maximum number of shared memory blocks.
diff --git a/platform/linux-generic/include/odp_event_internal.h b/platform/linux-generic/include/odp_event_internal.h
index 5a29e926e..4bc28d708 100644
--- a/platform/linux-generic/include/odp_event_internal.h
+++ b/platform/linux-generic/include/odp_event_internal.h
@@ -53,7 +53,7 @@ typedef struct _odp_event_hdr_t {
/* --- Mostly read only data --- */
- /* Initial buffer tail pointer */
+ /* Initial buffer tail pointer and endmark location (if enabled) */
uint8_t *buf_end;
/* Combined pool and event index */
@@ -85,6 +85,11 @@ static inline void _odp_event_type_set(odp_event_t event, int ev)
_odp_event_hdr(event)->event_type = ev;
}
+static inline uint64_t *_odp_event_endmark_get_ptr(odp_event_t event)
+{
+ return (uint64_t *)(uintptr_t)_odp_event_hdr(event)->buf_end;
+}
+
#ifdef __cplusplus
}
#endif
diff --git a/platform/linux-generic/include/odp_event_validation_internal.h b/platform/linux-generic/include/odp_event_validation_internal.h
new file mode 100644
index 000000000..f4ac16f31
--- /dev/null
+++ b/platform/linux-generic/include/odp_event_validation_internal.h
@@ -0,0 +1,52 @@
+/* Copyright (c) 2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_EVENT_VALIDATION_INTERNAL_H_
+#define ODP_EVENT_VALIDATION_INTERNAL_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp/autoheader_external.h>
+
+#include <odp/api/event.h>
+#include <odp/api/hints.h>
+
+#include <odp/api/plat/event_validation_external.h>
+
+#include <odp_event_internal.h>
+
+#include <stdint.h>
+
+#if _ODP_EVENT_VALIDATION
+
+#define _ODP_EV_ENDMARK_VAL 0xDEADBEEFDEADBEEF
+#define _ODP_EV_ENDMARK_SIZE (sizeof(uint64_t))
+
+static inline void _odp_event_endmark_set(odp_event_t event)
+{
+ uint64_t *endmark_ptr;
+
+ endmark_ptr = _odp_event_endmark_get_ptr(event);
+ *endmark_ptr = _ODP_EV_ENDMARK_VAL;
+}
+
+#else
+
+#define _ODP_EV_ENDMARK_VAL 0
+#define _ODP_EV_ENDMARK_SIZE 0
+
+static inline void _odp_event_endmark_set(odp_event_t event ODP_UNUSED)
+{
+}
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#endif
diff --git a/platform/linux-generic/include/odp_init_internal.h b/platform/linux-generic/include/odp_init_internal.h
index 2a1039854..24e8346ad 100644
--- a/platform/linux-generic/include/odp_init_internal.h
+++ b/platform/linux-generic/include/odp_init_internal.h
@@ -33,6 +33,9 @@ int _odp_pool_init_local(void);
int _odp_pool_term_global(void);
int _odp_pool_term_local(void);
+int _odp_event_validation_init_global(void);
+int _odp_event_validation_term_global(void);
+
int _odp_queue_init_global(void);
int _odp_queue_term_global(void);
diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h
index 954602959..187a3a76f 100644
--- a/platform/linux-generic/include/odp_packet_io_internal.h
+++ b/platform/linux-generic/include/odp_packet_io_internal.h
@@ -1,5 +1,5 @@
/* Copyright (c) 2013-2018, Linaro Limited
- * Copyright (c) 2019-2022, Nokia
+ * Copyright (c) 2019-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -68,12 +68,8 @@ struct pktio_if_ops;
#define PKTIO_PRIVATE_SIZE 33792
#elif defined(_ODP_PKTIO_XDP)
#define PKTIO_PRIVATE_SIZE 29696
-#elif defined(_ODP_PKTIO_DPDK) && ODP_CACHE_LINE_SIZE == 128
-#define PKTIO_PRIVATE_SIZE 4160
-#elif defined(_ODP_PKTIO_DPDK)
-#define PKTIO_PRIVATE_SIZE 3968
#else
-#define PKTIO_PRIVATE_SIZE 384
+#define PKTIO_PRIVATE_SIZE 9216
#endif
typedef struct ODP_ALIGNED_CACHE {
diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h
index 1c5b51c3d..c8d2168f3 100644
--- a/platform/linux-generic/include/odp_pool_internal.h
+++ b/platform/linux-generic/include/odp_pool_internal.h
@@ -87,6 +87,7 @@ typedef struct pool_t {
uint32_t block_size;
uint32_t block_offset;
uint32_t num_populated;
+ uint32_t trailer_size;
uint8_t *base_addr;
uint8_t *max_addr;
uint8_t *uarea_base_addr;
diff --git a/platform/linux-generic/include/odp_queue_basic_internal.h b/platform/linux-generic/include/odp_queue_basic_internal.h
index 830f50a9d..3cdcf8600 100644
--- a/platform/linux-generic/include/odp_queue_basic_internal.h
+++ b/platform/linux-generic/include/odp_queue_basic_internal.h
@@ -1,4 +1,5 @@
/* Copyright (c) 2013-2018, Linaro Limited
+ * Copyright (c) 2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -23,7 +24,7 @@ extern "C" {
#include <odp/api/ticketlock.h>
#include <odp_config_internal.h>
#include <odp_macros_internal.h>
-#include <odp_ring_mpmc_internal.h>
+#include <odp_ring_mpmc_u32_internal.h>
#include <odp_ring_st_internal.h>
#include <odp_ring_spsc_internal.h>
#include <odp_queue_lf.h>
@@ -47,7 +48,7 @@ typedef struct ODP_ALIGNED_CACHE queue_entry_s {
odp_queue_type_t type;
/* MPMC ring (2 cache lines). */
- ring_mpmc_t ring_mpmc;
+ ring_mpmc_u32_t ring_mpmc;
odp_ticketlock_t lock;
union {
diff --git a/platform/linux-generic/include/odp_ring_mpmc_internal.h b/platform/linux-generic/include/odp_ring_mpmc_internal.h
index 6ed4dd4d1..e35179267 100644
--- a/platform/linux-generic/include/odp_ring_mpmc_internal.h
+++ b/platform/linux-generic/include/odp_ring_mpmc_internal.h
@@ -1,4 +1,5 @@
/* Copyright (c) 2018, Linaro Limited
+ * Copyright (c) 2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -19,6 +20,8 @@ extern "C" {
#include <odp/api/plat/atomic_inlines.h>
#include <odp/api/plat/cpu_inlines.h>
+#include <odp_ring_common.h>
+
/* Ring of uint32_t data
*
* Ring stores head and tail counters. Ring indexes are formed from these
@@ -34,14 +37,22 @@ extern "C" {
* r_tail r_head w_tail w_head
*
*/
-typedef struct {
+
+struct ring_mpmc_common {
odp_atomic_u32_t r_head ODP_ALIGNED_CACHE;
odp_atomic_u32_t r_tail;
odp_atomic_u32_t w_head ODP_ALIGNED_CACHE;
odp_atomic_u32_t w_tail;
+};
+
+typedef struct ODP_ALIGNED_CACHE {
+ struct ring_mpmc_common r;
+} ring_mpmc_u32_t;
-} ring_mpmc_t;
+typedef struct ODP_ALIGNED_CACHE {
+ struct ring_mpmc_common r;
+} ring_mpmc_u64_t;
static inline int ring_mpmc_cas_u32(odp_atomic_u32_t *atom,
uint32_t *old_val, uint32_t new_val)
@@ -52,21 +63,63 @@ static inline int ring_mpmc_cas_u32(odp_atomic_u32_t *atom,
__ATOMIC_RELAXED);
}
+#endif /* End of include guards */
+
+#undef _ring_mpmc_gen_t
+#undef _ring_mpmc_data_t
+#undef _RING_MPMC_INIT
+#undef _RING_MPMC_DEQ_MULTI
+#undef _RING_MPMC_ENQ_MULTI
+#undef _RING_MPMC_DEQ_BATCH
+#undef _RING_MPMC_ENQ_BATCH
+#undef _RING_MPMC_IS_EMPTY
+#undef _RING_MPMC_LEN
+
+/* This header should NOT be included directly. There are no include guards for
+ * the following types and function definitions! */
+#ifndef _ODP_RING_TYPE
+#error Include type specific (u32/u64) ring header instead of this common file.
+#endif
+
+#if _ODP_RING_TYPE == _ODP_RING_TYPE_U32
+ #define _ring_mpmc_gen_t ring_mpmc_u32_t
+ #define _ring_mpmc_data_t uint32_t
+
+ #define _RING_MPMC_INIT ring_mpmc_u32_init
+ #define _RING_MPMC_DEQ_MULTI ring_mpmc_u32_deq_multi
+ #define _RING_MPMC_ENQ_MULTI ring_mpmc_u32_enq_multi
+ #define _RING_MPMC_DEQ_BATCH ring_mpmc_u32_deq_batch
+ #define _RING_MPMC_ENQ_BATCH ring_mpmc_u32_enq_batch
+ #define _RING_MPMC_IS_EMPTY ring_mpmc_u32_is_empty
+ #define _RING_MPMC_LEN ring_mpmc_u32_len
+#elif _ODP_RING_TYPE == _ODP_RING_TYPE_U64
+ #define _ring_mpmc_gen_t ring_mpmc_u64_t
+ #define _ring_mpmc_data_t uint64_t
+
+ #define _RING_MPMC_INIT ring_mpmc_u64_init
+ #define _RING_MPMC_DEQ_MULTI ring_mpmc_u64_deq_multi
+ #define _RING_MPMC_ENQ_MULTI ring_mpmc_u64_enq_multi
+ #define _RING_MPMC_DEQ_BATCH ring_mpmc_u64_deq_batch
+ #define _RING_MPMC_ENQ_BATCH ring_mpmc_u64_enq_batch
+ #define _RING_MPMC_IS_EMPTY ring_mpmc_u64_is_empty
+ #define _RING_MPMC_LEN ring_mpmc_u64_len
+#endif
+
/* Initialize ring */
-static inline void ring_mpmc_init(ring_mpmc_t *ring)
+static inline void _RING_MPMC_INIT(_ring_mpmc_gen_t *ring)
{
- odp_atomic_init_u32(&ring->w_head, 0);
- odp_atomic_init_u32(&ring->w_tail, 0);
- odp_atomic_init_u32(&ring->r_head, 0);
- odp_atomic_init_u32(&ring->r_tail, 0);
+ odp_atomic_init_u32(&ring->r.w_head, 0);
+ odp_atomic_init_u32(&ring->r.w_tail, 0);
+ odp_atomic_init_u32(&ring->r.r_head, 0);
+ odp_atomic_init_u32(&ring->r.r_tail, 0);
}
-/* Dequeue data from the ring head. Num is smaller than ring size. */
-static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring,
- uint32_t *ring_data,
- uint32_t ring_mask,
- uint32_t data[],
- uint32_t num)
+/* Dequeue data from the ring head */
+static inline uint32_t _RING_MPMC_DEQ_MULTI(_ring_mpmc_gen_t *ring,
+ _ring_mpmc_data_t *ring_data,
+ uint32_t ring_mask,
+ _ring_mpmc_data_t data[],
+ uint32_t num)
{
uint32_t old_head, new_head, w_tail, num_data, i;
@@ -75,9 +128,9 @@ static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring,
* When CAS operation succeeds, this thread owns data between old
* and new r_head. */
do {
- old_head = odp_atomic_load_acq_u32(&ring->r_head);
+ old_head = odp_atomic_load_acq_u32(&ring->r.r_head);
odp_prefetch(&ring_data[(old_head + 1) & ring_mask]);
- w_tail = odp_atomic_load_acq_u32(&ring->w_tail);
+ w_tail = odp_atomic_load_acq_u32(&ring->r.w_tail);
num_data = w_tail - old_head;
/* Ring is empty */
@@ -90,7 +143,49 @@ static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring,
new_head = old_head + num;
- } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r_head, &old_head,
+ } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r.r_head, &old_head,
+ new_head) == 0));
+
+ /* Read data. This will not move above load acquire of r_head. */
+ for (i = 0; i < num; i++)
+ data[i] = ring_data[(old_head + 1 + i) & ring_mask];
+
+ /* Wait until other readers have updated the tail */
+ while (odp_unlikely(odp_atomic_load_u32(&ring->r.r_tail) != old_head))
+ odp_cpu_pause();
+
+ /* Release the new reader tail, writers acquire it. */
+ odp_atomic_store_rel_u32(&ring->r.r_tail, new_head);
+
+ return num;
+}
+
+/* Dequeue num or 0 data from the ring head */
+static inline uint32_t _RING_MPMC_DEQ_BATCH(_ring_mpmc_gen_t *ring,
+ _ring_mpmc_data_t *ring_data,
+ uint32_t ring_mask,
+ _ring_mpmc_data_t data[],
+ uint32_t num)
+{
+ uint32_t old_head, new_head, w_tail, num_data, i;
+
+ /* Load acquires ensure that w_tail load happens after r_head load,
+ * and thus r_head value is always behind or equal to w_tail value.
+ * When CAS operation succeeds, this thread owns data between old
+ * and new r_head. */
+ do {
+ old_head = odp_atomic_load_acq_u32(&ring->r.r_head);
+ odp_prefetch(&ring_data[(old_head + 1) & ring_mask]);
+ w_tail = odp_atomic_load_acq_u32(&ring->r.w_tail);
+ num_data = w_tail - old_head;
+
+ /* Not enough data available */
+ if (num_data < num)
+ return 0;
+
+ new_head = old_head + num;
+
+ } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r.r_head, &old_head,
new_head) == 0));
/* Read data. This will not move above load acquire of r_head. */
@@ -98,21 +193,21 @@ static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring,
data[i] = ring_data[(old_head + 1 + i) & ring_mask];
/* Wait until other readers have updated the tail */
- while (odp_unlikely(odp_atomic_load_u32(&ring->r_tail) != old_head))
+ while (odp_unlikely(odp_atomic_load_u32(&ring->r.r_tail) != old_head))
odp_cpu_pause();
/* Release the new reader tail, writers acquire it. */
- odp_atomic_store_rel_u32(&ring->r_tail, new_head);
+ odp_atomic_store_rel_u32(&ring->r.r_tail, new_head);
return num;
}
-/* Enqueue multiple data into the ring tail. Num is smaller than ring size. */
-static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring,
- uint32_t *ring_data,
- uint32_t ring_mask,
- const uint32_t data[],
- uint32_t num)
+/* Enqueue multiple data into the ring tail */
+static inline uint32_t _RING_MPMC_ENQ_MULTI(_ring_mpmc_gen_t *ring,
+ _ring_mpmc_data_t *ring_data,
+ uint32_t ring_mask,
+ const _ring_mpmc_data_t data[],
+ uint32_t num)
{
uint32_t old_head, new_head, r_tail, num_free, i;
uint32_t size = ring_mask + 1;
@@ -122,8 +217,8 @@ static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring,
* When CAS operation succeeds, this thread owns data between old
* and new w_head. */
do {
- r_tail = odp_atomic_load_acq_u32(&ring->r_tail);
- old_head = odp_atomic_load_acq_u32(&ring->w_head);
+ r_tail = odp_atomic_load_acq_u32(&ring->r.r_tail);
+ old_head = odp_atomic_load_acq_u32(&ring->r.w_head);
num_free = size - (old_head - r_tail);
@@ -137,7 +232,50 @@ static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring,
new_head = old_head + num;
- } while (odp_unlikely(ring_mpmc_cas_u32(&ring->w_head, &old_head,
+ } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r.w_head, &old_head,
+ new_head) == 0));
+
+ /* Write data. This will not move above load acquire of w_head. */
+ for (i = 0; i < num; i++)
+ ring_data[(old_head + 1 + i) & ring_mask] = data[i];
+
+ /* Wait until other writers have updated the tail */
+ while (odp_unlikely(odp_atomic_load_u32(&ring->r.w_tail) != old_head))
+ odp_cpu_pause();
+
+ /* Release the new writer tail, readers acquire it. */
+ odp_atomic_store_rel_u32(&ring->r.w_tail, new_head);
+
+ return num;
+}
+
+/* Enqueue num or 0 data into the ring tail */
+static inline uint32_t _RING_MPMC_ENQ_BATCH(_ring_mpmc_gen_t *ring,
+ _ring_mpmc_data_t *ring_data,
+ uint32_t ring_mask,
+ const _ring_mpmc_data_t data[],
+ uint32_t num)
+{
+ uint32_t old_head, new_head, r_tail, num_free, i;
+ uint32_t size = ring_mask + 1;
+
+ /* Load acquires ensure that w_head load happens after r_tail load,
+ * and thus r_tail value is always behind or equal to w_head value.
+ * When CAS operation succeeds, this thread owns data between old
+ * and new w_head. */
+ do {
+ r_tail = odp_atomic_load_acq_u32(&ring->r.r_tail);
+ old_head = odp_atomic_load_acq_u32(&ring->r.w_head);
+
+ num_free = size - (old_head - r_tail);
+
+ /* Not enough free space available */
+ if (num_free < num)
+ return 0;
+
+ new_head = old_head + num;
+
+ } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r.w_head, &old_head,
new_head) == 0));
/* Write data. This will not move above load acquire of w_head. */
@@ -145,29 +283,29 @@ static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring,
ring_data[(old_head + 1 + i) & ring_mask] = data[i];
/* Wait until other writers have updated the tail */
- while (odp_unlikely(odp_atomic_load_u32(&ring->w_tail) != old_head))
+ while (odp_unlikely(odp_atomic_load_u32(&ring->r.w_tail) != old_head))
odp_cpu_pause();
/* Release the new writer tail, readers acquire it. */
- odp_atomic_store_rel_u32(&ring->w_tail, new_head);
+ odp_atomic_store_rel_u32(&ring->r.w_tail, new_head);
return num;
}
/* Check if ring is empty */
-static inline int ring_mpmc_is_empty(ring_mpmc_t *ring)
+static inline int _RING_MPMC_IS_EMPTY(_ring_mpmc_gen_t *ring)
{
- uint32_t head = odp_atomic_load_u32(&ring->r_head);
- uint32_t tail = odp_atomic_load_u32(&ring->w_tail);
+ uint32_t head = odp_atomic_load_u32(&ring->r.r_head);
+ uint32_t tail = odp_atomic_load_u32(&ring->r.w_tail);
return head == tail;
}
/* Return current ring length */
-static inline uint32_t ring_mpmc_length(ring_mpmc_t *ring)
+static inline uint32_t _RING_MPMC_LEN(_ring_mpmc_gen_t *ring)
{
- uint32_t head = odp_atomic_load_u32(&ring->r_head);
- uint32_t tail = odp_atomic_load_u32(&ring->w_tail);
+ uint32_t head = odp_atomic_load_u32(&ring->r.r_head);
+ uint32_t tail = odp_atomic_load_u32(&ring->r.w_tail);
return tail - head;
}
@@ -175,5 +313,3 @@ static inline uint32_t ring_mpmc_length(ring_mpmc_t *ring)
#ifdef __cplusplus
}
#endif
-
-#endif
diff --git a/platform/linux-generic/include/odp_ring_mpmc_u32_internal.h b/platform/linux-generic/include/odp_ring_mpmc_u32_internal.h
new file mode 100644
index 000000000..4699b5b47
--- /dev/null
+++ b/platform/linux-generic/include/odp_ring_mpmc_u32_internal.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_RING_MPMC_U32_INTERNAL_H_
+#define ODP_RING_MPMC_U32_INTERNAL_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp_ring_common.h>
+
+#undef _ODP_RING_TYPE
+#define _ODP_RING_TYPE _ODP_RING_TYPE_U32
+
+#include <odp_ring_mpmc_internal.h>
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/include/odp_ring_mpmc_u64_internal.h b/platform/linux-generic/include/odp_ring_mpmc_u64_internal.h
new file mode 100644
index 000000000..e7bf31a94
--- /dev/null
+++ b/platform/linux-generic/include/odp_ring_mpmc_u64_internal.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_RING_MPMC_U64_INTERNAL_H_
+#define ODP_RING_MPMC_U64_INTERNAL_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp_ring_common.h>
+
+#undef _ODP_RING_TYPE
+#define _ODP_RING_TYPE _ODP_RING_TYPE_U64
+
+#include <odp_ring_mpmc_internal.h>
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/libodp-linux.pc.in b/platform/linux-generic/libodp-linux.pc.in
index f9a339fb8..05ba5b9d6 100644
--- a/platform/linux-generic/libodp-linux.pc.in
+++ b/platform/linux-generic/libodp-linux.pc.in
@@ -8,5 +8,5 @@ Description: The ODP packet processing engine
Version: @PKGCONFIG_VERSION@
Requires.private: libconfig@AARCH64CRYPTO_PKG@
Libs: -L${libdir} -l@ODP_LIB_NAME@ @ATOMIC_LIBS_NON_ABI_COMPAT@
-Libs.private: @OPENSSL_STATIC_LIBS@ @DPDK_LIBS@ @PCAP_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ @LIBXDP_LIBS@ -lpthread @ATOMIC_LIBS_ABI_COMPAT@
+Libs.private: @OPENSSL_STATIC_LIBS@ @DPDK_LIBS@ @PCAP_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ @LIBXDP_LIBS@ -lpthread @ATOMIC_LIBS_ABI_COMPAT@ @IPSEC_MB_LIBS@
Cflags: -I${includedir}
diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4
index 70a393f56..515c85239 100644
--- a/platform/linux-generic/m4/configure.m4
+++ b/platform/linux-generic/m4/configure.m4
@@ -7,6 +7,7 @@ ODP_ATOMIC
ODP_PTHREAD
ODP_TIMER
m4_include([platform/linux-generic/m4/odp_cpu.m4])
+m4_include([platform/linux-generic/m4/odp_event_validation.m4])
m4_include([platform/linux-generic/m4/odp_pcap.m4])
m4_include([platform/linux-generic/m4/odp_scheduler.m4])
@@ -23,17 +24,20 @@ AM_CONDITIONAL([ODP_PKTIO_PCAP], [test x$have_pcap = xyes])
m4_include([platform/linux-generic/m4/odp_libconfig.m4])
m4_include([platform/linux-generic/m4/odp_openssl.m4])
m4_include([platform/linux-generic/m4/odp_crypto.m4])
+m4_include([platform/linux-generic/m4/odp_ipsec_mb.m4])
m4_include([platform/linux-generic/m4/odp_pcapng.m4])
m4_include([platform/linux-generic/m4/odp_netmap.m4])
m4_include([platform/linux-generic/m4/odp_dpdk.m4])
m4_include([platform/linux-generic/m4/odp_xdp.m4])
+ODP_EVENT_VALIDATION
ODP_SCHEDULER
-AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${AARCH64CRYPTO_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS} ${LIBXDP_LIBS}"])
+AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${AARCH64CRYPTO_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${IPSEC_MB_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS} ${LIBXDP_LIBS}"])
# Add text to the end of configure with platform specific settings.
# Make sure it's aligned same as other lines in configure.ac.
AS_VAR_APPEND([PLAT_CFG_TEXT], ["
+ event_validation: ${enable_event_validation}
openssl: ${with_openssl}
openssl_rand: ${openssl_rand}
crypto: ${with_crypto}
diff --git a/platform/linux-generic/m4/odp_crypto.m4 b/platform/linux-generic/m4/odp_crypto.m4
index 9bb99f7dd..1cec6edb4 100644
--- a/platform/linux-generic/m4/odp_crypto.m4
+++ b/platform/linux-generic/m4/odp_crypto.m4
@@ -3,7 +3,7 @@
# Select default crypto implementation
AC_ARG_WITH([crypto],
[AS_HELP_STRING([--with-crypto],
- [Choose crypto implementation (openssl/armv8crypto/null)]
+ [Choose crypto implementation (openssl/armv8crypto/ipsecmb/null)]
[[default=openssl] (linux-generic)])],
[], [with_crypto=openssl])
@@ -14,7 +14,7 @@ AS_IF([test "x$with_crypto" = "xyes"], [with_crypto=openssl])
AS_IF([test "x$with_crypto" = "xno"], [with_crypto=null])
AS_IF([test "x$with_crypto" = "xopenssl" -a "x$with_openssl" = "xno"], [with_crypto=null])
-AS_IF([test "x$with_crypto" != "xopenssl" -a "x$with_crypto" != "xarmv8crypto" -a "x$with_crypto" != "xnull"],
+AS_IF([test "x$with_crypto" != "xopenssl" -a "x$with_crypto" != "xarmv8crypto" -a "x$with_crypto" != "xipsecmb" -a "x$with_crypto" != "xnull"],
[AC_MSG_ERROR([Invalid crypto implementation name])])
##########################################################################
@@ -31,11 +31,19 @@ AS_IF([test "x$with_crypto" == "xarmv8crypto"],
[PKG_CHECK_MODULES([AARCH64CRYPTO], [libAArch64crypto])
AARCH64CRYPTO_PKG=", libAArch64crypto"
AC_SUBST([AARCH64CRYPTO_PKG])])
+
AC_CONFIG_COMMANDS_PRE([dnl
AM_CONDITIONAL([WITH_ARMV8_CRYPTO], [test "x$with_crypto" == "xarmv8crypto"])
])
##########################################################################
+# Multi-buffer IPSec library implementation
+##########################################################################
+AC_CONFIG_COMMANDS_PRE([dnl
+AM_CONDITIONAL([WITH_IPSECMB_CRYPTO], [test "x$with_crypto" == "xipsecmb"])
+])
+
+##########################################################################
# Null implementation
##########################################################################
AS_IF([test "x$with_crypto" == "xnull"],
diff --git a/platform/linux-generic/m4/odp_event_validation.m4 b/platform/linux-generic/m4/odp_event_validation.m4
new file mode 100644
index 000000000..08bb8902e
--- /dev/null
+++ b/platform/linux-generic/m4/odp_event_validation.m4
@@ -0,0 +1,23 @@
+# ODP_EVENT_VALIDATION
+# --------------------
+# Select event validation level
+AC_DEFUN([ODP_EVENT_VALIDATION], [dnl
+AC_ARG_ENABLE([event-validation],
+ [AS_HELP_STRING([--enable-event-validation],
+ [enable event validation (warn/abort)
+ [default=disabled] (linux-generic)])],
+ [], [AS_IF([test "x$enable_debug" = "xfull"],
+ [enable_event_validation=yes], [enable_event_validation=no])])
+
+# Default to abort mode if validation is enabled
+AS_IF([test "x$enable_event_validation" = "xyes"],
+ [enable_event_validation="abort"])
+
+validation_level=0
+AS_IF([test "x$enable_event_validation" = "xwarn"], [validation_level=1])
+AS_IF([test "x$enable_event_validation" = "xyes" -o "x$enable_event_validation" = "xabort"],
+ [validation_level=2])
+
+AC_DEFINE_UNQUOTED([_ODP_EVENT_VALIDATION], [$validation_level],
+ [Define to 1 or 2 to enable event validation])
+]) # ODP_EVENT_VALIDATION
diff --git a/platform/linux-generic/m4/odp_ipsec_mb.m4 b/platform/linux-generic/m4/odp_ipsec_mb.m4
new file mode 100644
index 000000000..3268d94c0
--- /dev/null
+++ b/platform/linux-generic/m4/odp_ipsec_mb.m4
@@ -0,0 +1,19 @@
+#########################################################################
+# Check for libIPSec_MB availability
+#########################################################################
+ipsecmb_support=no
+AC_CHECK_HEADERS([ipsec-mb.h],
+ [AC_CHECK_LIB([IPSec_MB], [init_mb_mgr_auto], [ipsecmb_support=yes],
+ [ipsecmb_support=no])],
+ [ipsecmb_support=no])
+
+AS_IF([test "x$with_crypto" = "xipsecmb" -a "x$ipsecmb_support" = "xno"],
+ [AC_MSG_ERROR([IPSec MB library not found on this platform])])
+
+if test "x$with_crypto" = "xipsecmb"; then
+ IPSEC_MB_LIBS="-lIPSec_MB"
+else
+ IPSEC_MB_LIBS=""
+fi
+
+AC_SUBST([IPSEC_MB_LIBS])
diff --git a/platform/linux-generic/m4/odp_libconfig.m4 b/platform/linux-generic/m4/odp_libconfig.m4
index 03dbc929d..0d268c935 100644
--- a/platform/linux-generic/m4/odp_libconfig.m4
+++ b/platform/linux-generic/m4/odp_libconfig.m4
@@ -3,7 +3,7 @@
##########################################################################
m4_define([_odp_config_version_generation], [0])
m4_define([_odp_config_version_major], [1])
-m4_define([_odp_config_version_minor], [22])
+m4_define([_odp_config_version_minor], [25])
m4_define([_odp_config_version],
[_odp_config_version_generation._odp_config_version_major._odp_config_version_minor])
diff --git a/platform/linux-generic/odp_crypto_api.c b/platform/linux-generic/odp_crypto_api.c
new file mode 100644
index 000000000..646472e2e
--- /dev/null
+++ b/platform/linux-generic/odp_crypto_api.c
@@ -0,0 +1,11 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/crypto.h>
+
+/* Non-inlined versions of API functions */
+#define _ODP_NO_INLINE
+#include <odp/api/plat/crypto_inlines.h>
diff --git a/platform/linux-generic/odp_crypto_ipsecmb.c b/platform/linux-generic/odp_crypto_ipsecmb.c
new file mode 100644
index 000000000..a7ce6077a
--- /dev/null
+++ b/platform/linux-generic/odp_crypto_ipsecmb.c
@@ -0,0 +1,881 @@
+/* Copyright (c) 2014-2018, Linaro Limited
+ * Copyright (c) 2021, ARM Limited
+ * Copyright (c) 2022-2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp_posix_extensions.h>
+#include <odp/autoheader_internal.h>
+
+#include <odp/api/crypto.h>
+#include <odp/api/spinlock.h>
+#include <odp/api/debug.h>
+#include <odp/api/align.h>
+#include <odp/api/shared_memory.h>
+#include <odp/api/hints.h>
+
+#include <odp/api/plat/event_inlines.h>
+#include <odp/api/plat/packet_inlines.h>
+#include <odp/api/plat/queue_inlines.h>
+#include <odp/api/plat/thread_inlines.h>
+
+#include <odp_debug_internal.h>
+#include <odp_global_data.h>
+#include <odp_init_internal.h>
+#include <odp_packet_internal.h>
+
+#include <ipsec-mb.h>
+
+#define MAX_SESSIONS 4000
+/* Length in bytes */
+#define IPSEC_MB_CRYPTO_MAX_CIPHER_KEY_LENGTH 32
+#define IPSEC_MB_CRYPTO_MAX_AUTH_KEY_LENGTH 32
+#define IPSEC_MB_CRYPTO_MAX_DATA_LENGTH 65536
+#define ZUC_DIGEST_LENGTH 4
+
+#define ODP_CRYPTO_IPSEC_MB_SHM_NAME "_odp_crypto_ipsecmb"
+/*
+ * Cipher algorithm capabilities
+ *
+ * Keep sorted: first by key length, then by IV length
+ */
+static const odp_crypto_cipher_capability_t cipher_capa_null[] = {
+{.key_len = 0, .iv_len = 0} };
+
+static const odp_crypto_cipher_capability_t cipher_capa_zuc_eea3[] = {
+{.key_len = 16, .iv_len = 16},
+{.key_len = 32, .iv_len = 25} };
+
+/*
+ * Authentication algorithm capabilities
+ *
+ * Keep sorted: first by digest length, then by key length
+ */
+static const odp_crypto_auth_capability_t auth_capa_null[] = {
+{.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0} } };
+
+static const odp_crypto_auth_capability_t auth_capa_zuc_eia3[] = {
+{.digest_len = 4, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0},
+ .iv_len = 16},
+{.digest_len = 4, .key_len = 32, .aad_len = {.min = 0, .max = 0, .inc = 0},
+ .iv_len = 25} };
+
+/** Forward declaration of session structure */
+typedef struct odp_crypto_generic_session_t odp_crypto_generic_session_t;
+
+/**
+ * Algorithm handler function prototype
+ */
+typedef odp_crypto_alg_err_t (*crypto_func_t)(odp_packet_t pkt,
+ const odp_crypto_packet_op_param_t *param,
+ odp_crypto_generic_session_t *session);
+
+/**
+ * Per crypto session data structure
+ */
+struct odp_crypto_generic_session_t {
+ odp_crypto_generic_session_t *next;
+
+ /* Session creation parameters */
+ odp_crypto_session_param_t p;
+
+ odp_bool_t do_cipher_first;
+
+ struct {
+ uint8_t key_data[IPSEC_MB_CRYPTO_MAX_CIPHER_KEY_LENGTH];
+ crypto_func_t func;
+ } cipher;
+
+ struct {
+ uint8_t key[IPSEC_MB_CRYPTO_MAX_AUTH_KEY_LENGTH];
+ crypto_func_t func;
+ } auth;
+
+ unsigned int idx;
+};
+
+typedef struct odp_crypto_global_s odp_crypto_global_t;
+
+struct odp_crypto_global_s {
+ odp_spinlock_t lock;
+ odp_crypto_generic_session_t *free;
+ odp_crypto_generic_session_t sessions[MAX_SESSIONS];
+};
+
+static odp_crypto_global_t *global;
+
+typedef struct crypto_local_t {
+ uint8_t buffer[IPSEC_MB_CRYPTO_MAX_DATA_LENGTH];
+ IMB_MGR *mb_mgr;
+} crypto_local_t;
+
+static __thread crypto_local_t local;
+
+static
+odp_crypto_generic_session_t *alloc_session(void)
+{
+ odp_crypto_generic_session_t *session = NULL;
+
+ odp_spinlock_lock(&global->lock);
+ session = global->free;
+ if (session) {
+ global->free = session->next;
+ session->next = NULL;
+ }
+ odp_spinlock_unlock(&global->lock);
+
+ if (!session)
+ return NULL;
+
+ session->idx = session - global->sessions;
+
+ return session;
+}
+
+static
+void free_session(odp_crypto_generic_session_t *session)
+{
+ odp_spinlock_lock(&global->lock);
+ session->next = global->free;
+ global->free = session;
+ odp_spinlock_unlock(&global->lock);
+}
+
+static odp_crypto_alg_err_t
+null_crypto_routine(odp_packet_t pkt ODP_UNUSED,
+ const odp_crypto_packet_op_param_t *param ODP_UNUSED,
+ odp_crypto_generic_session_t *session ODP_UNUSED)
+{
+ return ODP_CRYPTO_ALG_ERR_NONE;
+}
+
+static
+odp_crypto_alg_err_t zuc_eea3_cipher_op(odp_packet_t pkt,
+ const odp_crypto_packet_op_param_t *param,
+ odp_crypto_generic_session_t *session)
+{
+ IMB_MGR *mb_mgr = local.mb_mgr;
+ uint8_t *iv_ptr = param->cipher_iv_ptr;
+ uint32_t in_pos = param->cipher_range.offset;
+ uint32_t in_len = param->cipher_range.length;
+
+ _ODP_ASSERT(iv_ptr != NULL);
+
+ uint32_t seg_len = 0;
+ uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL);
+
+ if (odp_unlikely(seg_len < in_len)) {
+ if (odp_unlikely(in_len > IPSEC_MB_CRYPTO_MAX_DATA_LENGTH))
+ return ODP_CRYPTO_ALG_ERR_DATA_SIZE;
+
+ /* Packet is segmented within the cipher range. Copy the cipher
+ * range to a contiguous buffer. */
+ odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer);
+
+ data = local.buffer;
+ }
+
+ if (session->p.cipher_key.length == 16) {
+ /* ZUC128 EEA3 */
+ IMB_ZUC_EEA3_1_BUFFER(mb_mgr, session->cipher.key_data,
+ iv_ptr,
+ data,
+ data,
+ in_len);
+ } else {
+ /* Only 16 and 32 byte keys are supported
+ * ZUC256 EEA3 */
+ IMB_ZUC256_EEA3_1_BUFFER(mb_mgr, session->cipher.key_data,
+ iv_ptr,
+ data,
+ data,
+ in_len);
+ }
+ if (odp_unlikely(imb_get_errno(mb_mgr) != 0))
+ return ODP_CRYPTO_ALG_ERR_DATA_SIZE;
+
+ if (odp_unlikely(seg_len < in_len))
+ odp_packet_copy_from_mem(pkt, in_pos, in_len, data);
+
+ return ODP_CRYPTO_ALG_ERR_NONE;
+}
+
+static int process_zuc_eea3_param(odp_crypto_generic_session_t *session)
+{
+ if (!((16 == session->p.cipher_key.length &&
+ 16 == session->p.cipher_iv_len) ||
+ (32 == session->p.cipher_key.length &&
+ 25 == session->p.cipher_iv_len)))
+ return -1;
+
+ memcpy(session->cipher.key_data, session->p.cipher_key.data,
+ session->p.cipher_key.length);
+
+ session->cipher.func = zuc_eea3_cipher_op;
+
+ return 0;
+}
+
+static
+odp_crypto_alg_err_t auth_zuc_eia3_gen(odp_packet_t pkt,
+ const odp_crypto_packet_op_param_t *param,
+ odp_crypto_generic_session_t *session)
+{
+ IMB_MGR *mb_mgr = local.mb_mgr;
+ uint8_t *iv_ptr = param->auth_iv_ptr;
+ uint32_t in_pos = param->auth_range.offset;
+ uint32_t in_len = param->auth_range.length;
+ uint32_t auth_tag;
+
+ _ODP_ASSERT(iv_ptr != NULL);
+
+ uint32_t seg_len = 0;
+ uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL);
+
+ if (odp_unlikely(seg_len < in_len)) {
+ if (odp_unlikely(in_len > IPSEC_MB_CRYPTO_MAX_DATA_LENGTH))
+ return ODP_CRYPTO_ALG_ERR_DATA_SIZE;
+
+ /* Packet is segmented within the auth range. Copy the auth
+ * range to a contiguous buffer. */
+ odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer);
+
+ data = local.buffer;
+ }
+
+ if (session->p.auth_key.length == 16) {
+ /* ZUC128 EIA3 */
+ IMB_ZUC_EIA3_1_BUFFER(mb_mgr, session->auth.key,
+ iv_ptr,
+ data,
+ param->auth_range.length * 8,
+ &auth_tag);
+ } else {
+ /* Only 16 and 32 byte keys are supported
+ * ZUC256 EIA3 */
+ IMB_ZUC256_EIA3_1_BUFFER(mb_mgr, session->auth.key,
+ iv_ptr,
+ data,
+ param->auth_range.length * 8,
+ &auth_tag);
+ }
+ if (odp_unlikely(imb_get_errno(mb_mgr) != 0))
+ return ODP_CRYPTO_ALG_ERR_DATA_SIZE;
+
+ /* Copy to the output location */
+ odp_packet_copy_from_mem(pkt, param->hash_result_offset,
+ session->p.auth_digest_len,
+ &auth_tag);
+
+ return ODP_CRYPTO_ALG_ERR_NONE;
+}
+
+static
+odp_crypto_alg_err_t auth_zuc_eia3_check(odp_packet_t pkt,
+ const odp_crypto_packet_op_param_t *param,
+ odp_crypto_generic_session_t *session)
+{
+ IMB_MGR *mb_mgr = local.mb_mgr;
+ uint8_t *iv_ptr = param->auth_iv_ptr;
+ uint32_t in_pos = param->auth_range.offset;
+ uint32_t in_len = param->auth_range.length;
+ uint32_t bytes = ZUC_DIGEST_LENGTH;
+ uint32_t hash_in;
+ uint32_t hash_out;
+
+ /* Copy current value out and clear it before authentication */
+ odp_packet_copy_to_mem(pkt, param->hash_result_offset,
+ bytes, &hash_in);
+
+ if (odp_unlikely(session->p.hash_result_in_auth_range))
+ _odp_packet_set_data(pkt, param->hash_result_offset, 0, bytes);
+
+ _ODP_ASSERT(iv_ptr != NULL);
+
+ uint32_t seg_len = 0;
+ uint8_t *data = odp_packet_offset(pkt, in_pos, &seg_len, NULL);
+
+ if (odp_unlikely(seg_len < in_len)) {
+ if (odp_unlikely(in_len > IPSEC_MB_CRYPTO_MAX_DATA_LENGTH))
+ return ODP_CRYPTO_ALG_ERR_DATA_SIZE;
+
+ /* Packet is segmented within the auth range. Copy the auth
+ * range to a contiguous buffer. */
+ odp_packet_copy_to_mem(pkt, in_pos, in_len, local.buffer);
+
+ data = local.buffer;
+ }
+
+ if (session->p.auth_key.length == 16) {
+ /* ZUC128 EIA3 */
+ IMB_ZUC_EIA3_1_BUFFER(mb_mgr, session->auth.key,
+ iv_ptr,
+ data,
+ param->auth_range.length * 8,
+ &hash_out);
+ } else {
+ /* Only 16 and 32 byte keys are supported
+ * ZUC256 EIA3 */
+ IMB_ZUC256_EIA3_1_BUFFER(mb_mgr, session->auth.key,
+ iv_ptr,
+ data,
+ param->auth_range.length * 8,
+ &hash_out);
+ }
+ if (odp_unlikely(imb_get_errno(mb_mgr) != 0))
+ return ODP_CRYPTO_ALG_ERR_DATA_SIZE;
+
+ /* Verify match */
+ if (hash_in != hash_out)
+ return ODP_CRYPTO_ALG_ERR_ICV_CHECK;
+
+ return ODP_CRYPTO_ALG_ERR_NONE;
+}
+
+static int process_auth_zuc_eia3_param(odp_crypto_generic_session_t *session)
+{
+ if (!((16 == session->p.auth_key.length &&
+ 16 == session->p.auth_iv_len) ||
+ (32 == session->p.auth_key.length &&
+ 25 == session->p.auth_iv_len)))
+ return -1;
+
+ if (ODP_CRYPTO_OP_ENCODE == session->p.op)
+ session->auth.func = auth_zuc_eia3_gen;
+ else
+ session->auth.func = auth_zuc_eia3_check;
+
+ if (session->p.auth_digest_len != ZUC_DIGEST_LENGTH)
+ return -1;
+
+ memcpy(session->auth.key, session->p.auth_key.data,
+ session->p.auth_key.length);
+
+ return 0;
+}
+
+int odp_crypto_capability(odp_crypto_capability_t *capa)
+{
+ if (NULL == capa)
+ return -1;
+
+ memset(capa, 0, sizeof(odp_crypto_capability_t));
+
+ capa->sync_mode = ODP_SUPPORT_PREFERRED;
+ capa->async_mode = ODP_SUPPORT_YES;
+ capa->queue_type_plain = 1;
+ capa->queue_type_sched = 1;
+
+ capa->ciphers.bit.null = 1;
+ capa->auths.bit.null = 1;
+
+ capa->ciphers.bit.zuc_eea3 = 1;
+ capa->auths.bit.zuc_eia3 = 1;
+
+ capa->max_sessions = MAX_SESSIONS;
+
+ return 0;
+}
+
+int odp_crypto_cipher_capability(odp_cipher_alg_t cipher,
+ odp_crypto_cipher_capability_t dst[],
+ int num_copy)
+{
+ const odp_crypto_cipher_capability_t *src;
+ int num;
+ int size = sizeof(odp_crypto_cipher_capability_t);
+
+ switch (cipher) {
+ case ODP_CIPHER_ALG_NULL:
+ src = cipher_capa_null;
+ num = sizeof(cipher_capa_null) / size;
+ break;
+ case ODP_CIPHER_ALG_ZUC_EEA3:
+ src = cipher_capa_zuc_eea3;
+ num = sizeof(cipher_capa_zuc_eea3) / size;
+ break;
+ default:
+ return -1;
+ }
+
+ if (num < num_copy)
+ num_copy = num;
+
+ memcpy(dst, src, num_copy * size);
+
+ return num;
+}
+
+int odp_crypto_auth_capability(odp_auth_alg_t auth,
+ odp_crypto_auth_capability_t dst[], int num_copy)
+{
+ const odp_crypto_auth_capability_t *src;
+ int num;
+ int size = sizeof(odp_crypto_auth_capability_t);
+
+ switch (auth) {
+ case ODP_AUTH_ALG_NULL:
+ src = auth_capa_null;
+ num = sizeof(auth_capa_null) / size;
+ break;
+ case ODP_AUTH_ALG_ZUC_EIA3:
+ src = auth_capa_zuc_eia3;
+ num = sizeof(auth_capa_zuc_eia3) / size;
+ break;
+ default:
+ return -1;
+ }
+
+ if (num < num_copy)
+ num_copy = num;
+
+ memcpy(dst, src, num_copy * size);
+
+ return num;
+}
+
+int
+odp_crypto_session_create(const odp_crypto_session_param_t *param,
+ odp_crypto_session_t *session_out,
+ odp_crypto_ses_create_err_t *status)
+{
+ int rc = 0;
+ odp_crypto_generic_session_t *session;
+
+ if (odp_global_ro.disable.crypto) {
+ _ODP_ERR("Crypto is disabled\n");
+ /* Dummy output to avoid compiler warning about uninitialized
+ * variables */
+ *status = ODP_CRYPTO_SES_ERR_ENOMEM;
+ *session_out = ODP_CRYPTO_SESSION_INVALID;
+ return -1;
+ }
+
+ if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) {
+ *status = ODP_CRYPTO_SES_ERR_PARAMS;
+ *session_out = ODP_CRYPTO_SESSION_INVALID;
+ return -1;
+ }
+
+ session = alloc_session();
+ if (NULL == session) {
+ *status = ODP_CRYPTO_SES_ERR_ENOMEM;
+ goto err;
+ }
+
+ session->p = *param;
+
+ /* Derive order */
+ if (ODP_CRYPTO_OP_ENCODE == param->op)
+ session->do_cipher_first = param->auth_cipher_text;
+ else
+ session->do_cipher_first = !param->auth_cipher_text;
+
+ /* Process based on cipher */
+ switch (param->cipher_alg) {
+ case ODP_CIPHER_ALG_NULL:
+ session->cipher.func = null_crypto_routine;
+ rc = 0;
+ break;
+ case ODP_CIPHER_ALG_ZUC_EEA3:
+ rc = process_zuc_eea3_param(session);
+ break;
+ default:
+ rc = -1;
+ }
+
+ if (rc) {
+ *status = ODP_CRYPTO_SES_ERR_CIPHER;
+ goto err;
+ }
+
+ /* Process based on auth */
+ switch (param->auth_alg) {
+ case ODP_AUTH_ALG_NULL:
+ session->auth.func = null_crypto_routine;
+ rc = 0;
+ break;
+ case ODP_AUTH_ALG_ZUC_EIA3:
+ rc = process_auth_zuc_eia3_param(session);
+ break;
+ default:
+ rc = -1;
+ }
+
+ if (rc) {
+ *status = ODP_CRYPTO_SES_ERR_AUTH;
+ goto err;
+ }
+
+ *session_out = (intptr_t)session;
+ *status = ODP_CRYPTO_SES_ERR_NONE;
+ return 0;
+
+err:
+ /* error status should be set at this moment */
+ if (session != NULL)
+ free_session(session);
+ *session_out = ODP_CRYPTO_SESSION_INVALID;
+ return -1;
+}
+
+int odp_crypto_session_destroy(odp_crypto_session_t session)
+{
+ odp_crypto_generic_session_t *generic;
+
+ generic = (odp_crypto_generic_session_t *)(intptr_t)session;
+ memset(generic, 0, sizeof(*generic));
+ free_session(generic);
+ return 0;
+}
+
+#if ODP_DEPRECATED_API
+int
+odp_crypto_operation(odp_crypto_op_param_t *param,
+ odp_bool_t *posted,
+ odp_crypto_op_result_t *result)
+{
+ odp_crypto_packet_op_param_t packet_param;
+ odp_packet_t out_pkt = param->out_pkt;
+ odp_crypto_packet_result_t packet_result;
+ odp_crypto_op_result_t local_result;
+ int rc;
+
+ if (((odp_crypto_generic_session_t *)(intptr_t)param->session)->p.op_type !=
+ ODP_CRYPTO_OP_TYPE_LEGACY)
+ return -1;
+
+ packet_param.session = param->session;
+ packet_param.cipher_iv_ptr = param->cipher_iv_ptr;
+ packet_param.auth_iv_ptr = param->auth_iv_ptr;
+ packet_param.hash_result_offset = param->hash_result_offset;
+ packet_param.aad_ptr = param->aad_ptr;
+ packet_param.cipher_range = param->cipher_range;
+ packet_param.auth_range = param->auth_range;
+
+ rc = odp_crypto_op(&param->pkt, &out_pkt, &packet_param, 1);
+ if (rc <= 0)
+ return -1;
+
+ rc = odp_crypto_result(&packet_result, out_pkt);
+ if (rc < 0) {
+ /*
+ * We cannot fail since odp_crypto_op() has already processed
+ * the packet. Let's indicate error in the result instead.
+ */
+ packet_result.ok = false;
+ }
+
+ /* Indicate to caller operation was sync */
+ *posted = 0;
+
+ packet_subtype_set(out_pkt, ODP_EVENT_PACKET_BASIC);
+
+ /* Fill in result */
+ local_result.ctx = param->ctx;
+ local_result.pkt = out_pkt;
+ local_result.cipher_status = packet_result.cipher_status;
+ local_result.auth_status = packet_result.auth_status;
+ local_result.ok = packet_result.ok;
+
+ /*
+ * Be bug-to-bug compatible. Return output packet also through params.
+ */
+ param->out_pkt = out_pkt;
+
+ *result = local_result;
+
+ return 0;
+}
+#endif
+
+int _odp_crypto_init_global(void)
+{
+ size_t mem_size;
+ odp_shm_t shm;
+ int idx;
+
+ if (odp_global_ro.disable.crypto) {
+ _ODP_PRINT("\nODP crypto is DISABLED\n");
+ return 0;
+ }
+
+ /* Calculate the memory size we need */
+ mem_size = sizeof(odp_crypto_global_t);
+
+ /* Allocate our globally shared memory */
+ shm = odp_shm_reserve(ODP_CRYPTO_IPSEC_MB_SHM_NAME, mem_size,
+ ODP_CACHE_LINE_SIZE,
+ 0);
+ if (ODP_SHM_INVALID == shm) {
+ _ODP_ERR("unable to allocate crypto pool\n");
+ return -1;
+ }
+
+ global = odp_shm_addr(shm);
+
+ /* Clear it out */
+ memset(global, 0, mem_size);
+
+ /* Initialize free list and lock */
+ for (idx = 0; idx < MAX_SESSIONS; idx++) {
+ global->sessions[idx].next = global->free;
+ global->free = &global->sessions[idx];
+ }
+ odp_spinlock_init(&global->lock);
+
+ return 0;
+}
+
+int _odp_crypto_term_global(void)
+{
+ int rc = 0;
+ int ret;
+ int count = 0;
+ odp_crypto_generic_session_t *session;
+
+ if (odp_global_ro.disable.crypto)
+ return 0;
+
+ for (session = global->free; session != NULL; session = session->next)
+ count++;
+ if (count != MAX_SESSIONS) {
+ _ODP_ERR("crypto sessions still active\n");
+ rc = -1;
+ }
+
+ ret = odp_shm_free(odp_shm_lookup(ODP_CRYPTO_IPSEC_MB_SHM_NAME));
+ if (ret < 0) {
+ _ODP_ERR("shm free failed for %s\n", ODP_CRYPTO_IPSEC_MB_SHM_NAME);
+ rc = -1;
+ }
+
+ return rc;
+}
+
+int _odp_crypto_init_local(void)
+{
+ uint64_t flags = 0;
+
+ if (odp_global_ro.disable.crypto)
+ return 0;
+
+ memset(&local, 0, sizeof(local));
+
+ local.mb_mgr = alloc_mb_mgr(flags);
+ if (local.mb_mgr == NULL)
+ return -1;
+
+ init_mb_mgr_auto(local.mb_mgr, NULL);
+
+ return 0;
+}
+
+int _odp_crypto_term_local(void)
+{
+ if (odp_global_ro.disable.crypto)
+ return 0;
+
+ free_mb_mgr(local.mb_mgr);
+ return 0;
+}
+
+#if ODP_DEPRECATED_API
+odp_crypto_compl_t odp_crypto_compl_from_event(odp_event_t ev)
+{
+ /* This check not mandated by the API specification */
+ if (odp_event_type(ev) != ODP_EVENT_CRYPTO_COMPL)
+ _ODP_ABORT("Event not a crypto completion");
+ return (odp_crypto_compl_t)ev;
+}
+
+odp_event_t odp_crypto_compl_to_event(odp_crypto_compl_t completion_event)
+{
+ return (odp_event_t)completion_event;
+}
+
+void
+odp_crypto_compl_result(odp_crypto_compl_t completion_event,
+ odp_crypto_op_result_t *result)
+{
+ (void)completion_event;
+ (void)result;
+
+ /* We won't get such events anyway, so there can be no result */
+ _ODP_ASSERT(0);
+}
+
+void
+odp_crypto_compl_free(odp_crypto_compl_t completion_event)
+{
+ odp_event_t ev = odp_crypto_compl_to_event(completion_event);
+
+ odp_buffer_free(odp_buffer_from_event(ev));
+}
+
+uint64_t odp_crypto_compl_to_u64(odp_crypto_compl_t hdl)
+{
+ return _odp_pri(hdl);
+}
+#endif /* ODP_DEPRECATED_API */
+
+void odp_crypto_session_param_init(odp_crypto_session_param_t *param)
+{
+ memset(param, 0, sizeof(odp_crypto_session_param_t));
+}
+
+uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl)
+{
+ return (uint64_t)hdl;
+}
+
+static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src)
+{
+ int md_copy;
+ int rc;
+
+ md_copy = _odp_packet_copy_md_possible(odp_packet_pool(dst),
+ odp_packet_pool(src));
+ if (odp_unlikely(md_copy < 0)) {
+ _ODP_ERR("Unable to copy packet metadata\n");
+ return -1;
+ }
+
+ rc = odp_packet_copy_from_pkt(dst, 0, src, 0, odp_packet_len(src));
+ if (odp_unlikely(rc < 0)) {
+ _ODP_ERR("Unable to copy packet data\n");
+ return -1;
+ }
+
+ _odp_packet_copy_md(packet_hdr(dst), packet_hdr(src), md_copy);
+ return 0;
+}
+
+static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *session,
+ odp_packet_t pkt_in,
+ odp_packet_t pkt_out)
+{
+ int rc;
+
+ if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC))
+ return pkt_in;
+
+ if (odp_likely(pkt_in == pkt_out))
+ return pkt_out;
+
+ if (pkt_out == ODP_PACKET_INVALID) {
+ odp_pool_t pool = session->p.output_pool;
+
+ _ODP_ASSERT(pool != ODP_POOL_INVALID);
+ if (pool == odp_packet_pool(pkt_in)) {
+ pkt_out = pkt_in;
+ } else {
+ pkt_out = odp_packet_copy(pkt_in, pool);
+ if (odp_likely(pkt_out != ODP_PACKET_INVALID))
+ odp_packet_free(pkt_in);
+ }
+ return pkt_out;
+ }
+ rc = copy_data_and_metadata(pkt_out, pkt_in);
+ if (odp_unlikely(rc < 0))
+ return ODP_PACKET_INVALID;
+
+ odp_packet_free(pkt_in);
+ return pkt_out;
+}
+
+static
+int crypto_int(odp_packet_t pkt_in,
+ odp_packet_t *pkt_out,
+ const odp_crypto_packet_op_param_t *param)
+{
+ odp_crypto_alg_err_t rc_cipher = ODP_CRYPTO_ALG_ERR_NONE;
+ odp_crypto_alg_err_t rc_auth = ODP_CRYPTO_ALG_ERR_NONE;
+ odp_crypto_generic_session_t *session;
+ odp_packet_t out_pkt;
+ odp_crypto_packet_result_t *op_result;
+
+ session = (odp_crypto_generic_session_t *)(intptr_t)param->session;
+
+ out_pkt = get_output_packet(session, pkt_in, *pkt_out);
+ if (odp_unlikely(out_pkt == ODP_PACKET_INVALID))
+ return -1;
+
+ /* Invoke the crypto function */
+ if (session->do_cipher_first) {
+ rc_cipher = session->cipher.func(out_pkt, param, session);
+ rc_auth = session->auth.func(out_pkt, param, session);
+ } else {
+ rc_auth = session->auth.func(out_pkt, param, session);
+ rc_cipher = session->cipher.func(out_pkt, param, session);
+ }
+
+ packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO);
+ op_result = &packet_hdr(out_pkt)->crypto_op_result;
+ op_result->cipher_status.alg_err = rc_cipher;
+ op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
+ op_result->auth_status.alg_err = rc_auth;
+ op_result->auth_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
+ op_result->ok =
+ (rc_cipher == ODP_CRYPTO_ALG_ERR_NONE) &&
+ (rc_auth == ODP_CRYPTO_ALG_ERR_NONE);
+
+ /* Synchronous, simply return results */
+ *pkt_out = out_pkt;
+
+ return 0;
+}
+
+int odp_crypto_op(const odp_packet_t pkt_in[],
+ odp_packet_t pkt_out[],
+ const odp_crypto_packet_op_param_t param[],
+ int num_pkt)
+{
+ int i, rc;
+ odp_crypto_generic_session_t *session;
+
+ for (i = 0; i < num_pkt; i++) {
+ session = (odp_crypto_generic_session_t *)(intptr_t)param[i].session;
+ _ODP_ASSERT(ODP_CRYPTO_SYNC == session->p.op_mode);
+
+ rc = crypto_int(pkt_in[i], &pkt_out[i], &param[i]);
+ if (rc < 0)
+ break;
+ }
+
+ return i;
+}
+
+int odp_crypto_op_enq(const odp_packet_t pkt_in[],
+ const odp_packet_t pkt_out[],
+ const odp_crypto_packet_op_param_t param[],
+ int num_pkt)
+{
+ odp_packet_t pkt;
+ odp_event_t event;
+ odp_crypto_generic_session_t *session;
+ int i, rc;
+
+ for (i = 0; i < num_pkt; i++) {
+ session = (odp_crypto_generic_session_t *)(intptr_t)param[i].session;
+ _ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode);
+ _ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue);
+
+ if (session->p.op_type != ODP_CRYPTO_OP_TYPE_BASIC)
+ pkt = pkt_out[i];
+
+ rc = crypto_int(pkt_in[i], &pkt, &param[i]);
+ if (rc < 0)
+ break;
+
+ event = odp_packet_to_event(pkt);
+ if (odp_queue_enq(session->p.compl_queue, event)) {
+ odp_event_free(event);
+ break;
+ }
+ }
+
+ return i;
+}
diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c
index b9d319861..3ca27ca00 100644
--- a/platform/linux-generic/odp_crypto_null.c
+++ b/platform/linux-generic/odp_crypto_null.c
@@ -1,5 +1,5 @@
/* Copyright (c) 2014-2018, Linaro Limited
- * Copyright (c) 2021-2022, Nokia
+ * Copyright (c) 2021-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -201,7 +201,13 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param,
_ODP_ERR("Crypto is disabled\n");
/* Dummy output to avoid compiler warning about uninitialized
* variables */
- *status = ODP_CRYPTO_SES_CREATE_ERR_ENOMEM;
+ *status = ODP_CRYPTO_SES_ERR_ENOMEM;
+ *session_out = ODP_CRYPTO_SESSION_INVALID;
+ return -1;
+ }
+
+ if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) {
+ *status = ODP_CRYPTO_SES_ERR_PARAMS;
*session_out = ODP_CRYPTO_SESSION_INVALID;
return -1;
}
@@ -209,7 +215,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param,
/* Allocate memory for this session */
session = alloc_session();
if (NULL == session) {
- *status = ODP_CRYPTO_SES_CREATE_ERR_ENOMEM;
+ *status = ODP_CRYPTO_SES_ERR_ENOMEM;
goto err;
}
@@ -227,7 +233,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param,
/* Check result */
if (rc) {
- *status = ODP_CRYPTO_SES_CREATE_ERR_INV_CIPHER;
+ *status = ODP_CRYPTO_SES_ERR_CIPHER;
goto err;
}
@@ -242,13 +248,13 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param,
/* Check result */
if (rc) {
- *status = ODP_CRYPTO_SES_CREATE_ERR_INV_AUTH;
+ *status = ODP_CRYPTO_SES_ERR_AUTH;
goto err;
}
/* We're happy */
*session_out = (intptr_t)session;
- *status = ODP_CRYPTO_SES_CREATE_ERR_NONE;
+ *status = ODP_CRYPTO_SES_ERR_NONE;
return 0;
err:
@@ -281,6 +287,10 @@ odp_crypto_operation(odp_crypto_op_param_t *param,
odp_crypto_op_result_t local_result;
int rc;
+ if (((odp_crypto_generic_session_t *)(intptr_t)param->session)->p.op_type !=
+ ODP_CRYPTO_OP_TYPE_LEGACY)
+ return -1;
+
packet_param.session = param->session;
packet_param.cipher_iv_ptr = param->cipher_iv_ptr;
packet_param.auth_iv_ptr = param->auth_iv_ptr;
@@ -449,43 +459,6 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl)
return (uint64_t)hdl;
}
-odp_packet_t odp_crypto_packet_from_event(odp_event_t ev)
-{
- /* This check not mandated by the API specification */
- _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET);
- _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO);
-
- return odp_packet_from_event(ev);
-}
-
-odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt)
-{
- return odp_packet_to_event(pkt);
-}
-
-static
-odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt)
-{
- odp_packet_hdr_t *hdr = packet_hdr(pkt);
-
- return &hdr->crypto_op_result;
-}
-
-int odp_crypto_result(odp_crypto_packet_result_t *result,
- odp_packet_t packet)
-{
- odp_crypto_packet_result_t *op_result;
-
- _ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) ==
- ODP_EVENT_PACKET_CRYPTO);
-
- op_result = get_op_result_from_packet(packet);
-
- memcpy(result, op_result, sizeof(*result));
-
- return 0;
-}
-
static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src)
{
int md_copy;
@@ -514,6 +487,9 @@ static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *sessio
{
int rc;
+ if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC))
+ return pkt_in;
+
if (odp_likely(pkt_in == pkt_out))
return pkt_out;
@@ -555,7 +531,7 @@ int crypto_int(odp_packet_t pkt_in,
/* Fill in result */
packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO);
- op_result = get_op_result_from_packet(out_pkt);
+ op_result = &packet_hdr(out_pkt)->crypto_op_result;
op_result->cipher_status.alg_err = ODP_CRYPTO_ALG_ERR_NONE;
op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
op_result->auth_status.alg_err = ODP_CRYPTO_ALG_ERR_NONE;
@@ -603,7 +579,9 @@ int odp_crypto_op_enq(const odp_packet_t pkt_in[],
_ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode);
_ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue);
- pkt = pkt_out[i];
+ if (session->p.op_type != ODP_CRYPTO_OP_TYPE_BASIC)
+ pkt = pkt_out[i];
+
rc = crypto_int(pkt_in[i], &pkt, &param[i]);
if (rc < 0)
break;
diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c
index 0f637850f..19925fc09 100644
--- a/platform/linux-generic/odp_crypto_openssl.c
+++ b/platform/linux-generic/odp_crypto_openssl.c
@@ -1,5 +1,5 @@
/* Copyright (c) 2014-2018, Linaro Limited
- * Copyright (c) 2021-2022, Nokia
+ * Copyright (c) 2021-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -217,14 +217,12 @@ struct odp_crypto_generic_session_t {
odp_crypto_session_param_t p;
odp_bool_t do_cipher_first;
+ uint8_t cipher_bit_mode : 1;
+ uint8_t cipher_range_used : 1;
+ uint8_t auth_range_used : 1;
struct {
-#if ODP_DEPRECATED_API
- /* Copy of session IV data */
- uint8_t iv_data[EVP_MAX_IV_LENGTH];
-#endif
uint8_t key_data[EVP_MAX_KEY_LENGTH];
-
const EVP_CIPHER *evp_cipher;
crypto_func_t func;
crypto_init_func_t init;
@@ -232,9 +230,6 @@ struct odp_crypto_generic_session_t {
struct {
uint8_t key[EVP_MAX_KEY_LENGTH];
-#if ODP_DEPRECATED_API
- uint8_t iv_data[EVP_MAX_IV_LENGTH];
-#endif
union {
const EVP_MD *evp_md;
const EVP_CIPHER *evp_cipher;
@@ -710,23 +705,11 @@ int packet_cmac_eia2(odp_packet_t pkt,
uint8_t *hash)
{
CMAC_CTX *ctx = local.cmac_ctx[session->idx];
- void *iv_ptr;
+ void *iv_ptr = param->auth_iv_ptr;
uint32_t offset = param->auth_range.offset;
uint32_t len = param->auth_range.length;
size_t outlen;
-#if ODP_DEPRECATED_API
- if (param->auth_iv_ptr)
- iv_ptr = param->auth_iv_ptr;
- else if (session->p.auth_iv.data)
- iv_ptr = session->auth.iv_data;
- else
- return ODP_CRYPTO_ALG_ERR_IV_INVALID;
-#else
- iv_ptr = param->auth_iv_ptr;
- _ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL);
-#endif
-
_ODP_ASSERT(offset + len <= odp_packet_len(pkt));
/* Reinitialize CMAC calculation without resetting the key */
@@ -1073,22 +1056,9 @@ odp_crypto_alg_err_t cipher_encrypt(odp_packet_t pkt,
odp_crypto_generic_session_t *session)
{
EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx];
- void *iv_ptr;
int ret;
-#if ODP_DEPRECATED_API
- if (param->cipher_iv_ptr)
- iv_ptr = param->cipher_iv_ptr;
- else if (session->p.cipher_iv.data)
- iv_ptr = session->cipher.iv_data;
- else
- return ODP_CRYPTO_ALG_ERR_IV_INVALID;
-#else
- iv_ptr = param->cipher_iv_ptr;
- _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL);
-#endif
-
- EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr);
+ EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr);
ret = internal_encrypt(ctx, pkt, param);
@@ -1112,22 +1082,9 @@ odp_crypto_alg_err_t cipher_decrypt(odp_packet_t pkt,
odp_crypto_generic_session_t *session)
{
EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx];
- void *iv_ptr;
int ret;
-#if ODP_DEPRECATED_API
- if (param->cipher_iv_ptr)
- iv_ptr = param->cipher_iv_ptr;
- else if (session->p.cipher_iv.data)
- iv_ptr = session->cipher.iv_data;
- else
- return ODP_CRYPTO_ALG_ERR_IV_INVALID;
-#else
- iv_ptr = param->cipher_iv_ptr;
- _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL);
-#endif
-
- EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr);
+ EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr);
ret = internal_decrypt(ctx, pkt, param);
@@ -1172,7 +1129,6 @@ odp_crypto_alg_err_t cipher_encrypt_bits(odp_packet_t pkt,
odp_crypto_generic_session_t *session)
{
EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx];
- void *iv_ptr;
int dummy_len = 0;
int cipher_len;
uint32_t in_len = (param->cipher_range.length + 7) / 8;
@@ -1183,18 +1139,7 @@ odp_crypto_alg_err_t cipher_encrypt_bits(odp_packet_t pkt,
/* Range offset is in bits in bit mode but must be divisible by 8. */
offset = param->cipher_range.offset / 8;
-#if ODP_DEPRECATED_API
- if (param->cipher_iv_ptr)
- iv_ptr = param->cipher_iv_ptr;
- else if (session->p.cipher_iv.data)
- iv_ptr = session->cipher.iv_data;
- else
- return ODP_CRYPTO_ALG_ERR_IV_INVALID;
-#else
- iv_ptr = param->cipher_iv_ptr;
- _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL);
-#endif
- EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr);
+ EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr);
odp_packet_copy_to_mem(pkt, offset, in_len, data);
@@ -1216,7 +1161,6 @@ odp_crypto_alg_err_t cipher_decrypt_bits(odp_packet_t pkt,
odp_crypto_generic_session_t *session)
{
EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx];
- void *iv_ptr;
int dummy_len = 0;
int cipher_len;
uint32_t in_len = (param->cipher_range.length + 7) / 8;
@@ -1227,18 +1171,7 @@ odp_crypto_alg_err_t cipher_decrypt_bits(odp_packet_t pkt,
/* Range offset is in bits in bit mode but must be divisible by 8. */
offset = param->cipher_range.offset / 8;
-#if ODP_DEPRECATED_API
- if (param->cipher_iv_ptr)
- iv_ptr = param->cipher_iv_ptr;
- else if (session->p.cipher_iv.data)
- iv_ptr = session->cipher.iv_data;
- else
- return ODP_CRYPTO_ALG_ERR_IV_INVALID;
-#else
- iv_ptr = param->cipher_iv_ptr;
- _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL);
-#endif
- EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr);
+ EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr);
odp_packet_copy_to_mem(pkt, offset, in_len, data);
@@ -1266,6 +1199,7 @@ static int process_cipher_param_bits(odp_crypto_generic_session_t *session,
session->p.cipher_iv_len)
return -1;
+ session->cipher_bit_mode = 1;
session->cipher.evp_cipher = cipher;
memcpy(session->cipher.key_data, session->p.cipher_key.data,
@@ -1303,24 +1237,11 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_packet_t pkt,
EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx];
const uint8_t *aad_head = param->aad_ptr;
uint32_t aad_len = session->p.auth_aad_len;
- void *iv_ptr;
int dummy_len = 0;
uint8_t block[EVP_MAX_MD_SIZE];
int ret;
-#if ODP_DEPRECATED_API
- if (param->cipher_iv_ptr)
- iv_ptr = param->cipher_iv_ptr;
- else if (session->p.cipher_iv.data)
- iv_ptr = session->cipher.iv_data;
- else
- return ODP_CRYPTO_ALG_ERR_IV_INVALID;
-#else
- iv_ptr = param->cipher_iv_ptr;
- _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL);
-#endif
-
- EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr);
+ EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr);
/* Authenticate header data (if any) without encrypting them */
if (aad_len > 0)
@@ -1359,23 +1280,10 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_packet_t pkt,
const uint8_t *aad_head = param->aad_ptr;
uint32_t aad_len = session->p.auth_aad_len;
int dummy_len = 0;
- void *iv_ptr;
uint8_t block[EVP_MAX_MD_SIZE];
int ret;
-#if ODP_DEPRECATED_API
- if (param->cipher_iv_ptr)
- iv_ptr = param->cipher_iv_ptr;
- else if (session->p.cipher_iv.data)
- iv_ptr = session->cipher.iv_data;
- else
- return ODP_CRYPTO_ALG_ERR_IV_INVALID;
-#else
- iv_ptr = param->cipher_iv_ptr;
- _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL);
-#endif
-
- EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr);
+ EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr);
odp_packet_copy_to_mem(pkt, param->hash_result_offset,
session->p.auth_digest_len, block);
@@ -1440,23 +1348,10 @@ odp_crypto_alg_err_t aes_gmac_gen(odp_packet_t pkt,
odp_crypto_generic_session_t *session)
{
EVP_CIPHER_CTX *ctx = local.mac_cipher_ctx[session->idx];
- void *iv_ptr;
uint8_t block[EVP_MAX_MD_SIZE];
int ret;
-#if ODP_DEPRECATED_API
- if (param->auth_iv_ptr)
- iv_ptr = param->auth_iv_ptr;
- else if (session->p.auth_iv.data)
- iv_ptr = session->auth.iv_data;
- else
- return ODP_CRYPTO_ALG_ERR_IV_INVALID;
-#else
- iv_ptr = param->auth_iv_ptr;
- _ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL);
-#endif
-
- EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr);
+ EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, param->auth_iv_ptr);
ret = internal_aad(ctx, pkt, param, true);
@@ -1487,23 +1382,10 @@ odp_crypto_alg_err_t aes_gmac_check(odp_packet_t pkt,
odp_crypto_generic_session_t *session)
{
EVP_CIPHER_CTX *ctx = local.mac_cipher_ctx[session->idx];
- void *iv_ptr;
uint8_t block[EVP_MAX_MD_SIZE];
int ret;
-#if ODP_DEPRECATED_API
- if (param->auth_iv_ptr)
- iv_ptr = param->auth_iv_ptr;
- else if (session->p.auth_iv.data)
- iv_ptr = session->auth.iv_data;
- else
- return ODP_CRYPTO_ALG_ERR_IV_INVALID;
-#else
- iv_ptr = param->auth_iv_ptr;
- _ODP_ASSERT(session->p.auth_iv_len == 0 || iv_ptr != NULL);
-#endif
-
- EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr);
+ EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, param->auth_iv_ptr);
odp_packet_copy_to_mem(pkt, param->hash_result_offset,
session->p.auth_digest_len, block);
@@ -1568,7 +1450,6 @@ odp_crypto_alg_err_t aes_ccm_encrypt(odp_packet_t pkt,
EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx];
const uint8_t *aad_head = param->aad_ptr;
uint32_t aad_len = session->p.auth_aad_len;
- void *iv_ptr;
int dummy_len = 0;
int cipher_len;
uint32_t in_len = param->cipher_range.length;
@@ -1576,21 +1457,9 @@ odp_crypto_alg_err_t aes_ccm_encrypt(odp_packet_t pkt,
uint8_t block[EVP_MAX_MD_SIZE];
int ret;
-#if ODP_DEPRECATED_API
- if (param->cipher_iv_ptr)
- iv_ptr = param->cipher_iv_ptr;
- else if (session->p.cipher_iv.data)
- iv_ptr = session->cipher.iv_data;
- else
- return ODP_CRYPTO_ALG_ERR_IV_INVALID;
-#else
- iv_ptr = param->cipher_iv_ptr;
- _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL);
-#endif
-
EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG,
session->p.auth_digest_len, NULL);
- EVP_EncryptInit_ex(ctx, NULL, NULL, session->cipher.key_data, iv_ptr);
+ EVP_EncryptInit_ex(ctx, NULL, NULL, session->cipher.key_data, param->cipher_iv_ptr);
/* Set len */
EVP_EncryptUpdate(ctx, NULL, &dummy_len, NULL, in_len);
@@ -1640,7 +1509,6 @@ odp_crypto_alg_err_t aes_ccm_decrypt(odp_packet_t pkt,
EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx];
const uint8_t *aad_head = param->aad_ptr;
uint32_t aad_len = session->p.auth_aad_len;
- void *iv_ptr;
int dummy_len = 0;
int cipher_len;
uint32_t in_len = param->cipher_range.length;
@@ -1648,23 +1516,11 @@ odp_crypto_alg_err_t aes_ccm_decrypt(odp_packet_t pkt,
uint8_t block[EVP_MAX_MD_SIZE];
int ret;
-#if ODP_DEPRECATED_API
- if (param->cipher_iv_ptr)
- iv_ptr = param->cipher_iv_ptr;
- else if (session->p.cipher_iv.data)
- iv_ptr = session->cipher.iv_data;
- else
- return ODP_CRYPTO_ALG_ERR_IV_INVALID;
-#else
- iv_ptr = param->cipher_iv_ptr;
- _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL);
-#endif
-
odp_packet_copy_to_mem(pkt, param->hash_result_offset,
session->p.auth_digest_len, block);
EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG,
session->p.auth_digest_len, block);
- EVP_DecryptInit_ex(ctx, NULL, NULL, session->cipher.key_data, iv_ptr);
+ EVP_DecryptInit_ex(ctx, NULL, NULL, session->cipher.key_data, param->cipher_iv_ptr);
/* Set len */
EVP_DecryptUpdate(ctx, NULL, &dummy_len, NULL, in_len);
@@ -1725,26 +1581,13 @@ odp_crypto_alg_err_t xts_encrypt(odp_packet_t pkt,
odp_crypto_generic_session_t *session)
{
EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx];
- void *iv_ptr;
int dummy_len = 0;
int cipher_len;
uint32_t in_len = param->cipher_range.length;
uint8_t data[in_len];
int ret;
-#if ODP_DEPRECATED_API
- if (param->cipher_iv_ptr)
- iv_ptr = param->cipher_iv_ptr;
- else if (session->p.cipher_iv.data)
- iv_ptr = session->cipher.iv_data;
- else
- return ODP_CRYPTO_ALG_ERR_IV_INVALID;
-#else
- iv_ptr = param->cipher_iv_ptr;
- _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL);
-#endif
-
- EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr);
+ EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr);
odp_packet_copy_to_mem(pkt, param->cipher_range.offset, in_len,
data);
@@ -1767,26 +1610,13 @@ odp_crypto_alg_err_t xts_decrypt(odp_packet_t pkt,
odp_crypto_generic_session_t *session)
{
EVP_CIPHER_CTX *ctx = local.cipher_ctx[session->idx];
- void *iv_ptr;
int dummy_len = 0;
int cipher_len;
uint32_t in_len = param->cipher_range.length;
uint8_t data[in_len];
int ret;
-#if ODP_DEPRECATED_API
- if (param->cipher_iv_ptr)
- iv_ptr = param->cipher_iv_ptr;
- else if (session->p.cipher_iv.data)
- iv_ptr = session->cipher.iv_data;
- else
- return ODP_CRYPTO_ALG_ERR_IV_INVALID;
-#else
- iv_ptr = param->cipher_iv_ptr;
- _ODP_ASSERT(session->p.cipher_iv_len == 0 || iv_ptr != NULL);
-#endif
-
- EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr);
+ EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, param->cipher_iv_ptr);
odp_packet_copy_to_mem(pkt, param->cipher_range.offset, in_len,
data);
@@ -2222,6 +2052,10 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param,
/* Copy parameters */
session->p = *param;
+ session->cipher_bit_mode = 0;
+ session->auth_range_used = 1;
+ session->cipher_range_used = 1;
+
if (session->p.cipher_iv_len > EVP_MAX_IV_LENGTH) {
_ODP_DBG("Maximum IV length exceeded\n");
*status = ODP_CRYPTO_SES_ERR_CIPHER;
@@ -2234,17 +2068,6 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param,
goto err;
}
-#if ODP_DEPRECATED_API
- /* Copy IV data */
- if (session->p.cipher_iv.data)
- memcpy(session->cipher.iv_data, session->p.cipher_iv.data,
- session->p.cipher_iv.length);
-
- if (session->p.auth_iv.data)
- memcpy(session->auth.iv_data, session->p.auth_iv.data,
- session->p.auth_iv.length);
-#endif
-
/* Derive order */
if (ODP_CRYPTO_OP_ENCODE == param->op)
session->do_cipher_first = param->auth_cipher_text;
@@ -2256,6 +2079,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param,
case ODP_CIPHER_ALG_NULL:
session->cipher.func = null_crypto_routine;
session->cipher.init = null_crypto_init_routine;
+ session->cipher_range_used = 0;
rc = 0;
break;
case ODP_CIPHER_ALG_3DES_CBC:
@@ -2376,6 +2200,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param,
case ODP_AUTH_ALG_NULL:
session->auth.func = null_crypto_routine;
session->auth.init = null_crypto_init_routine;
+ session->auth_range_used = 0;
rc = 0;
break;
case ODP_AUTH_ALG_MD5_HMAC:
@@ -2409,6 +2234,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param,
} else {
rc = -1;
}
+ session->auth_range_used = 0;
break;
case ODP_AUTH_ALG_AES_GMAC:
if (param->auth_key.length == 16)
@@ -2430,6 +2256,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param,
} else {
rc = -1;
}
+ session->auth_range_used = 0;
break;
case ODP_AUTH_ALG_AES_CMAC:
if (param->auth_key.length == 16)
@@ -2455,6 +2282,7 @@ odp_crypto_session_create(const odp_crypto_session_param_t *param,
} else {
rc = -1;
}
+ session->auth_range_used = 0;
break;
#endif
case ODP_AUTH_ALG_AES_EIA2:
@@ -2530,6 +2358,10 @@ odp_crypto_operation(odp_crypto_op_param_t *param,
odp_crypto_op_result_t local_result;
int rc;
+ if (((odp_crypto_generic_session_t *)(intptr_t)param->session)->p.op_type !=
+ ODP_CRYPTO_OP_TYPE_LEGACY)
+ return -1;
+
packet_param.session = param->session;
packet_param.cipher_iv_ptr = param->cipher_iv_ptr;
packet_param.auth_iv_ptr = param->auth_iv_ptr;
@@ -2778,43 +2610,6 @@ uint64_t odp_crypto_session_to_u64(odp_crypto_session_t hdl)
return (uint64_t)hdl;
}
-odp_packet_t odp_crypto_packet_from_event(odp_event_t ev)
-{
- /* This check not mandated by the API specification */
- _ODP_ASSERT(odp_event_type(ev) == ODP_EVENT_PACKET);
- _ODP_ASSERT(odp_event_subtype(ev) == ODP_EVENT_PACKET_CRYPTO);
-
- return odp_packet_from_event(ev);
-}
-
-odp_event_t odp_crypto_packet_to_event(odp_packet_t pkt)
-{
- return odp_packet_to_event(pkt);
-}
-
-static
-odp_crypto_packet_result_t *get_op_result_from_packet(odp_packet_t pkt)
-{
- odp_packet_hdr_t *hdr = packet_hdr(pkt);
-
- return &hdr->crypto_op_result;
-}
-
-int odp_crypto_result(odp_crypto_packet_result_t *result,
- odp_packet_t packet)
-{
- odp_crypto_packet_result_t *op_result;
-
- _ODP_ASSERT(odp_event_subtype(odp_packet_to_event(packet)) ==
- ODP_EVENT_PACKET_CRYPTO);
-
- op_result = get_op_result_from_packet(packet);
-
- memcpy(result, op_result, sizeof(*result));
-
- return 0;
-}
-
static int copy_data_and_metadata(odp_packet_t dst, odp_packet_t src)
{
int md_copy;
@@ -2843,6 +2638,9 @@ static odp_packet_t get_output_packet(const odp_crypto_generic_session_t *sessio
{
int rc;
+ if (odp_likely(session->p.op_type == ODP_CRYPTO_OP_TYPE_BASIC))
+ return pkt_in;
+
if (odp_likely(pkt_in == pkt_out))
return pkt_out;
@@ -2894,6 +2692,8 @@ int crypto_int(odp_packet_t pkt_in,
goto out;
}
}
+ _ODP_ASSERT(session->p.cipher_iv_len == 0 || param->cipher_iv_ptr != NULL);
+ _ODP_ASSERT(session->p.auth_iv_len == 0 || param->auth_iv_ptr != NULL);
crypto_init(session);
@@ -2909,7 +2709,7 @@ int crypto_int(odp_packet_t pkt_in,
out:
/* Fill in result */
packet_subtype_set(out_pkt, ODP_EVENT_PACKET_CRYPTO);
- op_result = get_op_result_from_packet(out_pkt);
+ op_result = &packet_hdr(out_pkt)->crypto_op_result;
op_result->cipher_status.alg_err = rc_cipher;
op_result->cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
op_result->auth_status.alg_err = rc_auth;
@@ -2924,6 +2724,113 @@ out:
return 0;
}
+/*
+ * Copy cipher range and auth range from src to dst,
+ * with shifting by dst_offset_shift.
+ */
+static void copy_ranges(odp_packet_t dst,
+ odp_packet_t src,
+ const odp_crypto_generic_session_t *session,
+ const odp_crypto_packet_op_param_t *param)
+{
+ odp_packet_data_range_t c_range = param->cipher_range;
+ odp_packet_data_range_t a_range = param->auth_range;
+ int32_t shift = param->dst_offset_shift;
+ int rc;
+
+ if (session->cipher_bit_mode) {
+ c_range.offset /= 8;
+ c_range.length = (c_range.length + 7) / 8;
+ }
+
+ if (session->cipher_range_used) {
+ rc = odp_packet_copy_from_pkt(dst, c_range.offset + shift,
+ src, c_range.offset,
+ c_range.length);
+ if (rc) {
+ _ODP_ERR("cipher range copying failed\n");
+ return;
+ }
+ }
+ if (session->auth_range_used) {
+ rc = odp_packet_copy_from_pkt(dst, a_range.offset + shift,
+ src, a_range.offset,
+ a_range.length);
+ if (rc) {
+ _ODP_ERR("auth range copying failed\n");
+ return;
+ }
+ }
+}
+
+static int crypto_int_oop_encode(odp_packet_t pkt_in,
+ odp_packet_t *pkt_out,
+ const odp_crypto_generic_session_t *session,
+ const odp_crypto_packet_op_param_t *param)
+{
+ odp_crypto_packet_op_param_t new_param = *param;
+ const uint32_t scale = session->cipher_bit_mode ? 8 : 1;
+
+ copy_ranges(*pkt_out, pkt_in, session, param);
+
+ new_param.cipher_range.offset += param->dst_offset_shift * scale;
+ new_param.auth_range.offset += param->dst_offset_shift;
+
+ return crypto_int(*pkt_out, pkt_out, &new_param);
+}
+
+static int crypto_int_oop_decode(odp_packet_t pkt_in,
+ odp_packet_t *pkt_out,
+ const odp_crypto_generic_session_t *session,
+ const odp_crypto_packet_op_param_t *param)
+{
+ odp_packet_t copy;
+ int rc;
+
+ copy = odp_packet_copy(pkt_in, odp_packet_pool(pkt_in));
+ if (copy == ODP_PACKET_INVALID)
+ return -1;
+
+ rc = crypto_int(copy, &copy, param);
+ if (rc < 0) {
+ odp_packet_free(copy);
+ return rc;
+ }
+
+ copy_ranges(*pkt_out, copy, session, param);
+
+ packet_subtype_set(*pkt_out, ODP_EVENT_PACKET_CRYPTO);
+ packet_hdr(*pkt_out)->crypto_op_result = packet_hdr(copy)->crypto_op_result;
+ odp_packet_free(copy);
+
+ return 0;
+}
+
+/*
+ * Slow out-of-place operation implemented using copying and in-place operation
+ */
+static int crypto_int_oop(odp_packet_t pkt_in,
+ odp_packet_t *pkt_out,
+ const odp_crypto_packet_op_param_t *param)
+{
+ odp_crypto_generic_session_t *session;
+ int rc;
+
+ session = (odp_crypto_generic_session_t *)(intptr_t)param->session;
+
+ if (session->p.op == ODP_CRYPTO_OP_ENCODE)
+ rc = crypto_int_oop_encode(pkt_in, pkt_out, session, param);
+ else
+ rc = crypto_int_oop_decode(pkt_in, pkt_out, session, param);
+ if (rc)
+ return rc;
+
+ if (session->p.op_mode == ODP_CRYPTO_ASYNC)
+ packet_hdr(*pkt_out)->crypto_op_result.pkt_in = pkt_in;
+
+ return 0;
+}
+
int odp_crypto_op(const odp_packet_t pkt_in[],
odp_packet_t pkt_out[],
const odp_crypto_packet_op_param_t param[],
@@ -2936,7 +2843,10 @@ int odp_crypto_op(const odp_packet_t pkt_in[],
session = (odp_crypto_generic_session_t *)(intptr_t)param[i].session;
_ODP_ASSERT(ODP_CRYPTO_SYNC == session->p.op_mode);
- rc = crypto_int(pkt_in[i], &pkt_out[i], &param[i]);
+ if (odp_unlikely(session->p.op_type == ODP_CRYPTO_OP_TYPE_OOP))
+ rc = crypto_int_oop(pkt_in[i], &pkt_out[i], &param[i]);
+ else
+ rc = crypto_int(pkt_in[i], &pkt_out[i], &param[i]);
if (rc < 0)
break;
}
@@ -2959,8 +2869,13 @@ int odp_crypto_op_enq(const odp_packet_t pkt_in[],
_ODP_ASSERT(ODP_CRYPTO_ASYNC == session->p.op_mode);
_ODP_ASSERT(ODP_QUEUE_INVALID != session->p.compl_queue);
- pkt = pkt_out[i];
- rc = crypto_int(pkt_in[i], &pkt, &param[i]);
+ if (session->p.op_type != ODP_CRYPTO_OP_TYPE_BASIC)
+ pkt = pkt_out[i];
+
+ if (odp_unlikely(session->p.op_type == ODP_CRYPTO_OP_TYPE_OOP))
+ rc = crypto_int_oop(pkt_in[i], &pkt, &param[i]);
+ else
+ rc = crypto_int(pkt_in[i], &pkt, &param[i]);
if (rc < 0)
break;
diff --git a/platform/linux-generic/odp_event.c b/platform/linux-generic/odp_event.c
index c4e0f2c9d..edf77e2dc 100644
--- a/platform/linux-generic/odp_event.c
+++ b/platform/linux-generic/odp_event.c
@@ -1,5 +1,5 @@
/* Copyright (c) 2015-2018, Linaro Limited
- * Copyright (c) 2020-2022, Nokia
+ * Copyright (c) 2020-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -18,6 +18,7 @@
#include <odp_debug_internal.h>
#include <odp_packet_internal.h>
#include <odp_event_internal.h>
+#include <odp_event_validation_internal.h>
#include <odp_event_vector_internal.h>
/* Inlined API functions */
@@ -41,13 +42,15 @@ _odp_event_inline_offset ODP_ALIGNED_CACHE = {
#include <odp/visibility_end.h>
-void odp_event_free(odp_event_t event)
+static inline void event_free(odp_event_t event, _odp_ev_id_t id)
{
switch (odp_event_type(event)) {
case ODP_EVENT_BUFFER:
+ _odp_buffer_validate(odp_buffer_from_event(event), id);
odp_buffer_free(odp_buffer_from_event(event));
break;
case ODP_EVENT_PACKET:
+ _odp_packet_validate(odp_packet_from_event(event), id);
odp_packet_free(odp_packet_from_event(event));
break;
case ODP_EVENT_PACKET_VECTOR:
@@ -75,17 +78,21 @@ void odp_event_free(odp_event_t event)
}
}
-void odp_event_free_multi(const odp_event_t event[], int num)
+void odp_event_free(odp_event_t event)
{
- int i;
+ event_free(event, _ODP_EV_EVENT_FREE);
+}
- for (i = 0; i < num; i++)
- odp_event_free(event[i]);
+void odp_event_free_multi(const odp_event_t event[], int num)
+{
+ for (int i = 0; i < num; i++)
+ event_free(event[i], _ODP_EV_EVENT_FREE_MULTI);
}
void odp_event_free_sp(const odp_event_t event[], int num)
{
- odp_event_free_multi(event, num);
+ for (int i = 0; i < num; i++)
+ event_free(event[i], _ODP_EV_EVENT_FREE_SP);
}
uint64_t odp_event_to_u64(odp_event_t hdl)
@@ -103,9 +110,9 @@ int odp_event_is_valid(odp_event_t event)
switch (odp_event_type(event)) {
case ODP_EVENT_BUFFER:
- /* Fall through */
+ return !_odp_buffer_validate(odp_buffer_from_event(event), _ODP_EV_EVENT_IS_VALID);
case ODP_EVENT_PACKET:
- /* Fall through */
+ return !_odp_packet_validate(odp_packet_from_event(event), _ODP_EV_EVENT_IS_VALID);
case ODP_EVENT_TIMEOUT:
/* Fall through */
#if ODP_DEPRECATED_API
diff --git a/platform/linux-generic/odp_event_validation.c b/platform/linux-generic/odp_event_validation.c
new file mode 100644
index 000000000..c2d430f1a
--- /dev/null
+++ b/platform/linux-generic/odp_event_validation.c
@@ -0,0 +1,260 @@
+/* Copyright (c) 2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp/api/atomic.h>
+#include <odp/api/buffer.h>
+#include <odp/api/debug.h>
+#include <odp/api/event.h>
+#include <odp/api/hints.h>
+#include <odp/api/packet.h>
+#include <odp/api/shared_memory.h>
+
+#include <odp_buffer_internal.h>
+#include <odp_debug_internal.h>
+#include <odp_event_internal.h>
+#include <odp_event_validation_internal.h>
+#include <odp_global_data.h>
+#include <odp_init_internal.h>
+#include <odp_libconfig_internal.h>
+#include <odp_macros_internal.h>
+#include <odp_print_internal.h>
+
+#include <inttypes.h>
+#include <string.h>
+
+#define EVENT_VALIDATION_NONE 0
+#define EVENT_VALIDATION_WARN 1
+#define EVENT_VALIDATION_ABORT 2
+
+#define EVENT_DATA_PRINT_MAX_LEN 128
+
+typedef struct {
+ odp_atomic_u64_t err_count[_ODP_EV_MAX];
+ odp_shm_t shm;
+
+} event_validation_global_t;
+
+typedef struct {
+ const char *str;
+} _odp_ev_info_t;
+
+static event_validation_global_t *_odp_ev_glb;
+
+#if _ODP_EVENT_VALIDATION
+
+/* Table for mapping function IDs to API function names */
+static const _odp_ev_info_t ev_info_tbl[] = {
+ [_ODP_EV_BUFFER_FREE] = {.str = "odp_buffer_free()"},
+ [_ODP_EV_BUFFER_FREE_MULTI] = {.str = "odp_buffer_free_multi()"},
+ [_ODP_EV_BUFFER_IS_VALID] = {.str = "odp_buffer_is_valid()"},
+ [_ODP_EV_EVENT_FREE] = {.str = "odp_event_free()"},
+ [_ODP_EV_EVENT_FREE_MULTI] = {.str = "odp_event_free_multi()"},
+ [_ODP_EV_EVENT_FREE_SP] = {.str = "odp_event_free()_sp"},
+ [_ODP_EV_EVENT_IS_VALID] = {.str = "odp_event_is_valid()"},
+ [_ODP_EV_PACKET_FREE] = {.str = "odp_packet_free()"},
+ [_ODP_EV_PACKET_FREE_MULTI] = {.str = "odp_packet_free_multi()"},
+ [_ODP_EV_PACKET_FREE_SP] = {.str = "odp_packet_free_sp()"},
+ [_ODP_EV_PACKET_IS_VALID] = {.str = "odp_packet_is_valid()"},
+ [_ODP_EV_QUEUE_ENQ] = {.str = "odp_queue_enq()"},
+ [_ODP_EV_QUEUE_ENQ_MULTI] = {.str = "odp_queue_enq_multi()"}
+};
+
+ODP_STATIC_ASSERT(_ODP_ARRAY_SIZE(ev_info_tbl) == _ODP_EV_MAX, "ev_info_tbl missing entries");
+
+static void print_event_data(odp_event_t event, odp_event_type_t type)
+{
+ const char *type_str;
+ const uint32_t bytes_per_row = 16;
+ uint32_t byte_len;
+ int num_rows, max_len, n;
+ int len = 0;
+ uint8_t *data;
+
+ if (type == ODP_EVENT_PACKET) {
+ odp_packet_t pkt = odp_packet_from_event(event);
+
+ data = odp_packet_data(pkt);
+ byte_len = odp_packet_seg_len(pkt);
+ type_str = "Packet";
+ } else {
+ odp_buffer_t buf = odp_buffer_from_event(event);
+
+ data = odp_buffer_addr(buf);
+ byte_len = odp_buffer_size(buf);
+ type_str = "Buffer";
+ }
+
+ if (byte_len > EVENT_DATA_PRINT_MAX_LEN)
+ byte_len = EVENT_DATA_PRINT_MAX_LEN;
+
+ num_rows = (byte_len + bytes_per_row - 1) / bytes_per_row;
+ max_len = 256 + (3 * byte_len) + (3 * num_rows);
+ n = max_len - 1;
+
+ char str[max_len];
+
+ len += _odp_snprint(&str[len], n - len, "%s %p data %p:\n", type_str, event, data);
+ while (byte_len) {
+ uint32_t row_len = byte_len > bytes_per_row ? bytes_per_row : byte_len;
+
+ len += _odp_snprint(&str[len], n - len, " ");
+
+ for (uint32_t i = 0; i < row_len; i++)
+ len += _odp_snprint(&str[len], n - len, " %02x", data[i]);
+
+ len += _odp_snprint(&str[len], n - len, "\n");
+
+ byte_len -= row_len;
+ data += row_len;
+ }
+
+ _ODP_PRINT("%s\n", str);
+}
+
+static inline int validate_event_endmark(odp_event_t event, _odp_ev_id_t id, odp_event_type_t type)
+{
+ uint64_t err_count;
+ uint64_t *endmark_ptr = _odp_event_endmark_get_ptr(event);
+
+ if (odp_likely(*endmark_ptr == _ODP_EV_ENDMARK_VAL))
+ return 0;
+
+ err_count = odp_atomic_fetch_inc_u64(&_odp_ev_glb->err_count[id]) + 1;
+
+ _ODP_ERR("Event %p endmark mismatch in %s: endmark=0x%" PRIx64 " (expected 0x%" PRIx64 ") "
+ "err_count=%" PRIu64 "\n", event, ev_info_tbl[id].str, *endmark_ptr,
+ _ODP_EV_ENDMARK_VAL, err_count);
+
+ print_event_data(event, type);
+
+ if (_ODP_EVENT_VALIDATION == EVENT_VALIDATION_ABORT)
+ _ODP_ABORT("Abort due to event %p endmark mismatch\n", event);
+
+ /* Fix endmark value */
+ _odp_event_endmark_set(event);
+
+ return -1;
+}
+
+static inline int buffer_validate(odp_buffer_t buf, _odp_ev_id_t id)
+{
+ return validate_event_endmark(odp_buffer_to_event(buf), id, ODP_EVENT_BUFFER);
+}
+
+static inline int packet_validate(odp_packet_t pkt, _odp_ev_id_t id)
+{
+ return validate_event_endmark(odp_packet_to_event(pkt), id, ODP_EVENT_PACKET);
+}
+
+static inline int event_validate(odp_event_t event, int id)
+{
+ if (odp_event_type(event) == ODP_EVENT_BUFFER)
+ return buffer_validate(odp_buffer_from_event(event), id);
+ if (odp_event_type(event) == ODP_EVENT_PACKET)
+ return packet_validate(odp_packet_from_event(event), id);
+ return 0;
+}
+
+/* Enable usage from API inline files */
+#include <odp/visibility_begin.h>
+
+int _odp_buffer_validate(odp_buffer_t buf, _odp_ev_id_t id)
+{
+ return buffer_validate(buf, id);
+}
+
+int _odp_buffer_validate_multi(const odp_buffer_t buf[], int num,
+ _odp_ev_id_t id)
+{
+ for (int i = 0; i < num; i++) {
+ if (odp_unlikely(buffer_validate(buf[i], id)))
+ return -1;
+ }
+ return 0;
+}
+
+int _odp_packet_validate(odp_packet_t pkt, _odp_ev_id_t id)
+{
+ return packet_validate(pkt, id);
+}
+
+int _odp_packet_validate_multi(const odp_packet_t pkt[], int num,
+ _odp_ev_id_t id)
+{
+ for (int i = 0; i < num; i++) {
+ if (odp_unlikely(packet_validate(pkt[i], id)))
+ return -1;
+ }
+ return 0;
+}
+
+int _odp_event_validate(odp_event_t event, _odp_ev_id_t id)
+{
+ return event_validate(event, id);
+}
+
+int _odp_event_validate_multi(const odp_event_t event[], int num,
+ _odp_ev_id_t id)
+{
+ for (int i = 0; i < num; i++) {
+ if (odp_unlikely(event_validate(event[i], id)))
+ return -1;
+ }
+ return 0;
+}
+
+#include <odp/visibility_end.h>
+
+#endif /* _ODP_EVENT_VALIDATION */
+
+int _odp_event_validation_init_global(void)
+{
+ odp_shm_t shm;
+
+ _ODP_PRINT("\nEvent validation mode: %s\n\n",
+ _ODP_EVENT_VALIDATION == EVENT_VALIDATION_NONE ? "none" :
+ _ODP_EVENT_VALIDATION == EVENT_VALIDATION_WARN ? "warn" : "abort");
+
+ if (_ODP_EVENT_VALIDATION == EVENT_VALIDATION_NONE)
+ return 0;
+
+ shm = odp_shm_reserve("_odp_event_validation_global",
+ sizeof(event_validation_global_t),
+ ODP_CACHE_LINE_SIZE, ODP_SHM_EXPORT);
+ if (shm == ODP_SHM_INVALID)
+ return -1;
+
+ _odp_ev_glb = odp_shm_addr(shm);
+ if (_odp_ev_glb == NULL)
+ return -1;
+
+ memset(_odp_ev_glb, 0, sizeof(event_validation_global_t));
+ _odp_ev_glb->shm = shm;
+
+ for (int i = 0; i < _ODP_EV_MAX; i++)
+ odp_atomic_init_u64(&_odp_ev_glb->err_count[i], 0);
+
+ return 0;
+}
+
+int _odp_event_validation_term_global(void)
+{
+ int ret;
+
+ if (_ODP_EVENT_VALIDATION == EVENT_VALIDATION_NONE)
+ return 0;
+
+ if (_odp_ev_glb == NULL)
+ return 0;
+
+ ret = odp_shm_free(_odp_ev_glb->shm);
+ if (ret) {
+ _ODP_ERR("SHM free failed: %d\n", ret);
+ return -1;
+ }
+
+ return 0;
+}
diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c
index 284b3e566..bd27641aa 100644
--- a/platform/linux-generic/odp_init.c
+++ b/platform/linux-generic/odp_init.c
@@ -1,5 +1,5 @@
/* Copyright (c) 2013-2018, Linaro Limited
- * Copyright (c) 2021, Nokia
+ * Copyright (c) 2021-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -35,6 +35,7 @@ enum init_stage {
HASH_INIT,
THREAD_INIT,
POOL_INIT,
+ EVENT_VALIDATION_INIT,
STASH_INIT,
QUEUE_INIT,
SCHED_INIT,
@@ -242,6 +243,13 @@ static int term_global(enum init_stage stage)
}
/* Fall through */
+ case EVENT_VALIDATION_INIT:
+ if (_odp_event_validation_term_global()) {
+ _ODP_ERR("ODP event validation term failed.\n");
+ rc = -1;
+ }
+ /* Fall through */
+
case POOL_INIT:
if (_odp_pool_term_global()) {
_ODP_ERR("ODP buffer pool term failed.\n");
@@ -412,6 +420,12 @@ int odp_init_global(odp_instance_t *instance,
}
stage = POOL_INIT;
+ if (_odp_event_validation_init_global()) {
+ _ODP_ERR("ODP event validation init failed.\n");
+ goto init_failed;
+ }
+ stage = EVENT_VALIDATION_INIT;
+
if (_odp_stash_init_global()) {
_ODP_ERR("ODP stash init failed.\n");
goto init_failed;
diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c
index 1b3a90e6a..4cf265694 100644
--- a/platform/linux-generic/odp_ipsec_sad.c
+++ b/platform/linux-generic/odp_ipsec_sad.c
@@ -1,11 +1,12 @@
/* Copyright (c) 2017-2018, Linaro Limited
- * Copyright (c) 2018-2022, Nokia
+ * Copyright (c) 2018-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
#include <odp/api/atomic.h>
+#include <odp/api/crypto.h>
#include <odp/api/ipsec.h>
#include <odp/api/random.h>
#include <odp/api/shared_memory.h>
@@ -18,7 +19,7 @@
#include <odp_debug_internal.h>
#include <odp_ipsec_internal.h>
#include <odp_macros_internal.h>
-#include <odp_ring_mpmc_internal.h>
+#include <odp_ring_mpmc_u32_internal.h>
#include <odp_global_data.h>
#include <string.h>
@@ -102,7 +103,7 @@ typedef struct ODP_ALIGNED_CACHE ipsec_thread_local_s {
typedef struct ipsec_sa_table_t {
ipsec_sa_t ipsec_sa[CONFIG_IPSEC_MAX_NUM_SA];
struct ODP_ALIGNED_CACHE {
- ring_mpmc_t ipv4_id_ring;
+ ring_mpmc_u32_t ipv4_id_ring;
uint32_t ipv4_id_data[IPV4_ID_RING_SIZE] ODP_ALIGNED_CACHE;
} hot;
struct {
@@ -192,7 +193,7 @@ int _odp_ipsec_sad_init_global(void)
ipsec_sa_tbl->shm = shm;
ipsec_sa_tbl->max_num_sa = max_num_sa;
- ring_mpmc_init(&ipsec_sa_tbl->hot.ipv4_id_ring);
+ ring_mpmc_u32_init(&ipsec_sa_tbl->hot.ipv4_id_ring);
for (i = 0; i < thread_count_max; i++) {
/*
* Make the current ID block fully used, forcing allocation
@@ -210,11 +211,11 @@ int _odp_ipsec_sad_init_global(void)
for (i = 0; i < IPV4_ID_RING_SIZE - 1; i++) {
uint32_t data = i * IPV4_ID_BLOCK_SIZE;
- ring_mpmc_enq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring,
- ipsec_sa_tbl->hot.ipv4_id_data,
- IPV4_ID_RING_MASK,
- &data,
- 1);
+ ring_mpmc_u32_enq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring,
+ ipsec_sa_tbl->hot.ipv4_id_data,
+ IPV4_ID_RING_MASK,
+ &data,
+ 1);
}
for (i = 0; i < ipsec_sa_tbl->max_num_sa; i++) {
@@ -1118,17 +1119,17 @@ uint16_t _odp_ipsec_sa_alloc_ipv4_id(ipsec_sa_t *ipsec_sa)
tl->first_ipv4_id + IPV4_ID_BLOCK_SIZE)) {
/* Return used ID block to the ring */
data = tl->first_ipv4_id;
- ring_mpmc_enq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring,
- ipsec_sa_tbl->hot.ipv4_id_data,
- IPV4_ID_RING_MASK,
- &data,
- 1);
+ ring_mpmc_u32_enq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring,
+ ipsec_sa_tbl->hot.ipv4_id_data,
+ IPV4_ID_RING_MASK,
+ &data,
+ 1);
/* Get new ID block */
- ring_mpmc_deq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring,
- ipsec_sa_tbl->hot.ipv4_id_data,
- IPV4_ID_RING_MASK,
- &data,
- 1);
+ ring_mpmc_u32_deq_multi(&ipsec_sa_tbl->hot.ipv4_id_ring,
+ ipsec_sa_tbl->hot.ipv4_id_data,
+ IPV4_ID_RING_MASK,
+ &data,
+ 1);
tl->first_ipv4_id = data;
tl->next_ipv4_id = data;
}
diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c
index 212e48de0..639a74e0c 100644
--- a/platform/linux-generic/odp_packet.c
+++ b/platform/linux-generic/odp_packet.c
@@ -1,5 +1,5 @@
/* Copyright (c) 2013-2018, Linaro Limited
- * Copyright (c) 2019-2022, Nokia
+ * Copyright (c) 2019-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -21,6 +21,7 @@
#include <odp_debug_internal.h>
#include <odp_errno_define.h>
#include <odp_event_internal.h>
+#include <odp_event_validation_internal.h>
#include <odp_macros_internal.h>
#include <odp_packet_internal.h>
#include <odp_packet_io_internal.h>
@@ -69,6 +70,7 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = {
.subtype = offsetof(odp_packet_hdr_t, subtype),
.cls_mark = offsetof(odp_packet_hdr_t, cls_mark),
.ipsec_ctx = offsetof(odp_packet_hdr_t, ipsec_ctx),
+ .crypto_op = offsetof(odp_packet_hdr_t, crypto_op_result),
};
#include <odp/visibility_end.h>
@@ -695,6 +697,8 @@ void odp_packet_free(odp_packet_t pkt)
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
int num_seg = pkt_hdr->seg_count;
+ _odp_packet_validate(pkt, _ODP_EV_PACKET_FREE);
+
_ODP_ASSERT(segment_ref(pkt_hdr) > 0);
if (odp_likely(num_seg == 1))
@@ -703,12 +707,14 @@ void odp_packet_free(odp_packet_t pkt)
free_all_segments(pkt_hdr, num_seg);
}
-void odp_packet_free_multi(const odp_packet_t pkt[], int num)
+static inline void packet_free_multi_ev(const odp_packet_t pkt[], int num, _odp_ev_id_t id)
{
odp_packet_hdr_t *pkt_hdrs[num];
int i;
int num_freed = 0;
+ _odp_packet_validate_multi(pkt, num, id);
+
for (i = 0; i < num; i++) {
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt[i]);
int num_seg = pkt_hdr->seg_count;
@@ -728,9 +734,14 @@ void odp_packet_free_multi(const odp_packet_t pkt[], int num)
packet_free_multi(pkt_hdrs, num - num_freed);
}
+void odp_packet_free_multi(const odp_packet_t pkt[], int num)
+{
+ packet_free_multi_ev(pkt, num, _ODP_EV_PACKET_FREE_MULTI);
+}
+
void odp_packet_free_sp(const odp_packet_t pkt[], int num)
{
- odp_packet_free_multi(pkt, num);
+ packet_free_multi_ev(pkt, num, _ODP_EV_PACKET_FREE_SP);
}
int odp_packet_reset(odp_packet_t pkt, uint32_t len)
@@ -1587,6 +1598,9 @@ int odp_packet_is_valid(odp_packet_t pkt)
if (odp_event_type(ev) != ODP_EVENT_PACKET)
return 0;
+ if (odp_unlikely(_odp_packet_validate(pkt, _ODP_EV_PACKET_IS_VALID)))
+ return 0;
+
switch (odp_event_subtype(ev)) {
case ODP_EVENT_PACKET_BASIC:
/* Fall through */
@@ -2343,6 +2357,7 @@ odp_packet_t odp_packet_reassemble(odp_pool_t pool_hdl, odp_packet_buf_t pkt_buf
tailroom = pool->ext_param.pkt.buf_size - sizeof(odp_packet_hdr_t);
tailroom -= pool->ext_param.pkt.app_header_size;
tailroom -= odp_packet_buf_data_len(pkt_buf[num - 1]);
+ tailroom -= pool->trailer_size;
pkt_hdr->seg_count = num;
pkt_hdr->frame_len = data_len;
diff --git a/platform/linux-generic/odp_packet_flags.c b/platform/linux-generic/odp_packet_flags.c
index ef542a2cd..777da12ae 100644
--- a/platform/linux-generic/odp_packet_flags.c
+++ b/platform/linux-generic/odp_packet_flags.c
@@ -72,10 +72,12 @@ void odp_packet_has_jumbo_set(odp_packet_t pkt, int val)
void odp_packet_has_vlan_set(odp_packet_t pkt, int val)
{
setflag(pkt, input_flags.vlan, val);
+ setflag(pkt, input_flags.vlan_qinq, 0);
}
void odp_packet_has_vlan_qinq_set(odp_packet_t pkt, int val)
{
+ setflag(pkt, input_flags.vlan, val);
setflag(pkt, input_flags.vlan_qinq, val);
}
diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c
index 22b6bc916..5a6c0f460 100644
--- a/platform/linux-generic/odp_packet_io.c
+++ b/platform/linux-generic/odp_packet_io.c
@@ -1613,16 +1613,6 @@ odp_time_t odp_pktio_time(odp_pktio_t hdl, odp_time_t *global_ts)
return ts;
}
-uint64_t ODP_DEPRECATE(odp_pktin_ts_res)(odp_pktio_t hdl)
-{
- return odp_pktio_ts_res(hdl);
-}
-
-odp_time_t ODP_DEPRECATE(odp_pktin_ts_from_ns)(odp_pktio_t hdl, uint64_t ns)
-{
- return odp_pktio_ts_from_ns(hdl, ns);
-}
-
void odp_pktio_print(odp_pktio_t hdl)
{
pktio_entry_t *entry;
@@ -2166,8 +2156,7 @@ void odp_pktio_extra_stats_print(odp_pktio_t pktio)
_ODP_PRINT("\n");
}
-int odp_pktin_queue_config(odp_pktio_t pktio,
- const odp_pktin_queue_param_t *param)
+int odp_pktin_queue_config(odp_pktio_t pktio, const odp_pktin_queue_param_t *param)
{
pktio_entry_t *entry;
odp_pktin_mode_t mode;
@@ -2175,7 +2164,7 @@ int odp_pktin_queue_config(odp_pktio_t pktio,
uint32_t num_queues, i;
int rc;
odp_queue_t queue;
- odp_pktin_queue_param_t default_param;
+ odp_pktin_queue_param_t default_param, local_param;
if (param == NULL) {
odp_pktin_queue_param_init(&default_param);
@@ -2204,7 +2193,19 @@ int odp_pktin_queue_config(odp_pktio_t pktio,
return -1;
}
- num_queues = param->classifier_enable ? 1 : param->num_queues;
+ if (param->classifier_enable) {
+ num_queues = 1;
+
+ if (param->num_queues != num_queues) {
+ /* When classifier is enabled, ensure that only one input queue will be
+ * configured by driver. */
+ memcpy(&local_param, param, sizeof(odp_pktin_queue_param_t));
+ local_param.num_queues = num_queues;
+ param = &local_param;
+ }
+ } else {
+ num_queues = param->num_queues;
+ }
rc = odp_pktio_capability(pktio, &capa);
if (rc) {
diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c
index f414b0626..59c007ee2 100644
--- a/platform/linux-generic/odp_pool.c
+++ b/platform/linux-generic/odp_pool.c
@@ -1,5 +1,5 @@
/* Copyright (c) 2013-2018, Linaro Limited
- * Copyright (c) 2019-2022, Nokia
+ * Copyright (c) 2019-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -7,6 +7,7 @@
#include <odp/api/align.h>
#include <odp/api/atomic.h>
+#include <odp/api/hints.h>
#include <odp/api/pool.h>
#include <odp/api/shared_memory.h>
#include <odp/api/system_info.h>
@@ -21,6 +22,8 @@
#include <odp_packet_internal.h>
#include <odp_config_internal.h>
#include <odp_debug_internal.h>
+#include <odp_event_internal.h>
+#include <odp_event_validation_internal.h>
#include <odp_macros_internal.h>
#include <odp_ring_ptr_internal.h>
#include <odp_global_data.h>
@@ -77,6 +80,7 @@ const _odp_pool_inline_offset_t _odp_pool_inline ODP_ALIGNED_CACHE = {
.index = offsetof(pool_t, pool_idx),
.seg_len = offsetof(pool_t, seg_len),
.uarea_size = offsetof(pool_t, param_uarea_size),
+ .trailer_size = offsetof(pool_t, trailer_size),
.ext_head_offset = offsetof(pool_t, ext_head_offset),
.ext_pkt_buf_size = offsetof(pool_t, ext_param.pkt.buf_size)
};
@@ -472,6 +476,7 @@ static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t e
if (type == ODP_POOL_BUFFER || type == ODP_POOL_PACKET) {
event_hdr->base_data = data_ptr;
event_hdr->buf_end = data_ptr + pool->seg_len + pool->tailroom;
+ _odp_event_endmark_set(_odp_event_from_hdr(event_hdr));
}
if (type == ODP_POOL_BUFFER) {
@@ -697,7 +702,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
uint32_t uarea_size, headroom, tailroom;
odp_shm_t shm;
uint32_t seg_len, align, num, hdr_size, block_size;
- uint32_t max_len, cache_size;
+ uint32_t max_len, cache_size, trailer_size;
uint32_t ring_size;
odp_pool_type_t type = params->type;
uint32_t shmflags = 0;
@@ -743,6 +748,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
tailroom = 0;
seg_len = 0;
max_len = 0;
+ trailer_size = 0;
uarea_size = 0;
cache_size = 0;
@@ -752,6 +758,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
seg_len = params->buf.size;
uarea_size = params->buf.uarea_size;
cache_size = params->buf.cache_size;
+ trailer_size = _ODP_EV_ENDMARK_SIZE;
break;
case ODP_POOL_PACKET:
@@ -763,6 +770,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
num = params->pkt.num;
seg_len = CONFIG_PACKET_MAX_SEG_LEN;
max_len = _odp_pool_glb->config.pkt_max_len;
+ trailer_size = _ODP_EV_ENDMARK_SIZE;
if (params->pkt.len &&
params->pkt.len < CONFIG_PACKET_MAX_SEG_LEN)
@@ -840,7 +848,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
uint32_t adj_size;
hdr_size = _ODP_ROUNDUP_CACHE_LINE(sizeof(odp_packet_hdr_t));
- block_size = hdr_size + align + headroom + seg_len + tailroom;
+ block_size = hdr_size + align + headroom + seg_len + tailroom + trailer_size;
adj_size = block_size;
if (pool->mem_src_ops && pool->mem_src_ops->adjust_size) {
@@ -871,7 +879,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
else
hdr_size = _ODP_ROUNDUP_CACHE_LINE(sizeof(odp_event_vector_hdr_t));
- block_size = _ODP_ROUNDUP_CACHE_LINE(hdr_size + align_pad + seg_len);
+ block_size = _ODP_ROUNDUP_CACHE_LINE(hdr_size + align_pad + seg_len + trailer_size);
}
/* Allocate extra memory for skipping packet buffers which cross huge
@@ -894,6 +902,7 @@ odp_pool_t _odp_pool_create(const char *name, const odp_pool_param_t *params,
pool->align = align;
pool->headroom = headroom;
pool->seg_len = seg_len;
+ pool->trailer_size = trailer_size;
pool->max_seg_len = headroom + seg_len + tailroom;
pool->max_len = max_len;
pool->tailroom = tailroom;
@@ -1419,11 +1428,15 @@ int odp_buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int num)
void odp_buffer_free(odp_buffer_t buf)
{
+ _odp_buffer_validate(buf, _ODP_EV_BUFFER_FREE);
+
_odp_event_free_multi((_odp_event_hdr_t **)&buf, 1);
}
void odp_buffer_free_multi(const odp_buffer_t buf[], int num)
{
+ _odp_buffer_validate_multi(buf, num, _ODP_EV_BUFFER_FREE_MULTI);
+
_odp_event_free_multi((_odp_event_hdr_t **)(uintptr_t)buf, num);
}
@@ -1530,6 +1543,7 @@ void odp_pool_print(odp_pool_t pool_hdl)
_ODP_PRINT(" burst size %u\n", pool->burst_size);
_ODP_PRINT(" mem src %s\n",
pool->mem_src_ops ? pool->mem_src_ops->name : "(none)");
+ _ODP_PRINT(" event valid. %d\n", _ODP_EVENT_VALIDATION);
_ODP_PRINT("\n");
}
@@ -1727,6 +1741,9 @@ int odp_buffer_is_valid(odp_buffer_t buf)
if (odp_event_type(odp_buffer_to_event(buf)) != ODP_EVENT_BUFFER)
return 0;
+ if (odp_unlikely(_odp_buffer_validate(buf, _ODP_EV_BUFFER_IS_VALID)))
+ return 0;
+
return 1;
}
@@ -1753,7 +1770,7 @@ int odp_pool_ext_capability(odp_pool_type_t type, odp_pool_ext_capability_t *cap
capa->pkt.max_num_buf = _odp_pool_glb->config.pkt_max_num;
capa->pkt.max_buf_size = MAX_SIZE;
capa->pkt.odp_header_size = sizeof(odp_packet_hdr_t);
- capa->pkt.odp_trailer_size = 0;
+ capa->pkt.odp_trailer_size = _ODP_EV_ENDMARK_SIZE;
capa->pkt.min_mem_align = ODP_CACHE_LINE_SIZE;
capa->pkt.min_buf_align = ODP_CACHE_LINE_SIZE;
capa->pkt.min_head_align = MIN_HEAD_ALIGN;
@@ -1878,7 +1895,9 @@ odp_pool_t odp_pool_ext_create(const char *name, const odp_pool_ext_param_t *par
pool->num = num_buf;
pool->headroom = headroom;
pool->tailroom = 0;
- pool->seg_len = buf_size - head_offset - headroom - pool->tailroom;
+ pool->trailer_size = _ODP_EV_ENDMARK_SIZE;
+ pool->seg_len = buf_size - head_offset - headroom - pool->tailroom -
+ pool->trailer_size;
pool->max_seg_len = headroom + pool->seg_len + pool->tailroom;
pool->max_len = PKT_MAX_SEGS * pool->seg_len;
pool->ext_head_offset = head_offset;
diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c
index 360f907bf..83694f84f 100644
--- a/platform/linux-generic/odp_queue_basic.c
+++ b/platform/linux-generic/odp_queue_basic.c
@@ -1,43 +1,46 @@
/* Copyright (c) 2013-2018, Linaro Limited
- * Copyright (c) 2021-2022, Nokia
+ * Copyright (c) 2021-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
-#include <odp/api/queue.h>
-#include <odp_queue_basic_internal.h>
-#include <odp_queue_if.h>
-#include <odp/api/std_types.h>
#include <odp/api/align.h>
-#include <odp_pool_internal.h>
-#include <odp_init_internal.h>
-#include <odp_timer_internal.h>
-#include <odp/api/shared_memory.h>
-#include <odp/api/schedule.h>
-#include <odp_schedule_if.h>
-#include <odp_config_internal.h>
-#include <odp_packet_io_internal.h>
-#include <odp_debug_internal.h>
#include <odp/api/hints.h>
+#include <odp/api/packet_io.h>
+#include <odp/api/queue.h>
+#include <odp/api/schedule.h>
+#include <odp/api/shared_memory.h>
+#include <odp/api/std_types.h>
#include <odp/api/sync.h>
-#include <odp/api/plat/sync_inlines.h>
+#include <odp/api/ticketlock.h>
#include <odp/api/traffic_mngr.h>
-#include <odp_libconfig_internal.h>
+
#include <odp/api/plat/queue_inline_types.h>
-#include <odp_global_data.h>
-#include <odp_queue_basic_internal.h>
+#include <odp/api/plat/sync_inlines.h>
+#include <odp/api/plat/ticketlock_inlines.h>
+
+#include <odp_config_internal.h>
+#include <odp_debug_internal.h>
#include <odp_event_internal.h>
+#include <odp_global_data.h>
+#include <odp_init_internal.h>
+#include <odp_libconfig_internal.h>
#include <odp_macros_internal.h>
+#include <odp_packet_io_internal.h>
+#include <odp_pool_internal.h>
+#include <odp_queue_basic_internal.h>
+#include <odp_queue_if.h>
+#include <odp_schedule_if.h>
+#include <odp_timer_internal.h>
+
+#include <inttypes.h>
+#include <string.h>
-#include <odp/api/plat/ticketlock_inlines.h>
#define LOCK(queue_ptr) odp_ticketlock_lock(&((queue_ptr)->lock))
#define UNLOCK(queue_ptr) odp_ticketlock_unlock(&((queue_ptr)->lock))
#define LOCK_INIT(queue_ptr) odp_ticketlock_init(&((queue_ptr)->lock))
-#include <string.h>
-#include <inttypes.h>
-
#define MIN_QUEUE_SIZE 32
#define MAX_QUEUE_SIZE (1 * 1024 * 1024)
@@ -405,7 +408,7 @@ static int queue_destroy(odp_queue_t handle)
else if (queue->type == ODP_QUEUE_TYPE_SCHED)
empty = ring_st_is_empty(&queue->ring_st);
else
- empty = ring_mpmc_is_empty(&queue->ring_mpmc);
+ empty = ring_mpmc_u32_is_empty(&queue->ring_mpmc);
if (!empty) {
UNLOCK(queue);
@@ -494,7 +497,7 @@ static inline int _plain_queue_enq_multi(odp_queue_t handle,
{
queue_entry_t *queue;
int ret, num_enq;
- ring_mpmc_t *ring_mpmc;
+ ring_mpmc_u32_t *ring_mpmc;
uint32_t event_idx[num];
queue = qentry_from_handle(handle);
@@ -505,8 +508,8 @@ static inline int _plain_queue_enq_multi(odp_queue_t handle,
event_index_from_hdr(event_idx, event_hdr, num);
- num_enq = ring_mpmc_enq_multi(ring_mpmc, queue->ring_data,
- queue->ring_mask, event_idx, num);
+ num_enq = ring_mpmc_u32_enq_multi(ring_mpmc, queue->ring_data,
+ queue->ring_mask, event_idx, num);
return num_enq;
}
@@ -516,14 +519,14 @@ static inline int _plain_queue_deq_multi(odp_queue_t handle,
{
int num_deq;
queue_entry_t *queue;
- ring_mpmc_t *ring_mpmc;
+ ring_mpmc_u32_t *ring_mpmc;
uint32_t event_idx[num];
queue = qentry_from_handle(handle);
ring_mpmc = &queue->ring_mpmc;
- num_deq = ring_mpmc_deq_multi(ring_mpmc, queue->ring_data,
- queue->ring_mask, event_idx, num);
+ num_deq = ring_mpmc_u32_deq_multi(ring_mpmc, queue->ring_data,
+ queue->ring_mask, event_idx, num);
if (num_deq == 0)
return 0;
@@ -751,7 +754,7 @@ static void queue_print(odp_queue_t handle)
} else {
_ODP_PRINT(" implementation ring_mpmc\n");
_ODP_PRINT(" length %" PRIu32 "/%" PRIu32 "\n",
- ring_mpmc_length(&queue->ring_mpmc), queue->ring_mask + 1);
+ ring_mpmc_u32_len(&queue->ring_mpmc), queue->ring_mask + 1);
}
_ODP_PRINT("\n");
@@ -817,7 +820,7 @@ static void queue_print_all(void)
if (_odp_sched_id == _ODP_SCHED_ID_BASIC)
spr = _odp_sched_basic_get_spread(index);
} else {
- len = ring_mpmc_length(&queue->ring_mpmc);
+ len = ring_mpmc_u32_len(&queue->ring_mpmc);
max_len = queue->ring_mask + 1;
}
@@ -1070,7 +1073,7 @@ static int queue_init(queue_entry_t *queue, const char *name,
queue->ring_data = &_odp_queue_glb->ring_data[offset];
queue->ring_mask = queue_size - 1;
- ring_mpmc_init(&queue->ring_mpmc);
+ ring_mpmc_u32_init(&queue->ring_mpmc);
} else {
queue->enqueue = sched_queue_enq;
diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c
index 594360326..3fc9c1c17 100644
--- a/platform/linux-generic/odp_schedule_basic.c
+++ b/platform/linux-generic/odp_schedule_basic.c
@@ -238,13 +238,13 @@ typedef struct {
struct {
uint8_t burst_default[NUM_SCHED_SYNC][NUM_PRIO];
uint8_t burst_max[NUM_SCHED_SYNC][NUM_PRIO];
+ uint16_t order_stash_size;
uint8_t num_spread;
uint8_t prefer_ratio;
} config;
-
- uint8_t load_balance;
- uint16_t max_spread;
uint32_t ring_mask;
+ uint16_t max_spread;
+ uint8_t load_balance;
odp_atomic_u32_t grp_epoch;
odp_shm_t shm;
odp_ticketlock_t mask_lock[NUM_SCHED_GRPS];
@@ -437,6 +437,20 @@ static int read_config_file(sched_global_t *sched)
if (val == 0 || sched->config.num_spread == 1)
sched->load_balance = 0;
+ str = "sched_basic.order_stash_size";
+ if (!_odp_libconfig_lookup_int(str, &val)) {
+ _ODP_ERR("Config option '%s' not found.\n", str);
+ return -1;
+ }
+
+ if (val > MAX_ORDERED_STASH || val < 0) {
+ _ODP_ERR("Bad value %s = %i [min: 0, max: %u]\n", str, val, MAX_ORDERED_STASH);
+ return -1;
+ }
+
+ sched->config.order_stash_size = val;
+ _ODP_PRINT(" %s: %i\n", str, val);
+
/* Initialize default values for all queue types */
str = "sched_basic.burst_size_default";
if (read_burst_size_conf(sched->config.burst_default[ODP_SCHED_SYNC_ATOMIC], str, 1,
@@ -1204,7 +1218,7 @@ static int schedule_ord_enq_multi(odp_queue_t dst_queue, void *event_hdr[],
/* Pktout may drop packets, so the operation cannot be stashed. */
if (dst_qentry->pktout.pktio != ODP_PKTIO_INVALID ||
- odp_unlikely(stash_num >= MAX_ORDERED_STASH)) {
+ odp_unlikely(stash_num >= sched->config.order_stash_size)) {
/* If the local stash is full, wait until it is our turn and
* then release the stash and do enqueue directly. */
wait_for_order(src_queue);
diff --git a/platform/linux-generic/odp_shared_memory.c b/platform/linux-generic/odp_shared_memory.c
index a0e822a53..ef4df3a33 100644
--- a/platform/linux-generic/odp_shared_memory.c
+++ b/platform/linux-generic/odp_shared_memory.c
@@ -8,7 +8,6 @@
#include <odp_config_internal.h>
#include <odp_debug_internal.h>
#include <odp/api/debug.h>
-#include <odp/api/deprecated.h>
#include <odp/api/std_types.h>
#include <odp/api/shared_memory.h>
#include <odp/api/plat/strong_types.h>
@@ -18,14 +17,8 @@
#include <string.h>
/* Supported ODP_SHM_* flags */
-#if ODP_DEPRECATED_API
- #define DEPRECATED_SHM_FLAGS (ODP_SHM_SW_ONLY)
-#else
- #define DEPRECATED_SHM_FLAGS 0
-#endif
-
#define SUPPORTED_SHM_FLAGS (ODP_SHM_PROC | ODP_SHM_SINGLE_VA | ODP_SHM_EXPORT | \
- ODP_SHM_HP | ODP_SHM_NO_HP | DEPRECATED_SHM_FLAGS)
+ ODP_SHM_HP | ODP_SHM_NO_HP)
static inline uint32_t from_handle(odp_shm_t shm)
{
diff --git a/platform/linux-generic/odp_stash.c b/platform/linux-generic/odp_stash.c
index c7d4136ab..5ff499843 100644
--- a/platform/linux-generic/odp_stash.c
+++ b/platform/linux-generic/odp_stash.c
@@ -1,9 +1,10 @@
-/* Copyright (c) 2020-2022, Nokia
+/* Copyright (c) 2020-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
+#include <odp/api/align.h>
#include <odp/api/shared_memory.h>
#include <odp/api/stash.h>
#include <odp/api/std_types.h>
@@ -15,7 +16,10 @@
#include <odp_debug_internal.h>
#include <odp_global_data.h>
#include <odp_init_internal.h>
+#include <odp_libconfig_internal.h>
#include <odp_macros_internal.h>
+#include <odp_ring_mpmc_u32_internal.h>
+#include <odp_ring_mpmc_u64_internal.h>
#include <odp_ring_u32_internal.h>
#include <odp_ring_u64_internal.h>
@@ -26,18 +30,61 @@
ODP_STATIC_ASSERT(CONFIG_INTERNAL_STASHES < CONFIG_MAX_STASHES, "TOO_MANY_INTERNAL_STASHES");
-#define MAX_RING_SIZE (1024 * 1024)
#define MIN_RING_SIZE 64
+enum {
+ STASH_FREE = 0,
+ STASH_RESERVED,
+ STASH_ACTIVE
+};
+
+typedef struct stash_t stash_t;
+
+typedef void (*ring_u32_init_fn_t)(stash_t *stash);
+typedef int32_t (*ring_u32_enq_multi_fn_t)(stash_t *stash, const uint32_t val[], int32_t num);
+typedef int32_t (*ring_u32_enq_batch_fn_t)(stash_t *stash, const uint32_t val[], int32_t num);
+typedef int32_t (*ring_u32_deq_multi_fn_t)(stash_t *stash, uint32_t val[], int32_t num);
+typedef int32_t (*ring_u32_deq_batch_fn_t)(stash_t *stash, uint32_t val[], int32_t num);
+typedef int32_t (*ring_u32_len_fn_t)(stash_t *stash);
+
+typedef void (*ring_u64_init_fn_t)(stash_t *stash);
+typedef int32_t (*ring_u64_enq_multi_fn_t)(stash_t *stash, const uint64_t val[], int32_t num);
+typedef int32_t (*ring_u64_enq_batch_fn_t)(stash_t *stash, const uint64_t val[], int32_t num);
+typedef int32_t (*ring_u64_deq_multi_fn_t)(stash_t *stash, uint64_t val[], int32_t num);
+typedef int32_t (*ring_u64_deq_batch_fn_t)(stash_t *stash, uint64_t val[], int32_t num);
+typedef int32_t (*ring_u64_len_fn_t)(stash_t *stash);
+
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpedantic"
-typedef struct stash_t {
- char name[ODP_STASH_NAME_LEN];
- odp_shm_t shm;
- int index;
+typedef struct ODP_ALIGNED_CACHE stash_t {
+ /* Ring functions */
+ union {
+ struct {
+ ring_u32_enq_multi_fn_t enq_multi;
+ ring_u32_enq_batch_fn_t enq_batch;
+ ring_u32_deq_multi_fn_t deq_multi;
+ ring_u32_deq_batch_fn_t deq_batch;
+ ring_u32_init_fn_t init;
+ ring_u32_len_fn_t len;
+ } u32;
+
+ struct {
+ ring_u64_enq_multi_fn_t enq_multi;
+ ring_u64_enq_batch_fn_t enq_batch;
+ ring_u64_deq_multi_fn_t deq_multi;
+ ring_u64_deq_batch_fn_t deq_batch;
+ ring_u64_init_fn_t init;
+ ring_u64_len_fn_t len;
+ } u64;
+ } ring_fn;
+
uint32_t ring_mask;
+ uint32_t ring_size;
uint32_t obj_size;
+ char name[ODP_STASH_NAME_LEN];
+ int index;
+
/* Ring header followed by variable sized data (object handles) */
union {
struct ODP_ALIGNED_CACHE {
@@ -49,6 +96,16 @@ typedef struct stash_t {
ring_u64_t hdr;
uint64_t data[];
} ring_u64;
+
+ struct ODP_ALIGNED_CACHE {
+ ring_mpmc_u32_t hdr;
+ uint32_t data[];
+ } ring_mpmc_u32;
+
+ struct ODP_ALIGNED_CACHE {
+ ring_mpmc_u64_t hdr;
+ uint64_t data[];
+ } ring_mpmc_u64;
};
} stash_t;
@@ -57,23 +114,92 @@ typedef struct stash_t {
typedef struct stash_global_t {
odp_ticketlock_t lock;
odp_shm_t shm;
- uint8_t stash_reserved[CONFIG_MAX_STASHES];
+ uint32_t max_num;
+ uint32_t max_num_obj;
+ uint32_t num_internal;
+ uint8_t strict_size;
+ uint8_t stash_state[CONFIG_MAX_STASHES];
stash_t *stash[CONFIG_MAX_STASHES];
+ uint8_t data[] ODP_ALIGNED_CACHE;
} stash_global_t;
static stash_global_t *stash_global;
+static inline stash_t *stash_entry(odp_stash_t st)
+{
+ return (stash_t *)(uintptr_t)st;
+}
+
+static inline odp_stash_t stash_handle(stash_t *stash)
+{
+ return (odp_stash_t)(uintptr_t)stash;
+}
+
int _odp_stash_init_global(void)
{
odp_shm_t shm;
-
- if (odp_global_ro.disable.stash) {
+ uint32_t max_num, max_num_obj;
+ const char *str;
+ uint64_t ring_max_size, stash_max_size, stash_data_size, offset;
+ const uint32_t internal_stashes = odp_global_ro.disable.dma ? 0 : CONFIG_INTERNAL_STASHES;
+ uint8_t *stash_data;
+ uint8_t strict_size;
+ int val = 0;
+
+ if (odp_global_ro.disable.stash && odp_global_ro.disable.dma) {
_ODP_PRINT("Stash is DISABLED\n");
return 0;
}
- shm = odp_shm_reserve("_odp_stash_global", sizeof(stash_global_t),
+ _ODP_PRINT("Stash config:\n");
+
+ str = "stash.max_num";
+ if (!_odp_libconfig_lookup_int(str, &val)) {
+ _ODP_ERR("Config option '%s' not found.\n", str);
+ return -1;
+ }
+ _ODP_PRINT(" %s: %i\n", str, val);
+ max_num = val;
+
+ str = "stash.max_num_obj";
+ if (!_odp_libconfig_lookup_int(str, &val)) {
+ _ODP_ERR("Config option '%s' not found.\n", str);
+ return -1;
+ }
+ _ODP_PRINT(" %s: %i\n", str, val);
+ max_num_obj = val;
+
+ str = "stash.strict_size";
+ if (!_odp_libconfig_lookup_int(str, &val)) {
+ _ODP_ERR("Config option '%s' not found.\n", str);
+ return -1;
+ }
+ _ODP_PRINT(" %s: %i\n", str, val);
+ strict_size = !!val;
+
+ _ODP_PRINT("\n");
+
+ /* Reserve resources for implementation internal stashes */
+ if (max_num > CONFIG_MAX_STASHES - internal_stashes) {
+ _ODP_ERR("Maximum supported number of stashes: %d\n",
+ CONFIG_MAX_STASHES - internal_stashes);
+ return -1;
+ }
+ max_num += internal_stashes;
+
+ /* Must have room for minimum sized ring */
+ if (max_num_obj < MIN_RING_SIZE)
+ max_num_obj = MIN_RING_SIZE - 1;
+
+ /* Ring size must be larger than the number of items stored */
+ ring_max_size = _ODP_ROUNDUP_POWER2_U32(max_num_obj + 1);
+
+ stash_max_size = _ODP_ROUNDUP_CACHE_LINE(sizeof(stash_t) +
+ (ring_max_size * sizeof(uint64_t)));
+ stash_data_size = max_num * stash_max_size;
+
+ shm = odp_shm_reserve("_odp_stash_global", sizeof(stash_global_t) + stash_data_size,
ODP_CACHE_LINE_SIZE, 0);
stash_global = odp_shm_addr(shm);
@@ -85,8 +211,21 @@ int _odp_stash_init_global(void)
memset(stash_global, 0, sizeof(stash_global_t));
stash_global->shm = shm;
+ stash_global->max_num = max_num;
+ stash_global->max_num_obj = max_num_obj;
+ stash_global->strict_size = strict_size;
+ stash_global->num_internal = internal_stashes;
odp_ticketlock_init(&stash_global->lock);
+ /* Initialize stash pointers */
+ stash_data = stash_global->data;
+ offset = 0;
+
+ for (uint32_t i = 0; i < max_num; i++) {
+ stash_global->stash[i] = (stash_t *)(uintptr_t)(stash_data + offset);
+ offset += stash_max_size;
+ }
+
return 0;
}
@@ -108,17 +247,21 @@ int _odp_stash_term_global(void)
int odp_stash_capability(odp_stash_capability_t *capa, odp_stash_type_t type)
{
+ uint32_t max_stashes;
+
if (odp_global_ro.disable.stash) {
_ODP_ERR("Stash is disabled\n");
return -1;
}
(void)type;
+ max_stashes = stash_global->max_num - stash_global->num_internal;
+
memset(capa, 0, sizeof(odp_stash_capability_t));
- capa->max_stashes_any_type = CONFIG_MAX_STASHES - CONFIG_INTERNAL_STASHES;
- capa->max_stashes = CONFIG_MAX_STASHES - CONFIG_INTERNAL_STASHES;
- capa->max_num_obj = MAX_RING_SIZE;
+ capa->max_stashes_any_type = max_stashes;
+ capa->max_stashes = max_stashes;
+ capa->max_num_obj = stash_global->max_num_obj;
capa->max_obj_size = sizeof(uint64_t);
capa->max_get_batch = MIN_RING_SIZE;
capa->max_put_batch = MIN_RING_SIZE;
@@ -137,15 +280,14 @@ void odp_stash_param_init(odp_stash_param_t *param)
static int reserve_index(void)
{
- int i;
int index = -1;
odp_ticketlock_lock(&stash_global->lock);
- for (i = 0; i < CONFIG_MAX_STASHES; i++) {
- if (stash_global->stash_reserved[i] == 0) {
+ for (uint32_t i = 0; i < stash_global->max_num; i++) {
+ if (stash_global->stash_state[i] == STASH_FREE) {
index = i;
- stash_global->stash_reserved[i] = 1;
+ stash_global->stash_state[i] = STASH_RESERVED;
break;
}
}
@@ -159,20 +301,152 @@ static void free_index(int i)
{
odp_ticketlock_lock(&stash_global->lock);
- stash_global->stash[i] = NULL;
- stash_global->stash_reserved[i] = 0;
+ stash_global->stash_state[i] = STASH_FREE;
odp_ticketlock_unlock(&stash_global->lock);
}
+static inline void strict_ring_u32_init(stash_t *stash)
+{
+ ring_u32_init(&stash->ring_u32.hdr);
+
+ for (uint32_t i = 0; i < stash->ring_size; i++)
+ stash->ring_u32.data[i] = 0;
+}
+
+static inline void strict_ring_u64_init(stash_t *stash)
+{
+ ring_u64_init(&stash->ring_u64.hdr);
+
+ for (uint32_t i = 0; i < stash->ring_size; i++)
+ stash->ring_u64.data[i] = 0;
+}
+
+static inline int32_t strict_ring_u32_enq_multi(stash_t *stash, const uint32_t val[], int32_t num)
+{
+ /* Success always */
+ ring_u32_enq_multi(&stash->ring_u32.hdr, stash->ring_mask, (uint32_t *)(uintptr_t)val, num);
+
+ return num;
+}
+
+static inline int32_t strict_ring_u64_enq_multi(stash_t *stash, const uint64_t val[], int32_t num)
+{
+ /* Success always */
+ ring_u64_enq_multi(&stash->ring_u64.hdr, stash->ring_mask, (uint64_t *)(uintptr_t)val, num);
+
+ return num;
+}
+
+static inline int32_t strict_ring_u32_deq_multi(stash_t *stash, uint32_t val[], int32_t num)
+{
+ return ring_u32_deq_multi(&stash->ring_u32.hdr, stash->ring_mask, val, num);
+}
+
+static inline int32_t strict_ring_u64_deq_multi(stash_t *stash, uint64_t val[], int32_t num)
+{
+ return ring_u64_deq_multi(&stash->ring_u64.hdr, stash->ring_mask, val, num);
+}
+
+static inline int32_t strict_ring_u32_deq_batch(stash_t *stash, uint32_t val[], int32_t num)
+{
+ return ring_u32_deq_batch(&stash->ring_u32.hdr, stash->ring_mask, val, num);
+}
+
+static inline int32_t strict_ring_u64_deq_batch(stash_t *stash, uint64_t val[], int32_t num)
+{
+ return ring_u64_deq_batch(&stash->ring_u64.hdr, stash->ring_mask, val, num);
+}
+
+static inline int32_t strict_ring_u32_len(stash_t *stash)
+{
+ return ring_u32_len(&stash->ring_u32.hdr);
+}
+
+static inline int32_t strict_ring_u64_len(stash_t *stash)
+{
+ return ring_u64_len(&stash->ring_u64.hdr);
+}
+
+static inline void mpmc_ring_u32_init(stash_t *stash)
+{
+ ring_mpmc_u32_init(&stash->ring_mpmc_u32.hdr);
+
+ for (uint32_t i = 0; i < stash->ring_size; i++)
+ stash->ring_mpmc_u32.data[i] = 0;
+}
+
+static inline void mpmc_ring_u64_init(stash_t *stash)
+{
+ ring_mpmc_u64_init(&stash->ring_mpmc_u64.hdr);
+
+ for (uint32_t i = 0; i < stash->ring_size; i++)
+ stash->ring_mpmc_u64.data[i] = 0;
+}
+
+static inline int32_t mpmc_ring_u32_enq_multi(stash_t *stash, const uint32_t val[], int32_t num)
+{
+ return ring_mpmc_u32_enq_multi(&stash->ring_mpmc_u32.hdr, stash->ring_mpmc_u32.data,
+ stash->ring_mask, val, num);
+}
+
+static inline int32_t mpmc_ring_u64_enq_multi(stash_t *stash, const uint64_t val[], int32_t num)
+{
+ return ring_mpmc_u64_enq_multi(&stash->ring_mpmc_u64.hdr, stash->ring_mpmc_u64.data,
+ stash->ring_mask, val, num);
+}
+
+static inline int32_t mpmc_ring_u32_enq_batch(stash_t *stash, const uint32_t val[], int32_t num)
+{
+ return ring_mpmc_u32_enq_batch(&stash->ring_mpmc_u32.hdr, stash->ring_mpmc_u32.data,
+ stash->ring_mask, val, num);
+}
+
+static inline int32_t mpmc_ring_u64_enq_batch(stash_t *stash, const uint64_t val[], int32_t num)
+{
+ return ring_mpmc_u64_enq_batch(&stash->ring_mpmc_u64.hdr, stash->ring_mpmc_u64.data,
+ stash->ring_mask, val, num);
+}
+
+static inline int32_t mpmc_ring_u32_deq_multi(stash_t *stash, uint32_t val[], int32_t num)
+{
+ return ring_mpmc_u32_deq_multi(&stash->ring_mpmc_u32.hdr, stash->ring_mpmc_u32.data,
+ stash->ring_mask, val, num);
+}
+
+static inline int32_t mpmc_ring_u64_deq_multi(stash_t *stash, uint64_t val[], int32_t num)
+{
+ return ring_mpmc_u64_deq_multi(&stash->ring_mpmc_u64.hdr, stash->ring_mpmc_u64.data,
+ stash->ring_mask, val, num);
+}
+
+static inline int32_t mpmc_ring_u32_deq_batch(stash_t *stash, uint32_t val[], int32_t num)
+{
+ return ring_mpmc_u32_deq_batch(&stash->ring_mpmc_u32.hdr, stash->ring_mpmc_u32.data,
+ stash->ring_mask, val, num);
+}
+
+static inline int32_t mpmc_ring_u64_deq_batch(stash_t *stash, uint64_t val[], int32_t num)
+{
+ return ring_mpmc_u64_deq_batch(&stash->ring_mpmc_u64.hdr, stash->ring_mpmc_u64.data,
+ stash->ring_mask, val, num);
+}
+
+static inline int32_t mpmc_ring_u32_len(stash_t *stash)
+{
+ return ring_mpmc_u32_len(&stash->ring_mpmc_u32.hdr);
+}
+
+static inline int32_t mpmc_ring_u64_len(stash_t *stash)
+{
+ return ring_mpmc_u64_len(&stash->ring_mpmc_u64.hdr);
+}
+
odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param)
{
- odp_shm_t shm;
stash_t *stash;
- uint64_t i, ring_size, shm_size;
+ uint64_t ring_size;
int ring_u64, index;
- char shm_name[ODP_STASH_NAME_LEN + 8];
- uint32_t shm_flags = 0;
if (odp_global_ro.disable.stash) {
_ODP_ERR("Stash is disabled\n");
@@ -184,7 +458,7 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param)
return ODP_STASH_INVALID;
}
- if (param->num_obj > MAX_RING_SIZE) {
+ if (param->num_obj > stash_global->max_num_obj) {
_ODP_ERR("Too many objects.\n");
return ODP_STASH_INVALID;
}
@@ -213,75 +487,71 @@ odp_stash_t odp_stash_create(const char *name, const odp_stash_param_t *param)
else
ring_size = _ODP_ROUNDUP_POWER2_U32(ring_size + 1);
- memset(shm_name, 0, sizeof(shm_name));
- snprintf(shm_name, sizeof(shm_name) - 1, "_stash_%s", name);
-
- if (ring_u64)
- shm_size = sizeof(stash_t) + (ring_size * sizeof(uint64_t));
- else
- shm_size = sizeof(stash_t) + (ring_size * sizeof(uint32_t));
-
- if (odp_global_ro.shm_single_va)
- shm_flags |= ODP_SHM_SINGLE_VA;
-
- shm = odp_shm_reserve(shm_name, shm_size, ODP_CACHE_LINE_SIZE, shm_flags);
-
- if (shm == ODP_SHM_INVALID) {
- _ODP_ERR("SHM reserve failed.\n");
- free_index(index);
- return ODP_STASH_INVALID;
- }
-
- stash = odp_shm_addr(shm);
+ stash = stash_global->stash[index];
memset(stash, 0, sizeof(stash_t));
- if (ring_u64) {
- ring_u64_init(&stash->ring_u64.hdr);
-
- for (i = 0; i < ring_size; i++)
- stash->ring_u64.data[i] = 0;
+ /* Set ring function pointers */
+ if (stash_global->strict_size) {
+ if (ring_u64) {
+ stash->ring_fn.u64.init = strict_ring_u64_init;
+ stash->ring_fn.u64.enq_multi = strict_ring_u64_enq_multi;
+ stash->ring_fn.u64.enq_batch = strict_ring_u64_enq_multi;
+ stash->ring_fn.u64.deq_multi = strict_ring_u64_deq_multi;
+ stash->ring_fn.u64.deq_batch = strict_ring_u64_deq_batch;
+ stash->ring_fn.u64.len = strict_ring_u64_len;
+ } else {
+ stash->ring_fn.u32.init = strict_ring_u32_init;
+ stash->ring_fn.u32.enq_multi = strict_ring_u32_enq_multi;
+ stash->ring_fn.u32.enq_batch = strict_ring_u32_enq_multi;
+ stash->ring_fn.u32.deq_multi = strict_ring_u32_deq_multi;
+ stash->ring_fn.u32.deq_batch = strict_ring_u32_deq_batch;
+ stash->ring_fn.u32.len = strict_ring_u32_len;
+ }
} else {
- ring_u32_init(&stash->ring_u32.hdr);
-
- for (i = 0; i < ring_size; i++)
- stash->ring_u32.data[i] = 0;
+ if (ring_u64) {
+ stash->ring_fn.u64.init = mpmc_ring_u64_init;
+ stash->ring_fn.u64.enq_multi = mpmc_ring_u64_enq_multi;
+ stash->ring_fn.u64.enq_batch = mpmc_ring_u64_enq_batch;
+ stash->ring_fn.u64.deq_multi = mpmc_ring_u64_deq_multi;
+ stash->ring_fn.u64.deq_batch = mpmc_ring_u64_deq_batch;
+ stash->ring_fn.u64.len = mpmc_ring_u64_len;
+ } else {
+ stash->ring_fn.u32.init = mpmc_ring_u32_init;
+ stash->ring_fn.u32.enq_multi = mpmc_ring_u32_enq_multi;
+ stash->ring_fn.u32.enq_batch = mpmc_ring_u32_enq_batch;
+ stash->ring_fn.u32.deq_multi = mpmc_ring_u32_deq_multi;
+ stash->ring_fn.u32.deq_batch = mpmc_ring_u32_deq_batch;
+ stash->ring_fn.u32.len = mpmc_ring_u32_len;
+ }
}
if (name)
strcpy(stash->name, name);
stash->index = index;
- stash->shm = shm;
stash->obj_size = param->obj_size;
stash->ring_mask = ring_size - 1;
+ stash->ring_size = ring_size;
+
+ if (ring_u64)
+ stash->ring_fn.u64.init(stash);
+ else
+ stash->ring_fn.u32.init(stash);
/* This makes stash visible to lookups */
odp_ticketlock_lock(&stash_global->lock);
- stash_global->stash[index] = stash;
+ stash_global->stash_state[index] = STASH_ACTIVE;
odp_ticketlock_unlock(&stash_global->lock);
- return (odp_stash_t)stash;
+ return stash_handle(stash);
}
int odp_stash_destroy(odp_stash_t st)
{
- stash_t *stash;
- int index;
- odp_shm_t shm;
-
if (st == ODP_STASH_INVALID)
return -1;
- stash = (stash_t *)(uintptr_t)st;
- index = stash->index;
- shm = stash->shm;
-
- free_index(index);
-
- if (odp_shm_free(shm)) {
- _ODP_ERR("SHM free failed.\n");
- return -1;
- }
+ free_index(stash_entry(st)->index);
return 0;
}
@@ -293,7 +563,6 @@ uint64_t odp_stash_to_u64(odp_stash_t st)
odp_stash_t odp_stash_lookup(const char *name)
{
- int i;
stash_t *stash;
if (name == NULL)
@@ -301,12 +570,13 @@ odp_stash_t odp_stash_lookup(const char *name)
odp_ticketlock_lock(&stash_global->lock);
- for (i = 0; i < CONFIG_MAX_STASHES; i++) {
+ for (uint32_t i = 0; i < stash_global->max_num; i++) {
stash = stash_global->stash[i];
- if (stash && strcmp(stash->name, name) == 0) {
+ if (stash_global->stash_state[i] == STASH_ACTIVE &&
+ strcmp(stash->name, name) == 0) {
odp_ticketlock_unlock(&stash_global->lock);
- return (odp_stash_t)stash;
+ return stash_handle(stash);
}
}
@@ -315,57 +585,51 @@ odp_stash_t odp_stash_lookup(const char *name)
return ODP_STASH_INVALID;
}
-static inline int32_t stash_put(odp_stash_t st, const void *obj, int32_t num)
+static inline int32_t stash_put(odp_stash_t st, const void *obj, int32_t num, odp_bool_t is_batch)
{
- stash_t *stash;
+ int32_t (*ring_u32_enq)(stash_t *stash, const uint32_t val[], int32_t num);
+ int32_t (*ring_u64_enq)(stash_t *stash, const uint64_t val[], int32_t num);
+ stash_t *stash = stash_entry(st);
uint32_t obj_size;
int32_t i;
- stash = (stash_t *)(uintptr_t)st;
-
if (odp_unlikely(st == ODP_STASH_INVALID))
return -1;
- obj_size = stash->obj_size;
-
- if (obj_size == sizeof(uint64_t)) {
- ring_u64_t *ring_u64 = &stash->ring_u64.hdr;
-
- ring_u64_enq_multi(ring_u64, stash->ring_mask,
- (uint64_t *)(uintptr_t)obj, num);
- return num;
+ if (is_batch) {
+ ring_u32_enq = stash->ring_fn.u32.enq_batch;
+ ring_u64_enq = stash->ring_fn.u64.enq_batch;
+ } else {
+ ring_u32_enq = stash->ring_fn.u32.enq_multi;
+ ring_u64_enq = stash->ring_fn.u64.enq_multi;
}
- if (obj_size == sizeof(uint32_t)) {
- ring_u32_t *ring_u32 = &stash->ring_u32.hdr;
+ obj_size = stash->obj_size;
- ring_u32_enq_multi(ring_u32, stash->ring_mask,
- (uint32_t *)(uintptr_t)obj, num);
- return num;
- }
+ if (obj_size == sizeof(uint64_t))
+ return ring_u64_enq(stash, (uint64_t *)(uintptr_t)obj, num);
+
+ if (obj_size == sizeof(uint32_t))
+ return ring_u32_enq(stash, (uint32_t *)(uintptr_t)obj, num);
if (obj_size == sizeof(uint16_t)) {
const uint16_t *u16_ptr = obj;
- ring_u32_t *ring_u32 = &stash->ring_u32.hdr;
uint32_t u32[num];
for (i = 0; i < num; i++)
u32[i] = u16_ptr[i];
- ring_u32_enq_multi(ring_u32, stash->ring_mask, u32, num);
- return num;
+ return ring_u32_enq(stash, u32, num);
}
if (obj_size == sizeof(uint8_t)) {
const uint8_t *u8_ptr = obj;
- ring_u32_t *ring_u32 = &stash->ring_u32.hdr;
uint32_t u32[num];
for (i = 0; i < num; i++)
u32[i] = u8_ptr[i];
- ring_u32_enq_multi(ring_u32, stash->ring_mask, u32, num);
- return num;
+ return ring_u32_enq(stash, u32, num);
}
return -1;
@@ -373,73 +637,66 @@ static inline int32_t stash_put(odp_stash_t st, const void *obj, int32_t num)
int32_t odp_stash_put(odp_stash_t st, const void *obj, int32_t num)
{
- return stash_put(st, obj, num);
+ return stash_put(st, obj, num, false);
}
int32_t odp_stash_put_batch(odp_stash_t st, const void *obj, int32_t num)
{
- /* Returns always 'num', or -1 on failure. */
- return stash_put(st, obj, num);
+ return stash_put(st, obj, num, true);
}
-static inline int32_t stash_put_u32(odp_stash_t st, const uint32_t val[],
- int32_t num)
+int32_t odp_stash_put_u32(odp_stash_t st, const uint32_t val[], int32_t num)
{
- stash_t *stash = (stash_t *)(uintptr_t)st;
+ stash_t *stash = stash_entry(st);
if (odp_unlikely(st == ODP_STASH_INVALID))
return -1;
_ODP_ASSERT(stash->obj_size == sizeof(uint32_t));
- ring_u32_enq_multi(&stash->ring_u32.hdr, stash->ring_mask,
- (uint32_t *)(uintptr_t)val, num);
- return num;
+ return stash->ring_fn.u32.enq_multi(stash, val, num);
}
-int32_t odp_stash_put_u32(odp_stash_t st, const uint32_t val[], int32_t num)
+int32_t odp_stash_put_u32_batch(odp_stash_t st, const uint32_t val[], int32_t num)
{
- return stash_put_u32(st, val, num);
-}
+ stash_t *stash = stash_entry(st);
-int32_t odp_stash_put_u32_batch(odp_stash_t st, const uint32_t val[],
- int32_t num)
-{
- /* Returns always 'num', or -1 on failure. */
- return stash_put_u32(st, val, num);
+ if (odp_unlikely(st == ODP_STASH_INVALID))
+ return -1;
+
+ _ODP_ASSERT(stash->obj_size == sizeof(uint32_t));
+
+ return stash->ring_fn.u32.enq_batch(stash, val, num);
}
-static inline int32_t stash_put_u64(odp_stash_t st, const uint64_t val[],
- int32_t num)
+int32_t odp_stash_put_u64(odp_stash_t st, const uint64_t val[], int32_t num)
{
- stash_t *stash = (stash_t *)(uintptr_t)st;
+ stash_t *stash = stash_entry(st);
if (odp_unlikely(st == ODP_STASH_INVALID))
return -1;
_ODP_ASSERT(stash->obj_size == sizeof(uint64_t));
- ring_u64_enq_multi(&stash->ring_u64.hdr, stash->ring_mask,
- (uint64_t *)(uintptr_t)val, num);
- return num;
-}
-
-int32_t odp_stash_put_u64(odp_stash_t st, const uint64_t val[], int32_t num)
-{
- return stash_put_u64(st, val, num);
+ return stash->ring_fn.u64.enq_multi(stash, (uint64_t *)(uintptr_t)val, num);
}
int32_t odp_stash_put_u64_batch(odp_stash_t st, const uint64_t val[],
int32_t num)
{
- /* Returns always 'num', or -1 on failure. */
- return stash_put_u64(st, val, num);
+ stash_t *stash = stash_entry(st);
+
+ if (odp_unlikely(st == ODP_STASH_INVALID))
+ return -1;
+
+ _ODP_ASSERT(stash->obj_size == sizeof(uint64_t));
+
+ return stash->ring_fn.u64.enq_batch(stash, (uint64_t *)(uintptr_t)val, num);
}
-static inline int32_t stash_put_ptr(odp_stash_t st, const uintptr_t ptr[],
- int32_t num)
+int32_t odp_stash_put_ptr(odp_stash_t st, const uintptr_t ptr[], int32_t num)
{
- stash_t *stash = (stash_t *)(uintptr_t)st;
+ stash_t *stash = stash_entry(st);
if (odp_unlikely(st == ODP_STASH_INVALID))
return -1;
@@ -447,69 +704,65 @@ static inline int32_t stash_put_ptr(odp_stash_t st, const uintptr_t ptr[],
_ODP_ASSERT(stash->obj_size == sizeof(uintptr_t));
if (sizeof(uintptr_t) == sizeof(uint32_t))
- ring_u32_enq_multi(&stash->ring_u32.hdr, stash->ring_mask,
- (uint32_t *)(uintptr_t)ptr, num);
- else if (sizeof(uintptr_t) == sizeof(uint64_t))
- ring_u64_enq_multi(&stash->ring_u64.hdr, stash->ring_mask,
- (uint64_t *)(uintptr_t)ptr, num);
- else
- return -1;
+ return stash->ring_fn.u32.enq_multi(stash, (uint32_t *)(uintptr_t)ptr, num);
- return num;
-}
+ if (sizeof(uintptr_t) == sizeof(uint64_t))
+ return stash->ring_fn.u64.enq_multi(stash, (uint64_t *)(uintptr_t)ptr, num);
-int32_t odp_stash_put_ptr(odp_stash_t st, const uintptr_t ptr[], int32_t num)
-{
- return stash_put_ptr(st, ptr, num);
+ return -1;
}
int32_t odp_stash_put_ptr_batch(odp_stash_t st, const uintptr_t ptr[],
int32_t num)
{
- /* Returns always 'num', or -1 on failure. */
- return stash_put_ptr(st, ptr, num);
+ stash_t *stash = stash_entry(st);
+
+ if (odp_unlikely(st == ODP_STASH_INVALID))
+ return -1;
+
+ _ODP_ASSERT(stash->obj_size == sizeof(uintptr_t));
+
+ if (sizeof(uintptr_t) == sizeof(uint32_t))
+ return stash->ring_fn.u32.enq_batch(stash, (uint32_t *)(uintptr_t)ptr, num);
+
+ if (sizeof(uintptr_t) == sizeof(uint64_t))
+ return stash->ring_fn.u64.enq_batch(stash, (uint64_t *)(uintptr_t)ptr, num);
+
+ return -1;
}
-static inline int32_t stash_get(odp_stash_t st, void *obj, int32_t num, odp_bool_t batch)
+static inline int32_t stash_get(odp_stash_t st, void *obj, int32_t num, odp_bool_t is_batch)
{
- stash_t *stash;
+ int32_t (*ring_u32_deq)(stash_t *stash, uint32_t val[], int32_t num);
+ int32_t (*ring_u64_deq)(stash_t *stash, uint64_t val[], int32_t num);
+ stash_t *stash = stash_entry(st);
uint32_t obj_size;
uint32_t i, num_deq;
- stash = (stash_t *)(uintptr_t)st;
-
if (odp_unlikely(st == ODP_STASH_INVALID))
return -1;
- obj_size = stash->obj_size;
-
- if (obj_size == sizeof(uint64_t)) {
- ring_u64_t *ring_u64 = &stash->ring_u64.hdr;
-
- if (batch)
- return ring_u64_deq_batch(ring_u64, stash->ring_mask, obj, num);
- else
- return ring_u64_deq_multi(ring_u64, stash->ring_mask, obj, num);
+ if (is_batch) {
+ ring_u32_deq = stash->ring_fn.u32.deq_batch;
+ ring_u64_deq = stash->ring_fn.u64.deq_batch;
+ } else {
+ ring_u32_deq = stash->ring_fn.u32.deq_multi;
+ ring_u64_deq = stash->ring_fn.u64.deq_multi;
}
- if (obj_size == sizeof(uint32_t)) {
- ring_u32_t *ring_u32 = &stash->ring_u32.hdr;
+ obj_size = stash->obj_size;
+
+ if (obj_size == sizeof(uint64_t))
+ return ring_u64_deq(stash, obj, num);
- if (batch)
- return ring_u32_deq_batch(ring_u32, stash->ring_mask, obj, num);
- else
- return ring_u32_deq_multi(ring_u32, stash->ring_mask, obj, num);
- }
+ if (obj_size == sizeof(uint32_t))
+ return ring_u32_deq(stash, obj, num);
if (obj_size == sizeof(uint16_t)) {
uint16_t *u16_ptr = obj;
- ring_u32_t *ring_u32 = &stash->ring_u32.hdr;
uint32_t u32[num];
- if (batch)
- num_deq = ring_u32_deq_batch(ring_u32, stash->ring_mask, u32, num);
- else
- num_deq = ring_u32_deq_multi(ring_u32, stash->ring_mask, u32, num);
+ num_deq = ring_u32_deq(stash, u32, num);
for (i = 0; i < num_deq; i++)
u16_ptr[i] = u32[i];
@@ -519,13 +772,9 @@ static inline int32_t stash_get(odp_stash_t st, void *obj, int32_t num, odp_bool
if (obj_size == sizeof(uint8_t)) {
uint8_t *u8_ptr = obj;
- ring_u32_t *ring_u32 = &stash->ring_u32.hdr;
uint32_t u32[num];
- if (batch)
- num_deq = ring_u32_deq_batch(ring_u32, stash->ring_mask, u32, num);
- else
- num_deq = ring_u32_deq_multi(ring_u32, stash->ring_mask, u32, num);
+ num_deq = ring_u32_deq(stash, u32, num);
for (i = 0; i < num_deq; i++)
u8_ptr[i] = u32[i];
@@ -538,67 +787,65 @@ static inline int32_t stash_get(odp_stash_t st, void *obj, int32_t num, odp_bool
int32_t odp_stash_get(odp_stash_t st, void *obj, int32_t num)
{
- return stash_get(st, obj, num, 0);
+ return stash_get(st, obj, num, false);
}
int32_t odp_stash_get_batch(odp_stash_t st, void *obj, int32_t num)
{
- return stash_get(st, obj, num, 1);
+ return stash_get(st, obj, num, true);
}
int32_t odp_stash_get_u32(odp_stash_t st, uint32_t val[], int32_t num)
{
- stash_t *stash = (stash_t *)(uintptr_t)st;
+ stash_t *stash = stash_entry(st);
if (odp_unlikely(st == ODP_STASH_INVALID))
return -1;
_ODP_ASSERT(stash->obj_size == sizeof(uint32_t));
- return ring_u32_deq_multi(&stash->ring_u32.hdr, stash->ring_mask, val,
- num);
+ return stash->ring_fn.u32.deq_multi(stash, val, num);
}
int32_t odp_stash_get_u32_batch(odp_stash_t st, uint32_t val[], int32_t num)
{
- stash_t *stash = (stash_t *)(uintptr_t)st;
+ stash_t *stash = stash_entry(st);
if (odp_unlikely(st == ODP_STASH_INVALID))
return -1;
_ODP_ASSERT(stash->obj_size == sizeof(uint32_t));
- return ring_u32_deq_batch(&stash->ring_u32.hdr, stash->ring_mask, val, num);
+ return stash->ring_fn.u32.deq_batch(stash, val, num);
}
int32_t odp_stash_get_u64(odp_stash_t st, uint64_t val[], int32_t num)
{
- stash_t *stash = (stash_t *)(uintptr_t)st;
+ stash_t *stash = stash_entry(st);
if (odp_unlikely(st == ODP_STASH_INVALID))
return -1;
_ODP_ASSERT(stash->obj_size == sizeof(uint64_t));
- return ring_u64_deq_multi(&stash->ring_u64.hdr, stash->ring_mask, val,
- num);
+ return stash->ring_fn.u64.deq_multi(stash, val, num);
}
int32_t odp_stash_get_u64_batch(odp_stash_t st, uint64_t val[], int32_t num)
{
- stash_t *stash = (stash_t *)(uintptr_t)st;
+ stash_t *stash = stash_entry(st);
if (odp_unlikely(st == ODP_STASH_INVALID))
return -1;
_ODP_ASSERT(stash->obj_size == sizeof(uint64_t));
- return ring_u64_deq_batch(&stash->ring_u64.hdr, stash->ring_mask, val, num);
+ return stash->ring_fn.u64.deq_batch(stash, val, num);
}
int32_t odp_stash_get_ptr(odp_stash_t st, uintptr_t ptr[], int32_t num)
{
- stash_t *stash = (stash_t *)(uintptr_t)st;
+ stash_t *stash = stash_entry(st);
if (odp_unlikely(st == ODP_STASH_INVALID))
return -1;
@@ -606,19 +853,17 @@ int32_t odp_stash_get_ptr(odp_stash_t st, uintptr_t ptr[], int32_t num)
_ODP_ASSERT(stash->obj_size == sizeof(uintptr_t));
if (sizeof(uintptr_t) == sizeof(uint32_t))
- return ring_u32_deq_multi(&stash->ring_u32.hdr,
- stash->ring_mask,
- (uint32_t *)(uintptr_t)ptr, num);
- else if (sizeof(uintptr_t) == sizeof(uint64_t))
- return ring_u64_deq_multi(&stash->ring_u64.hdr,
- stash->ring_mask,
- (uint64_t *)(uintptr_t)ptr, num);
+ return stash->ring_fn.u32.deq_multi(stash, (uint32_t *)(uintptr_t)ptr, num);
+
+ if (sizeof(uintptr_t) == sizeof(uint64_t))
+ return stash->ring_fn.u64.deq_multi(stash, (uint64_t *)(uintptr_t)ptr, num);
+
return -1;
}
int32_t odp_stash_get_ptr_batch(odp_stash_t st, uintptr_t ptr[], int32_t num)
{
- stash_t *stash = (stash_t *)(uintptr_t)st;
+ stash_t *stash = stash_entry(st);
if (odp_unlikely(st == ODP_STASH_INVALID))
return -1;
@@ -626,11 +871,11 @@ int32_t odp_stash_get_ptr_batch(odp_stash_t st, uintptr_t ptr[], int32_t num)
_ODP_ASSERT(stash->obj_size == sizeof(uintptr_t));
if (sizeof(uintptr_t) == sizeof(uint32_t))
- return ring_u32_deq_batch(&stash->ring_u32.hdr, stash->ring_mask,
- (uint32_t *)(uintptr_t)ptr, num);
- else if (sizeof(uintptr_t) == sizeof(uint64_t))
- return ring_u64_deq_batch(&stash->ring_u64.hdr, stash->ring_mask,
- (uint64_t *)(uintptr_t)ptr, num);
+ return stash->ring_fn.u32.deq_batch(stash, (uint32_t *)(uintptr_t)ptr, num);
+
+ if (sizeof(uintptr_t) == sizeof(uint64_t))
+ return stash->ring_fn.u64.deq_batch(stash, (uint64_t *)(uintptr_t)ptr, num);
+
return -1;
}
@@ -644,23 +889,17 @@ int odp_stash_flush_cache(odp_stash_t st)
static uint32_t stash_obj_count(stash_t *stash)
{
- ring_u32_t *ring_u32;
uint32_t obj_size = stash->obj_size;
- if (obj_size == sizeof(uint64_t)) {
- ring_u64_t *ring_u64 = &stash->ring_u64.hdr;
-
- return ring_u64_len(ring_u64);
- }
-
- ring_u32 = &stash->ring_u32.hdr;
+ if (obj_size == sizeof(uint64_t))
+ return stash->ring_fn.u64.len(stash);
- return ring_u32_len(ring_u32);
+ return stash->ring_fn.u32.len(stash);
}
void odp_stash_print(odp_stash_t st)
{
- stash_t *stash = (stash_t *)(uintptr_t)st;
+ stash_t *stash = stash_entry(st);
if (st == ODP_STASH_INVALID) {
_ODP_ERR("Bad stash handle\n");
@@ -674,13 +913,13 @@ void odp_stash_print(odp_stash_t st)
_ODP_PRINT(" index %i\n", stash->index);
_ODP_PRINT(" obj size %u\n", stash->obj_size);
_ODP_PRINT(" obj count %u\n", stash_obj_count(stash));
- _ODP_PRINT(" ring size %u\n", stash->ring_mask + 1);
+ _ODP_PRINT(" ring size %u\n", stash->ring_size);
_ODP_PRINT("\n");
}
int odp_stash_stats(odp_stash_t st, odp_stash_stats_t *stats)
{
- stash_t *stash = (stash_t *)(uintptr_t)st;
+ stash_t *stash = stash_entry(st);
if (st == ODP_STASH_INVALID) {
_ODP_ERR("Bad stash handle\n");
diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c
index 05660eb7a..8597c3f4e 100644
--- a/platform/linux-generic/odp_timer.c
+++ b/platform/linux-generic/odp_timer.c
@@ -17,6 +17,7 @@
#include <odp/api/atomic.h>
#include <odp/api/cpu.h>
#include <odp/api/debug.h>
+#include <odp/api/deprecated.h>
#include <odp/api/event.h>
#include <odp/api/hints.h>
#include <odp/api/pool.h>
@@ -1494,9 +1495,7 @@ odp_event_t odp_timer_free(odp_timer_t hdl)
return timer_free(tp, idx);
}
-int odp_timer_set_abs(odp_timer_t hdl,
- uint64_t abs_tck,
- odp_event_t *tmo_ev)
+int ODP_DEPRECATE(odp_timer_set_abs)(odp_timer_t hdl, uint64_t abs_tck, odp_event_t *tmo_ev)
{
timer_pool_t *tp = handle_to_tp(hdl);
uint64_t cur_tick = current_nsec(tp);
@@ -1512,9 +1511,7 @@ int odp_timer_set_abs(odp_timer_t hdl,
return ODP_TIMER_FAIL;
}
-int odp_timer_set_rel(odp_timer_t hdl,
- uint64_t rel_tck,
- odp_event_t *tmo_ev)
+int ODP_DEPRECATE(odp_timer_set_rel)(odp_timer_t hdl, uint64_t rel_tck, odp_event_t *tmo_ev)
{
timer_pool_t *tp = handle_to_tp(hdl);
uint64_t cur_tick = current_nsec(tp);
diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c
index 284a6f7f5..c8d8bc5c0 100644
--- a/platform/linux-generic/odp_traffic_mngr.c
+++ b/platform/linux-generic/odp_traffic_mngr.c
@@ -2665,12 +2665,6 @@ static int tm_capabilities(odp_tm_capabilities_t capabilities[],
return 1;
}
-int ODP_DEPRECATE(odp_tm_capabilities)(odp_tm_capabilities_t capabilities[],
- uint32_t capabilities_size)
-{
- return tm_capabilities(capabilities, capabilities_size);
-}
-
int odp_tm_egress_capabilities(odp_tm_capabilities_t *capabilities,
const odp_tm_egress_t *egress)
{
diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c
index 7e9db90ce..fa40d1bde 100644
--- a/platform/linux-generic/pktio/dpdk.c
+++ b/platform/linux-generic/pktio/dpdk.c
@@ -49,6 +49,7 @@
#include <rte_tcp.h>
#include <rte_udp.h>
#include <rte_version.h>
+#include <rte_vfio.h>
/* NUMA is not supported on all platforms */
#ifdef _ODP_HAVE_NUMA_LIBRARY
@@ -438,6 +439,15 @@ static struct rte_mempool *mbuf_pool_create(const char *name,
goto fail;
}
+ /* Map pages for DMA access to enable VFIO usage */
+ for (uint64_t i = 0; i < pool_entry->shm_size; i += page_size) {
+ addr = pool_entry->base_addr + i;
+
+ rte_vfio_container_dma_map(RTE_VFIO_DEFAULT_CONTAINER_FD,
+ (uint64_t)(uintptr_t)addr,
+ rte_mem_virt2iova(addr), page_size);
+ }
+
rte_mempool_obj_iter(mp, pktmbuf_init, NULL);
return mp;
diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c
index b30535f22..a69daf56b 100644
--- a/platform/linux-generic/pktio/loop.c
+++ b/platform/linux-generic/pktio/loop.c
@@ -1,5 +1,5 @@
/* Copyright (c) 2013-2018, Linaro Limited
- * Copyright (c) 2013-2022, Nokia Solutions and Networks
+ * Copyright (c) 2013-2023, Nokia Solutions and Networks
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -7,11 +7,11 @@
#include <odp/api/debug.h>
#include <odp/api/event.h>
+#include <odp/api/hash.h>
#include <odp/api/hints.h>
#include <odp/api/packet.h>
#include <odp/api/packet_io.h>
#include <odp/api/queue.h>
-#include <odp/api/ticketlock.h>
#include <odp/api/time.h>
#include <odp/api/plat/byteorder_inlines.h>
@@ -39,6 +39,9 @@
#include <stdint.h>
#include <stdlib.h>
+#define MAX_QUEUES (ODP_PKTIN_MAX_QUEUES > ODP_PKTOUT_MAX_QUEUES ? \
+ ODP_PKTIN_MAX_QUEUES : ODP_PKTOUT_MAX_QUEUES)
+
#define MAX_LOOP 16
#define LOOP_MTU_MIN 68
@@ -47,12 +50,40 @@
#define LOOP_MAX_QUEUE_SIZE 1024
typedef struct {
- odp_queue_t loopq; /**< loopback queue for "loop" device */
- uint32_t pktin_queue_size; /**< input queue size */
- uint32_t pktout_queue_size; /**< output queue size */
- uint16_t mtu; /**< link MTU */
- uint8_t idx; /**< index of "loop" device */
- uint8_t queue_create; /**< create or re-create queue during start */
+ odp_atomic_u64_t in_octets;
+ odp_atomic_u64_t in_packets;
+ odp_atomic_u64_t in_discards;
+ odp_atomic_u64_t in_errors;
+ odp_atomic_u64_t out_octets;
+ odp_atomic_u64_t out_packets;
+} stats_t;
+
+typedef struct ODP_ALIGNED_CACHE {
+ /* queue handle as the "wire" */
+ odp_queue_t queue;
+ /* queue specific statistics */
+ stats_t stats;
+ /* config input queue size */
+ uint32_t in_size;
+ /* config output queue size */
+ uint32_t out_size;
+} loop_queue_t;
+
+typedef struct {
+ /* loopback entries for "loop" device */
+ loop_queue_t loopqs[MAX_QUEUES];
+ /* hash config */
+ odp_pktin_hash_proto_t hash;
+ /* config queue count */
+ uint32_t num_conf_qs;
+ /* actual number queues */
+ uint32_t num_qs;
+ /* link MTU */
+ uint16_t mtu;
+ /* index of "loop" device */
+ uint8_t idx;
+ /* create or re-create queue during start */
+ uint8_t queue_create;
} pkt_loop_t;
ODP_STATIC_ASSERT(PKTIO_PRIVATE_SIZE >= sizeof(pkt_loop_t),
@@ -66,7 +97,6 @@ static inline pkt_loop_t *pkt_priv(pktio_entry_t *pktio_entry)
/* MAC address for the "loop" interface */
static const uint8_t pktio_loop_mac[] = {0x02, 0xe9, 0x34, 0x80, 0x73, 0x01};
-static int loopback_stats_reset(pktio_entry_t *pktio_entry);
static int loopback_init_capability(pktio_entry_t *pktio_entry);
static int loopback_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry,
@@ -88,14 +118,20 @@ static int loopback_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry,
}
memset(pkt_loop, 0, sizeof(pkt_loop_t));
- pkt_loop->idx = idx;
pkt_loop->mtu = LOOP_MTU_MAX;
- pkt_loop->loopq = ODP_QUEUE_INVALID;
+ pkt_loop->idx = idx;
pkt_loop->queue_create = 1;
-
- loopback_stats_reset(pktio_entry);
loopback_init_capability(pktio_entry);
+ for (uint32_t i = 0; i < MAX_QUEUES; i++) {
+ odp_atomic_init_u64(&pkt_loop->loopqs[i].stats.in_octets, 0);
+ odp_atomic_init_u64(&pkt_loop->loopqs[i].stats.in_packets, 0);
+ odp_atomic_init_u64(&pkt_loop->loopqs[i].stats.in_discards, 0);
+ odp_atomic_init_u64(&pkt_loop->loopqs[i].stats.in_errors, 0);
+ odp_atomic_init_u64(&pkt_loop->loopqs[i].stats.out_octets, 0);
+ odp_atomic_init_u64(&pkt_loop->loopqs[i].stats.out_packets, 0);
+ }
+
return 0;
}
@@ -117,6 +153,18 @@ static int loopback_queue_destroy(odp_queue_t queue)
return 0;
}
+static int loopback_queues_destroy(loop_queue_t *queues, uint32_t num_queues)
+{
+ int ret = 0;
+
+ for (uint32_t i = 0; i < num_queues; i++) {
+ if (loopback_queue_destroy(queues[i].queue))
+ ret = -1;
+ }
+
+ return ret;
+}
+
static int loopback_start(pktio_entry_t *pktio_entry)
{
pkt_loop_t *pkt_loop = pkt_priv(pktio_entry);
@@ -127,23 +175,28 @@ static int loopback_start(pktio_entry_t *pktio_entry)
if (!pkt_loop->queue_create)
return 0;
- /* Destroy old queue */
- if (pkt_loop->loopq != ODP_QUEUE_INVALID && loopback_queue_destroy(pkt_loop->loopq))
+ /* Destroy old queues */
+ if (loopback_queues_destroy(pkt_loop->loopqs, pkt_loop->num_qs))
return -1;
- odp_queue_param_init(&queue_param);
- queue_param.size = pkt_loop->pktin_queue_size > pkt_loop->pktout_queue_size ?
- pkt_loop->pktin_queue_size : pkt_loop->pktout_queue_size;
+ pkt_loop->num_qs = 0;
- snprintf(queue_name, sizeof(queue_name), "_odp_pktio_loopq-%" PRIu64 "",
- odp_pktio_to_u64(pktio_entry->handle));
+ for (uint32_t i = 0; i < pkt_loop->num_conf_qs; i++) {
+ odp_queue_param_init(&queue_param);
+ queue_param.size = _ODP_MAX(pkt_loop->loopqs[i].in_size,
+ pkt_loop->loopqs[i].out_size);
+ snprintf(queue_name, sizeof(queue_name), "_odp_pktio_loopq-%" PRIu64 "-%u",
+ odp_pktio_to_u64(pktio_entry->handle), i);
+ pkt_loop->loopqs[i].queue = odp_queue_create(queue_name, &queue_param);
- pkt_loop->loopq = odp_queue_create(queue_name, &queue_param);
- if (pkt_loop->loopq == ODP_QUEUE_INVALID) {
- _ODP_ERR("Creating loopback pktio queue failed\n");
- return -1;
+ if (pkt_loop->loopqs[i].queue == ODP_QUEUE_INVALID) {
+ _ODP_ERR("Creating loopback pktio queue %s failed\n", queue_name);
+ (void)loopback_queues_destroy(pkt_loop->loopqs, i);
+ return -1;
+ }
}
- pkt_loop->queue_create = 0;
+
+ pkt_loop->num_qs = pkt_loop->num_conf_qs;
return 0;
}
@@ -153,9 +206,17 @@ static int loopback_pktin_queue_config(pktio_entry_t *pktio_entry,
{
pkt_loop_t *pkt_loop = pkt_priv(pktio_entry);
+ pkt_loop->num_conf_qs = param->num_queues;
+ pkt_loop->queue_create = 1;
+ pkt_loop->hash.all_bits = param->hash_enable ? param->hash_proto.all_bits : 0;
+
if (pktio_entry->param.in_mode == ODP_PKTIN_MODE_DIRECT) {
- pkt_loop->pktin_queue_size = param->queue_size[0];
- pkt_loop->queue_create = 1;
+ for (uint32_t i = 0; i < MAX_QUEUES; i++) {
+ if (i < pkt_loop->num_conf_qs)
+ pkt_loop->loopqs[i].in_size = param->queue_size[i];
+ else
+ pkt_loop->loopqs[i].in_size = 0;
+ }
}
return 0;
@@ -166,9 +227,15 @@ static int loopback_pktout_queue_config(pktio_entry_t *pktio_entry,
{
pkt_loop_t *pkt_loop = pkt_priv(pktio_entry);
- pkt_loop->pktout_queue_size = param->queue_size[0];
pkt_loop->queue_create = 1;
+ for (uint32_t i = 0; i < MAX_QUEUES; i++) {
+ if (i < param->num_queues)
+ pkt_loop->loopqs[i].out_size = param->queue_size[i];
+ else
+ pkt_loop->loopqs[i].out_size = 0;
+ }
+
return 0;
}
@@ -176,18 +243,16 @@ static int loopback_close(pktio_entry_t *pktio_entry)
{
pkt_loop_t *pkt_loop = pkt_priv(pktio_entry);
- if (pkt_loop->loopq != ODP_QUEUE_INVALID)
- return loopback_queue_destroy(pkt_loop->loopq);
-
- return 0;
+ return loopback_queues_destroy(pkt_loop->loopqs, pkt_loop->num_qs);
}
-static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
- odp_packet_t pkts[], int num)
+static int loopback_recv(pktio_entry_t *pktio_entry, int index, odp_packet_t pkts[], int num)
{
int nbr, i;
+ loop_queue_t *entry = &pkt_priv(pktio_entry)->loopqs[index];
+ odp_queue_t queue = entry->queue;
+ stats_t *stats = &entry->stats;
_odp_event_hdr_t *hdr_tbl[QUEUE_MULTI_MAX];
- odp_queue_t queue;
odp_packet_hdr_t *pkt_hdr;
odp_packet_t pkt;
odp_time_t ts_val;
@@ -201,9 +266,6 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
if (odp_unlikely(num > QUEUE_MULTI_MAX))
num = QUEUE_MULTI_MAX;
- odp_ticketlock_lock(&pktio_entry->rxl);
-
- queue = pkt_priv(pktio_entry)->loopq;
nbr = odp_queue_deq_multi(queue, (odp_event_t *)hdr_tbl, num);
if (opt.bit.ts_all || opt.bit.ts_ptp) {
@@ -239,7 +301,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
ret = _odp_packet_parse_common(pkt_hdr, pkt_addr, pkt_len,
seg_len, layer, opt);
if (ret)
- odp_atomic_inc_u64(&pktio_entry->stats_extra.in_errors);
+ odp_atomic_inc_u64(&stats->in_errors);
if (ret < 0) {
odp_packet_free(pkt);
@@ -252,7 +314,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
ret = _odp_cls_classify_packet(pktio_entry, pkt_addr,
&new_pool, pkt_hdr);
if (ret < 0)
- odp_atomic_inc_u64(&pktio_entry->stats_extra.in_discards);
+ odp_atomic_inc_u64(&stats->in_discards);
if (ret) {
odp_packet_free(pkt);
@@ -262,7 +324,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
if (odp_unlikely(_odp_pktio_packet_to_pool(
&pkt, &pkt_hdr, new_pool))) {
odp_packet_free(pkt);
- odp_atomic_inc_u64(&pktio_entry->stats_extra.in_discards);
+ odp_atomic_inc_u64(&stats->in_discards);
continue;
}
}
@@ -285,10 +347,8 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
pkts[num_rx++] = pkt;
}
- pktio_entry->stats.in_octets += octets;
- pktio_entry->stats.in_packets += packets;
-
- odp_ticketlock_unlock(&pktio_entry->rxl);
+ odp_atomic_add_u64(&stats->in_octets, octets);
+ odp_atomic_add_u64(&stats->in_packets, packets);
return num_rx;
}
@@ -381,22 +441,82 @@ static inline void loopback_fix_checksums(odp_packet_t pkt,
_odp_packet_sctp_chksum_insert(pkt);
}
-static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
- const odp_packet_t pkt_tbl[], int num)
+static inline uint8_t *add_data(uint8_t *data, void *src, uint32_t len)
+{
+ return (uint8_t *)memcpy(data, src, len) + len;
+}
+
+static inline odp_queue_t get_dest_queue(const pkt_loop_t *pkt_loop, odp_packet_t pkt, int index)
+{
+ const odp_pktin_hash_proto_t *hash = &pkt_loop->hash;
+ _odp_udphdr_t udp;
+ _odp_tcphdr_t tcp;
+ _odp_ipv4hdr_t ipv4;
+ _odp_ipv6hdr_t ipv6;
+ uint32_t off;
+ /* Space for UDP/TCP source and destination ports and IPv4/IPv6 source and destination
+ * addresses. */
+ uint8_t data[2 * sizeof(uint16_t) + 2 * 4 * sizeof(uint32_t)];
+ uint8_t *head = data;
+
+ if (hash->all_bits == 0)
+ return pkt_loop->loopqs[index % pkt_loop->num_qs].queue;
+
+ memset(data, 0, sizeof(data));
+ off = odp_packet_l4_offset(pkt);
+
+ if (off != ODP_PACKET_OFFSET_INVALID) {
+ if ((hash->proto.ipv4_udp || hash->proto.ipv6_udp) && odp_packet_has_udp(pkt)) {
+ if (odp_packet_copy_to_mem(pkt, off, _ODP_UDPHDR_LEN, &udp) == 0) {
+ head = add_data(head, &udp.src_port, sizeof(udp.src_port));
+ head = add_data(head, &udp.dst_port, sizeof(udp.dst_port));
+ }
+ } else if ((hash->proto.ipv4_tcp || hash->proto.ipv6_tcp) &&
+ odp_packet_has_tcp(pkt)) {
+ if (odp_packet_copy_to_mem(pkt, off, _ODP_TCPHDR_LEN, &tcp) == 0) {
+ head = add_data(head, &tcp.src_port, sizeof(tcp.src_port));
+ head = add_data(head, &tcp.dst_port, sizeof(tcp.dst_port));
+ }
+ }
+ }
+
+ off = odp_packet_l3_offset(pkt);
+
+ if (off != ODP_PACKET_OFFSET_INVALID) {
+ if (hash->proto.ipv4 && odp_packet_has_ipv4(pkt)) {
+ if (odp_packet_copy_to_mem(pkt, off, _ODP_IPV4HDR_LEN, &ipv4) == 0) {
+ head = add_data(head, &ipv4.src_addr, sizeof(ipv4.src_addr));
+ head = add_data(head, &ipv4.dst_addr, sizeof(ipv4.dst_addr));
+ }
+ } else if (hash->proto.ipv6 && odp_packet_has_ipv6(pkt)) {
+ if (odp_packet_copy_to_mem(pkt, off, _ODP_IPV6HDR_LEN, &ipv6) == 0) {
+ head = add_data(head, &ipv6.src_addr, sizeof(ipv6.src_addr));
+ head = add_data(head, &ipv6.dst_addr, sizeof(ipv6.dst_addr));
+ }
+ }
+ }
+
+ return pkt_loop->loopqs[odp_hash_crc32c(data, head - data, 0) % pkt_loop->num_qs].queue;
+}
+
+static int loopback_send(pktio_entry_t *pktio_entry, int index, const odp_packet_t pkt_tbl[],
+ int num)
{
pkt_loop_t *pkt_loop = pkt_priv(pktio_entry);
- _odp_event_hdr_t *hdr_tbl[QUEUE_MULTI_MAX];
odp_queue_t queue;
+ stats_t *stats;
int i;
int ret;
int nb_tx = 0;
int tx_ts_idx = 0;
uint8_t tx_ts_enabled = _odp_pktio_tx_ts_enabled(pktio_entry);
- uint32_t bytes = 0;
- uint32_t out_octets_tbl[num];
odp_pktout_config_opt_t *pktout_cfg = &pktio_entry->config.pktout;
- odp_pktout_config_opt_t *pktout_capa =
- &pktio_entry->capa.config.pktout;
+ odp_pktout_config_opt_t *pktout_capa = &pktio_entry->capa.config.pktout;
+
+ if (pkt_loop->num_qs == 0)
+ return 0;
+
+ stats = &pkt_loop->loopqs[index].stats;
if (odp_unlikely(num > QUEUE_MULTI_MAX))
num = QUEUE_MULTI_MAX;
@@ -411,44 +531,34 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
}
break;
}
- hdr_tbl[i] = packet_to_event_hdr(pkt_tbl[i]);
- bytes += pkt_len;
- /* Store cumulative byte counts to update 'stats.out_octets'
- * correctly in case enq_multi() fails to enqueue all packets.
- */
- out_octets_tbl[i] = bytes;
- nb_tx++;
if (tx_ts_enabled && tx_ts_idx == 0) {
if (odp_unlikely(packet_hdr(pkt_tbl[i])->p.flags.ts_set))
tx_ts_idx = i + 1;
}
- }
- for (i = 0; i < nb_tx; ++i) {
packet_subtype_set(pkt_tbl[i], ODP_EVENT_PACKET_BASIC);
loopback_fix_checksums(pkt_tbl[i], pktout_cfg, pktout_capa);
- }
+ queue = get_dest_queue(pkt_loop, pkt_tbl[i], index);
+ ret = odp_queue_enq(queue, odp_packet_to_event(pkt_tbl[i]));
- odp_ticketlock_lock(&pktio_entry->txl);
+ if (ret < 0) {
+ _ODP_DBG("queue enqueue failed %i to queue: %" PRIu64 "\n", ret,
+ odp_queue_to_u64(queue));
+ break;
+ }
- queue = pkt_priv(pktio_entry)->loopq;
- ret = odp_queue_enq_multi(queue, (odp_event_t *)hdr_tbl, nb_tx);
+ nb_tx++;
+ odp_atomic_inc_u64(&stats->out_packets);
+ odp_atomic_add_u64(&stats->out_octets, pkt_len);
+ }
- if (ret > 0) {
- if (odp_unlikely(tx_ts_idx) && ret >= tx_ts_idx)
+ if (nb_tx > 0) {
+ if (odp_unlikely(tx_ts_idx) && nb_tx >= tx_ts_idx)
_odp_pktio_tx_ts_set(pktio_entry);
-
- pktio_entry->stats.out_packets += ret;
- pktio_entry->stats.out_octets += out_octets_tbl[ret - 1];
- } else {
- _ODP_DBG("queue enqueue failed %i\n", ret);
- ret = -1;
}
- odp_ticketlock_unlock(&pktio_entry->txl);
-
- return ret;
+ return nb_tx;
}
static uint32_t loopback_mtu_get(pktio_entry_t *pktio_entry)
@@ -508,8 +618,8 @@ static int loopback_init_capability(pktio_entry_t *pktio_entry)
memset(capa, 0, sizeof(odp_pktio_capability_t));
- capa->max_input_queues = 1;
- capa->max_output_queues = 1;
+ capa->max_input_queues = ODP_PKTIN_MAX_QUEUES;
+ capa->max_output_queues = ODP_PKTOUT_MAX_QUEUES;
capa->set_op.op.promisc_mode = 0;
capa->set_op.op.maxlen = 1;
@@ -566,6 +676,7 @@ static int loopback_init_capability(pktio_entry_t *pktio_entry)
capa->stats.pktin_queue.counter.octets = 1;
capa->stats.pktin_queue.counter.packets = 1;
capa->stats.pktin_queue.counter.errors = 1;
+ capa->stats.pktin_queue.counter.discards = 1;
capa->stats.pktout_queue.counter.octets = 1;
capa->stats.pktout_queue.counter.packets = 1;
return 0;
@@ -582,37 +693,67 @@ static int loopback_promisc_mode_get(pktio_entry_t *pktio_entry ODP_UNUSED)
return 1;
}
-static int loopback_stats(pktio_entry_t *pktio_entry,
- odp_pktio_stats_t *stats)
+static int loopback_stats(pktio_entry_t *pktio_entry, odp_pktio_stats_t *stats)
{
- memcpy(stats, &pktio_entry->stats, sizeof(odp_pktio_stats_t));
+ pkt_loop_t *pkt_loop = pkt_priv(pktio_entry);
+
+ memset(stats, 0, sizeof(odp_pktio_stats_t));
+
+ for (uint32_t i = 0; i < MAX_QUEUES; i++) {
+ stats_t *qs = &pkt_loop->loopqs[i].stats;
+
+ stats->in_octets += odp_atomic_load_u64(&qs->in_octets);
+ stats->in_packets += odp_atomic_load_u64(&qs->in_packets);
+ stats->in_discards += odp_atomic_load_u64(&qs->in_discards);
+ stats->in_errors += odp_atomic_load_u64(&qs->in_errors);
+ stats->out_octets += odp_atomic_load_u64(&qs->out_octets);
+ stats->out_packets += odp_atomic_load_u64(&qs->out_packets);
+ }
+
return 0;
}
static int loopback_stats_reset(pktio_entry_t *pktio_entry)
{
- memset(&pktio_entry->stats, 0, sizeof(odp_pktio_stats_t));
+ pkt_loop_t *pkt_loop = pkt_priv(pktio_entry);
+
+ for (uint32_t i = 0; i < MAX_QUEUES; i++) {
+ stats_t *qs = &pkt_loop->loopqs[i].stats;
+
+ odp_atomic_store_u64(&qs->in_octets, 0);
+ odp_atomic_store_u64(&qs->in_packets, 0);
+ odp_atomic_store_u64(&qs->in_discards, 0);
+ odp_atomic_store_u64(&qs->in_errors, 0);
+ odp_atomic_store_u64(&qs->out_octets, 0);
+ odp_atomic_store_u64(&qs->out_packets, 0);
+ }
+
return 0;
}
-static int loopback_pktin_stats(pktio_entry_t *pktio_entry,
- uint32_t index ODP_UNUSED,
+static int loopback_pktin_stats(pktio_entry_t *pktio_entry, uint32_t index,
odp_pktin_queue_stats_t *pktin_stats)
{
+ stats_t *qs = &pkt_priv(pktio_entry)->loopqs[index].stats;
+
memset(pktin_stats, 0, sizeof(odp_pktin_queue_stats_t));
- pktin_stats->octets = pktio_entry->stats.in_octets;
- pktin_stats->packets = pktio_entry->stats.in_packets;
- pktin_stats->errors = pktio_entry->stats.in_errors;
+ pktin_stats->octets = odp_atomic_load_u64(&qs->in_octets);
+ pktin_stats->packets = odp_atomic_load_u64(&qs->in_packets);
+ pktin_stats->discards = odp_atomic_load_u64(&qs->in_discards);
+ pktin_stats->errors = odp_atomic_load_u64(&qs->in_errors);
+
return 0;
}
-static int loopback_pktout_stats(pktio_entry_t *pktio_entry,
- uint32_t index ODP_UNUSED,
+static int loopback_pktout_stats(pktio_entry_t *pktio_entry, uint32_t index,
odp_pktout_queue_stats_t *pktout_stats)
{
+ stats_t *qs = &pkt_priv(pktio_entry)->loopqs[index].stats;
+
memset(pktout_stats, 0, sizeof(odp_pktout_queue_stats_t));
- pktout_stats->octets = pktio_entry->stats.out_octets;
- pktout_stats->packets = pktio_entry->stats.out_packets;
+ pktout_stats->octets = odp_atomic_load_u64(&qs->out_octets);
+ pktout_stats->packets = odp_atomic_load_u64(&qs->out_packets);
+
return 0;
}
diff --git a/platform/linux-generic/pktio/stats/packet_io_stats.c b/platform/linux-generic/pktio/stats/packet_io_stats.c
index ac61c0343..280aca250 100644
--- a/platform/linux-generic/pktio/stats/packet_io_stats.c
+++ b/platform/linux-generic/pktio/stats/packet_io_stats.c
@@ -5,7 +5,6 @@
* SPDX-License-Identifier: BSD-3-Clause
*/
-#include <odp/api/deprecated.h>
#include <odp_packet_io_stats.h>
#include <odp_ethtool_stats.h>
#include <odp_sysfs_stats.h>
@@ -77,10 +76,6 @@ int _odp_sock_stats_fd(pktio_entry_t *pktio_entry,
pktio_entry->stats.in_discards;
stats->in_errors = cur_stats.in_errors -
pktio_entry->stats.in_errors;
-#if ODP_DEPRECATED_API
- stats->in_unknown_protos = cur_stats.in_unknown_protos -
- pktio_entry->stats.in_unknown_protos;
-#endif
stats->out_octets = cur_stats.out_octets -
pktio_entry->stats.out_octets;
stats->out_packets = cur_stats.out_packets -
diff --git a/platform/linux-generic/test/inline-timer.conf b/platform/linux-generic/test/inline-timer.conf
index 261aa0141..44db4e337 100644
--- a/platform/linux-generic/test/inline-timer.conf
+++ b/platform/linux-generic/test/inline-timer.conf
@@ -1,6 +1,6 @@
# Mandatory fields
odp_implementation = "linux-generic"
-config_file_version = "0.1.22"
+config_file_version = "0.1.25"
timer: {
# Enable inline timer implementation
diff --git a/platform/linux-generic/test/packet_align.conf b/platform/linux-generic/test/packet_align.conf
index 8d2d00e63..26491bd53 100644
--- a/platform/linux-generic/test/packet_align.conf
+++ b/platform/linux-generic/test/packet_align.conf
@@ -1,6 +1,6 @@
# Mandatory fields
odp_implementation = "linux-generic"
-config_file_version = "0.1.22"
+config_file_version = "0.1.25"
pool: {
pkt: {
diff --git a/platform/linux-generic/test/process-mode.conf b/platform/linux-generic/test/process-mode.conf
index 1e0e7cc95..2277aabdf 100644
--- a/platform/linux-generic/test/process-mode.conf
+++ b/platform/linux-generic/test/process-mode.conf
@@ -1,6 +1,6 @@
# Mandatory fields
odp_implementation = "linux-generic"
-config_file_version = "0.1.22"
+config_file_version = "0.1.25"
# Shared memory options
shm: {
diff --git a/platform/linux-generic/test/sched-basic.conf b/platform/linux-generic/test/sched-basic.conf
index e63ffa2f3..c9f7c79fd 100644
--- a/platform/linux-generic/test/sched-basic.conf
+++ b/platform/linux-generic/test/sched-basic.conf
@@ -1,9 +1,10 @@
# Mandatory fields
odp_implementation = "linux-generic"
-config_file_version = "0.1.22"
+config_file_version = "0.1.25"
# Test scheduler with an odd spread value and without dynamic load balance
sched_basic: {
prio_spread = 3
load_balance = 0
+ order_stash_size = 0
}
diff --git a/platform/linux-generic/test/stash-custom.conf b/platform/linux-generic/test/stash-custom.conf
new file mode 100644
index 000000000..95af7a259
--- /dev/null
+++ b/platform/linux-generic/test/stash-custom.conf
@@ -0,0 +1,8 @@
+# Mandatory fields
+odp_implementation = "linux-generic"
+config_file_version = "0.1.25"
+
+# Test overflow safe stash variant
+stash: {
+ strict_size = 0
+}
diff --git a/test/Makefile.inc b/test/Makefile.inc
index f5235f471..6d14f9c17 100644
--- a/test/Makefile.inc
+++ b/test/Makefile.inc
@@ -10,6 +10,7 @@ LIBODP = $(LIB)/libodphelper.la $(LIB)/lib$(ODP_LIB_NAME).la
LIBCUNIT_COMMON = $(COMMON_DIR)/libcunit_common.la
LIBCPUMASK_COMMON = $(COMMON_DIR)/libcpumask_common.la
+LIBPACKET_COMMON = $(COMMON_DIR)/libpacket_common.la
LIBTHRMASK_COMMON = $(COMMON_DIR)/libthrmask_common.la
#in the following line, the libs using the symbols should come before
diff --git a/test/common/Makefile.am b/test/common/Makefile.am
index 694b442c2..72658df73 100644
--- a/test/common/Makefile.am
+++ b/test/common/Makefile.am
@@ -2,13 +2,19 @@ include $(top_srcdir)/test/Makefile.inc
if cunit_support
-noinst_LTLIBRARIES = libcunit_common.la libcpumask_common.la libthrmask_common.la
+noinst_LTLIBRARIES = \
+ libcunit_common.la \
+ libcpumask_common.la \
+ libpacket_common.la \
+ libthrmask_common.la
libcunit_common_la_SOURCES = odp_cunit_common.c odp_cunit_common.h
libcunit_common_la_LIBADD = $(CUNIT_LIBS)
libcpumask_common_la_SOURCES = mask_common.c mask_common.h
+libpacket_common_la_SOURCES = packet_common.c packet_common.h
+
libthrmask_common_la_SOURCES = mask_common.c mask_common.h
libthrmask_common_la_CFLAGS = $(AM_CFLAGS) -DTEST_THRMASK
diff --git a/test/common/packet_common.c b/test/common/packet_common.c
new file mode 100644
index 000000000..e0bca3147
--- /dev/null
+++ b/test/common/packet_common.c
@@ -0,0 +1,133 @@
+/* Copyright (c) 2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <packet_common.h>
+#include <string.h>
+
+void test_packet_set_md(odp_packet_t pkt)
+{
+ const int binary_flag = 1;
+ const uint32_t offset = 1;
+ uint8_t *uarea = odp_packet_user_area(pkt);
+ uint32_t uarea_size = odp_packet_user_area_size(pkt);
+
+ for (uint32_t n = 0; n < uarea_size; n++)
+ uarea[n] = n;
+
+ /*
+ * Some of these flags cannot be set simultaneously, but we do not
+ * care here if some flags get cleared.
+ */
+ odp_packet_has_l2_set(pkt, binary_flag);
+ odp_packet_has_l3_set(pkt, binary_flag);
+ odp_packet_has_l4_set(pkt, binary_flag);
+ odp_packet_has_eth_set(pkt, binary_flag);
+ odp_packet_has_eth_bcast_set(pkt, binary_flag);
+ odp_packet_has_eth_mcast_set(pkt, !binary_flag);
+ odp_packet_has_jumbo_set(pkt, binary_flag);
+ odp_packet_has_vlan_set(pkt, binary_flag);
+ odp_packet_has_vlan_qinq_set(pkt, binary_flag);
+ odp_packet_has_arp_set(pkt, binary_flag);
+ odp_packet_has_ipv4_set(pkt, binary_flag);
+ odp_packet_has_ipv6_set(pkt, !binary_flag);
+ odp_packet_has_ip_bcast_set(pkt, binary_flag);
+ odp_packet_has_ip_mcast_set(pkt, binary_flag);
+ odp_packet_has_ipfrag_set(pkt, binary_flag);
+ odp_packet_has_ipopt_set(pkt, binary_flag);
+ odp_packet_has_ipsec_set(pkt, binary_flag);
+ odp_packet_has_udp_set(pkt, binary_flag);
+ odp_packet_has_tcp_set(pkt, !binary_flag);
+ odp_packet_has_sctp_set(pkt, binary_flag);
+ odp_packet_has_icmp_set(pkt, binary_flag);
+
+ odp_packet_user_ptr_set(pkt, &pkt);
+ odp_packet_user_flag_set(pkt, binary_flag);
+ (void)odp_packet_l2_offset_set(pkt, offset);
+ (void)odp_packet_l3_offset_set(pkt, offset);
+ (void)odp_packet_l4_offset_set(pkt, offset);
+ odp_packet_flow_hash_set(pkt, 0x12345678);
+ odp_packet_ts_set(pkt, odp_time_local_from_ns(ODP_TIME_SEC_IN_NS));
+ odp_packet_color_set(pkt, ODP_PACKET_YELLOW);
+ odp_packet_drop_eligible_set(pkt, binary_flag);
+ odp_packet_shaper_len_adjust_set(pkt, -42);
+ (void)odp_packet_payload_offset_set(pkt, offset);
+}
+
+void test_packet_get_md(odp_packet_t pkt, test_packet_md_t *md)
+{
+ uint8_t *uarea = odp_packet_user_area(pkt);
+ uint32_t uarea_size = odp_packet_user_area_size(pkt);
+
+ memset(md, 0, sizeof(*md));
+
+ if (uarea)
+ md->user_area_chksum = odp_chksum_ones_comp16(uarea, uarea_size);
+
+ md->has_error = !!odp_packet_has_error(pkt);
+ md->has_l2_error = !!odp_packet_has_l2_error(pkt);
+ md->has_l3_error = !!odp_packet_has_l3_error(pkt);
+ md->has_l4_error = !!odp_packet_has_l4_error(pkt);
+ md->has_l2 = !!odp_packet_has_l2(pkt);
+ md->has_l3 = !!odp_packet_has_l3(pkt);
+ md->has_l4 = !!odp_packet_has_l4(pkt);
+ md->has_eth = !!odp_packet_has_eth(pkt);
+ md->has_eth_bcast = !!odp_packet_has_eth_bcast(pkt);
+ md->has_eth_mcast = !!odp_packet_has_eth_mcast(pkt);
+ md->has_jumbo = !!odp_packet_has_jumbo(pkt);
+ md->has_vlan = !!odp_packet_has_vlan(pkt);
+ md->has_vlan_qinq = !!odp_packet_has_vlan_qinq(pkt);
+ md->has_arp = !!odp_packet_has_arp(pkt);
+ md->has_ipv4 = !!odp_packet_has_ipv4(pkt);
+ md->has_ipv6 = !!odp_packet_has_ipv6(pkt);
+ md->has_ip_bcast = !!odp_packet_has_ip_bcast(pkt);
+ md->has_ip_mcast = !!odp_packet_has_ip_mcast(pkt);
+ md->has_ipfrag = !!odp_packet_has_ipfrag(pkt);
+ md->has_ipopt = !!odp_packet_has_ipopt(pkt);
+ md->has_ipsec = !!odp_packet_has_ipsec(pkt);
+ md->has_udp = !!odp_packet_has_udp(pkt);
+ md->has_tcp = !!odp_packet_has_tcp(pkt);
+ md->has_sctp = !!odp_packet_has_sctp(pkt);
+ md->has_icmp = !!odp_packet_has_icmp(pkt);
+ md->has_flow_hash = !!odp_packet_has_flow_hash(pkt);
+ md->has_ts = !!odp_packet_has_ts(pkt);
+
+ md->len = odp_packet_len(pkt);
+ md->packet_pool = odp_packet_pool(pkt);
+ md->packet_input = odp_packet_input(pkt);
+ md->user_ptr = odp_packet_user_ptr(pkt);
+ md->user_flag = odp_packet_user_flag(pkt);
+ md->l2_offset = odp_packet_l2_offset(pkt);
+ md->l3_offset = odp_packet_l3_offset(pkt);
+ md->l4_offset = odp_packet_l4_offset(pkt);
+ md->l2_type = odp_packet_l2_type(pkt);
+ md->l3_type = odp_packet_l3_type(pkt);
+ md->l4_type = odp_packet_l4_type(pkt);
+ md->l3_chksum_status = odp_packet_l3_chksum_status(pkt);
+ md->l4_chksum_status = odp_packet_l4_chksum_status(pkt);
+ md->flow_hash = md->has_flow_hash ? odp_packet_flow_hash(pkt) : 0;
+ md->ts = md->has_ts ? odp_packet_ts(pkt) : odp_time_global_from_ns(0);
+ md->color = odp_packet_color(pkt);
+ md->drop_eligible = !!odp_packet_drop_eligible(pkt);
+ md->shaper_len_adjust = odp_packet_shaper_len_adjust(pkt);
+ md->cls_mark = odp_packet_cls_mark(pkt);
+ md->has_lso_request = !!odp_packet_has_lso_request(pkt);
+ md->payload_offset = odp_packet_payload_offset(pkt);
+ md->aging_tmo = odp_packet_aging_tmo(pkt);
+ md->has_tx_compl_request = !!odp_packet_has_tx_compl_request(pkt);
+ md->proto_stats = odp_packet_proto_stats(pkt);
+}
+
+int test_packet_is_md_equal(const test_packet_md_t *md_1,
+ const test_packet_md_t *md_2)
+{
+ /*
+ * With certain assumptions this should typically work. If it does
+ * not, we would get false negatives which we should spot as test
+ * failures.
+ */
+
+ return (!memcmp(md_1, md_2, sizeof(*md_1)));
+}
diff --git a/test/common/packet_common.h b/test/common/packet_common.h
new file mode 100644
index 000000000..c7cd5e27f
--- /dev/null
+++ b/test/common/packet_common.h
@@ -0,0 +1,67 @@
+/* Copyright (c) 2023, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp_api.h>
+
+typedef struct test_packet_md_t {
+ int has_error;
+ int has_l2_error;
+ int has_l3_error;
+ int has_l4_error;
+ int has_l2;
+ int has_l3;
+ int has_l4;
+ int has_eth;
+ int has_eth_bcast;
+ int has_eth_mcast;
+ int has_jumbo;
+ int has_vlan;
+ int has_vlan_qinq;
+ int has_arp;
+ int has_ipv4;
+ int has_ipv6;
+ int has_ip_bcast;
+ int has_ip_mcast;
+ int has_ipfrag;
+ int has_ipopt;
+ int has_ipsec;
+ int has_udp;
+ int has_tcp;
+ int has_sctp;
+ int has_icmp;
+ int has_flow_hash;
+ int has_ts;
+ uint32_t len;
+ odp_pool_t packet_pool;
+ odp_pktio_t packet_input;
+ void *user_ptr;
+ uint16_t user_area_chksum;
+ int user_flag;
+ uint32_t l2_offset;
+ uint32_t l3_offset;
+ uint32_t l4_offset;
+ odp_proto_l2_type_t l2_type;
+ odp_proto_l3_type_t l3_type;
+ odp_proto_l4_type_t l4_type;
+ odp_packet_chksum_status_t l3_chksum_status;
+ odp_packet_chksum_status_t l4_chksum_status;
+ uint32_t flow_hash;
+ odp_time_t ts;
+ odp_packet_color_t color;
+ odp_bool_t drop_eligible;
+ int8_t shaper_len_adjust;
+ uint64_t cls_mark;
+ int has_lso_request;
+ uint32_t payload_offset;
+ uint64_t aging_tmo;
+ int has_tx_compl_request;
+ odp_proto_stats_t proto_stats;
+} test_packet_md_t;
+
+void test_packet_set_md(odp_packet_t pkt);
+void test_packet_get_md(odp_packet_t pkt, test_packet_md_t *md);
+int test_packet_is_md_equal(const test_packet_md_t *md_1,
+ const test_packet_md_t *md_2);
diff --git a/test/performance/.gitignore b/test/performance/.gitignore
index ec1915bba..b86699c91 100644
--- a/test/performance/.gitignore
+++ b/test/performance/.gitignore
@@ -3,6 +3,7 @@
odp_atomic
odp_atomic_perf
odp_bench_buffer
+odp_bench_misc
odp_bench_packet
odp_cpu_bench
odp_crc
diff --git a/test/performance/Makefile.am b/test/performance/Makefile.am
index 336a46d22..beeb652cc 100644
--- a/test/performance/Makefile.am
+++ b/test/performance/Makefile.am
@@ -4,6 +4,7 @@ TESTS_ENVIRONMENT += TEST_DIR=${builddir}
EXECUTABLES = odp_atomic_perf \
odp_bench_buffer \
+ odp_bench_misc \
odp_bench_packet \
odp_crc \
odp_lock_perf \
@@ -54,6 +55,7 @@ bin_PROGRAMS = $(EXECUTABLES) $(COMPILE_ONLY)
odp_atomic_perf_SOURCES = odp_atomic_perf.c
odp_bench_buffer_SOURCES = odp_bench_buffer.c
+odp_bench_misc_SOURCES = odp_bench_misc.c
odp_bench_packet_SOURCES = odp_bench_packet.c
odp_cpu_bench_SOURCES = odp_cpu_bench.c
odp_crc_SOURCES = odp_crc.c
diff --git a/test/performance/odp_bench_buffer.c b/test/performance/odp_bench_buffer.c
index e7029f57d..e0e7c85cd 100644
--- a/test/performance/odp_bench_buffer.c
+++ b/test/performance/odp_bench_buffer.c
@@ -199,7 +199,7 @@ static int run_benchmarks(void *arg)
/* Skip unsupported tests */
if (args->bench[j].cond != NULL && !args->bench[j].cond()) {
j++;
- k = 0;
+ k = 1;
if (i > 0)
printf("[%02d] odp_%-26s: n/a\n", j, desc);
continue;
@@ -236,7 +236,7 @@ static int run_benchmarks(void *arg)
printf("[%02d] odp_%-26s: %8.1f\n", j + 1, desc, cycles);
j++;
- k = 0;
+ k = 1;
tot_cycles = 0;
}
}
diff --git a/test/performance/odp_bench_misc.c b/test/performance/odp_bench_misc.c
new file mode 100644
index 000000000..8b9e27bf2
--- /dev/null
+++ b/test/performance/odp_bench_misc.c
@@ -0,0 +1,1069 @@
+/* Copyright (c) 2022, Nokia
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE /* Needed for sigaction */
+#endif
+
+#include <odp_api.h>
+#include <odp/helper/odph_api.h>
+
+#include <getopt.h>
+#include <inttypes.h>
+#include <signal.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+/* Number of API function calls per test case */
+#define REPEAT_COUNT 1000
+
+/* Default number of rounds per test case */
+#define ROUNDS 1000u
+
+#define BENCH_INFO(run, init, max, name) \
+ {#run, run, init, max, name}
+
+typedef struct {
+ /* Measure time vs CPU cycles */
+ int time;
+
+ /* Benchmark index to run indefinitely */
+ int bench_idx;
+
+ /* Rounds per test case */
+ uint32_t rounds;
+
+} appl_args_t;
+
+/* Initialize benchmark resources */
+typedef void (*bench_init_fn_t)(void);
+
+/* Run benchmark, returns >0 on success */
+typedef int (*bench_run_fn_t)(void);
+
+/* Benchmark data */
+typedef struct {
+ /* Default test name */
+ const char *name;
+
+ /* Test function to run */
+ bench_run_fn_t run;
+
+ /* Test init function */
+ bench_init_fn_t init;
+
+ /* Test specific limit for rounds (tuning for slow implementation) */
+ uint32_t max_rounds;
+
+ /* Override default test name */
+ const char *desc;
+
+} bench_info_t;
+
+/* Global data */
+typedef struct {
+ appl_args_t appl;
+
+ /* Benchmark functions */
+ bench_info_t *bench;
+
+ /* Number of benchmark functions */
+ int num_bench;
+
+ /* Break worker loop if set to 1 */
+ odp_atomic_u32_t exit_thread;
+
+ /* Test case input / output data */
+ odp_time_t t1[REPEAT_COUNT];
+ odp_time_t t2[REPEAT_COUNT];
+ odp_time_t t3[REPEAT_COUNT];
+ uint64_t a1[REPEAT_COUNT];
+ uint64_t a2[REPEAT_COUNT];
+ uint32_t b1[REPEAT_COUNT];
+ uint32_t b2[REPEAT_COUNT];
+ uint16_t c1[REPEAT_COUNT];
+ uint16_t c2[REPEAT_COUNT];
+
+ /* Dummy result */
+ uint64_t dummy;
+
+ /* Benchmark run failed */
+ int bench_failed;
+
+ /* CPU mask as string */
+ char cpumask_str[ODP_CPUMASK_STR_SIZE];
+
+} gbl_args_t;
+
+static gbl_args_t *gbl_args;
+
+static void sig_handler(int signo ODP_UNUSED)
+{
+ if (gbl_args == NULL)
+ return;
+ odp_atomic_store_u32(&gbl_args->exit_thread, 1);
+}
+
+static int setup_sig_handler(void)
+{
+ struct sigaction action;
+
+ memset(&action, 0, sizeof(action));
+ action.sa_handler = sig_handler;
+
+ /* No additional signals blocked. By default, the signal which triggered
+ * the handler is blocked. */
+ if (sigemptyset(&action.sa_mask))
+ return -1;
+
+ if (sigaction(SIGINT, &action, NULL))
+ return -1;
+
+ return 0;
+}
+
+/* Run given benchmark indefinitely */
+static void run_indef(gbl_args_t *args, int idx)
+{
+ const char *desc;
+ const bench_info_t *bench = &args->bench[idx];
+
+ desc = bench->desc != NULL ? bench->desc : bench->name;
+
+ printf("Running odp_%s test indefinitely\n", desc);
+
+ while (!odp_atomic_load_u32(&gbl_args->exit_thread)) {
+ int ret;
+
+ if (bench->init != NULL)
+ bench->init();
+
+ ret = bench->run();
+
+ if (!ret)
+ ODPH_ABORT("Benchmark %s failed\n", desc);
+ }
+}
+
+static int run_benchmarks(void *arg)
+{
+ int i, j;
+ uint64_t c1, c2;
+ odp_time_t t1, t2;
+ gbl_args_t *args = arg;
+ const int meas_time = args->appl.time;
+
+ printf("\nAverage %s per function call\n", meas_time ? "time (nsec)" : "CPU cycles");
+ printf("-------------------------------------------------\n");
+
+ /* Run each test twice. Results from the first warm-up round are ignored. */
+ for (i = 0; i < 2; i++) {
+ uint64_t total = 0;
+ uint32_t round = 1;
+
+ for (j = 0; j < gbl_args->num_bench; round++) {
+ int ret;
+ const char *desc;
+ const bench_info_t *bench = &args->bench[j];
+ uint32_t max_rounds = args->appl.rounds;
+
+ if (bench->max_rounds && max_rounds > bench->max_rounds)
+ max_rounds = bench->max_rounds;
+
+ /* Run selected test indefinitely */
+ if (args->appl.bench_idx) {
+ if ((j + 1) != args->appl.bench_idx) {
+ j++;
+ continue;
+ }
+
+ run_indef(args, j);
+ return 0;
+ }
+
+ desc = bench->desc != NULL ? bench->desc : bench->name;
+
+ if (bench->init != NULL)
+ bench->init();
+
+ if (meas_time)
+ t1 = odp_time_local();
+ else
+ c1 = odp_cpu_cycles();
+
+ ret = bench->run();
+
+ if (meas_time)
+ t2 = odp_time_local();
+ else
+ c2 = odp_cpu_cycles();
+
+ if (!ret) {
+ ODPH_ERR("Benchmark odp_%s failed\n", desc);
+ args->bench_failed = -1;
+ return -1;
+ }
+
+ if (meas_time)
+ total += odp_time_diff_ns(t2, t1);
+ else
+ total += odp_cpu_cycles_diff(c2, c1);
+
+ if (round >= max_rounds) {
+ double result;
+
+ /* Each benchmark runs internally REPEAT_COUNT times. */
+ result = ((double)total) / (max_rounds * REPEAT_COUNT);
+
+ /* No print from warm-up round */
+ if (i > 0)
+ printf("[%02d] odp_%-26s: %12.2f\n", j + 1, desc, result);
+
+ j++;
+ total = 0;
+ round = 1;
+ }
+ }
+ }
+
+ /* Print dummy result to prevent compiler to optimize it away*/
+ printf("\n(dummy result: 0x%" PRIx64 ")\n", args->dummy);
+
+ printf("\n");
+
+ return 0;
+}
+
+static void init_time_global(void)
+{
+ int i;
+ odp_time_t *t1 = gbl_args->t1;
+ odp_time_t *t2 = gbl_args->t2;
+ uint64_t *a1 = gbl_args->a1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ t1[i] = odp_time_global();
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ t2[i] = odp_time_global();
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a1[i] = odp_time_global_ns();
+}
+
+static void init_time_local(void)
+{
+ int i;
+ odp_time_t *t1 = gbl_args->t1;
+ odp_time_t *t2 = gbl_args->t2;
+ uint64_t *a1 = gbl_args->a1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ t1[i] = odp_time_local();
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ t2[i] = odp_time_local();
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a1[i] = odp_time_local_ns();
+}
+
+static void init_cpu_cycles(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+ uint64_t *a2 = gbl_args->a2;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a1[i] = odp_cpu_cycles();
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a2[i] = odp_cpu_cycles();
+}
+
+static int time_local(void)
+{
+ int i;
+ odp_time_t *t1 = gbl_args->t1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ t1[i] = odp_time_local();
+
+ return i;
+}
+
+static int time_local_strict(void)
+{
+ int i;
+ odp_time_t *t1 = gbl_args->t1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ t1[i] = odp_time_local_strict();
+
+ return i;
+}
+
+static int time_local_ns(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a1[i] = odp_time_local_ns();
+
+ return i;
+}
+
+static int time_local_strict_ns(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a1[i] = odp_time_local_strict_ns();
+
+ return i;
+}
+
+static int time_global(void)
+{
+ int i;
+ odp_time_t *t1 = gbl_args->t1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ t1[i] = odp_time_global();
+
+ return i;
+}
+
+static int time_global_strict(void)
+{
+ int i;
+ odp_time_t *t1 = gbl_args->t1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ t1[i] = odp_time_global_strict();
+
+ return i;
+}
+
+static int time_global_ns(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a1[i] = odp_time_global_ns();
+
+ return i;
+}
+
+static int time_global_strict_ns(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a1[i] = odp_time_global_strict_ns();
+
+ return i;
+}
+
+static int time_diff(void)
+{
+ int i;
+ odp_time_t *t1 = gbl_args->t1;
+ odp_time_t *t2 = gbl_args->t2;
+ odp_time_t *t3 = gbl_args->t3;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ t3[i] = odp_time_diff(t2[i], t1[i]);
+
+ return i;
+}
+
+static int time_diff_ns(void)
+{
+ int i;
+ odp_time_t *t1 = gbl_args->t1;
+ odp_time_t *t2 = gbl_args->t2;
+ uint64_t res = 0;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ res += odp_time_diff_ns(t2[i], t1[i]);
+
+ gbl_args->dummy += res;
+
+ return i;
+}
+
+static int time_sum(void)
+{
+ int i;
+ odp_time_t *t1 = gbl_args->t1;
+ odp_time_t *t2 = gbl_args->t2;
+ odp_time_t *t3 = gbl_args->t3;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ t3[i] = odp_time_sum(t1[i], t2[i]);
+
+ return i;
+}
+
+static int time_to_ns(void)
+{
+ int i;
+ odp_time_t *t1 = gbl_args->t1;
+ uint64_t res = 0;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ res += odp_time_to_ns(t1[i]);
+
+ gbl_args->dummy += res;
+
+ return i;
+}
+
+static int time_local_from_ns(void)
+{
+ int i;
+ odp_time_t *t1 = gbl_args->t1;
+ uint64_t *a1 = gbl_args->a1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ t1[i] = odp_time_local_from_ns(a1[i]);
+
+ return i;
+}
+
+static int time_global_from_ns(void)
+{
+ int i;
+ odp_time_t *t1 = gbl_args->t1;
+ uint64_t *a1 = gbl_args->a1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ t1[i] = odp_time_global_from_ns(a1[i]);
+
+ return i;
+}
+
+static int time_cmp(void)
+{
+ int i;
+ odp_time_t *t1 = gbl_args->t1;
+ odp_time_t *t2 = gbl_args->t2;
+ int res = 0;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ res += odp_time_cmp(t1[i], t2[i]);
+
+ gbl_args->dummy += res;
+
+ return i;
+}
+
+static int time_local_res(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a1[i] = odp_time_local_res();
+
+ return i;
+}
+
+static int time_global_res(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a1[i] = odp_time_global_res();
+
+ return i;
+}
+
+static int cpu_id(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a1[i] = odp_cpu_id();
+
+ return i;
+}
+
+static int cpu_count(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a1[i] = odp_cpu_count();
+
+ return i;
+}
+
+static int cpu_hz(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a1[i] = odp_cpu_hz();
+
+ return i;
+}
+
+static int cpu_hz_max(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a1[i] = odp_cpu_hz_max();
+
+ return i;
+}
+
+static int cpu_cycles(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a1[i] = odp_cpu_cycles();
+
+ return i;
+}
+
+static int cpu_cycles_diff(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+ uint64_t *a2 = gbl_args->a2;
+ uint64_t res = 0;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ res += odp_cpu_cycles_diff(a2[i], a1[i]);
+
+ gbl_args->dummy += res;
+
+ return i;
+}
+
+static int cpu_cycles_max(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a1[i] = odp_cpu_cycles_max();
+
+ return i;
+}
+
+static int cpu_pause(void)
+{
+ int i;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ odp_cpu_pause();
+
+ return i;
+}
+
+static int thread_id(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a1[i] = odp_thread_id();
+
+ return i;
+}
+
+static int thread_count(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a1[i] = odp_thread_count();
+
+ return i;
+}
+
+static int thread_count_max(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a1[i] = odp_thread_count_max();
+
+ return i;
+}
+
+static int thread_type(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a1[i] = (int)odp_thread_type();
+
+ return i;
+}
+
+static int be_to_cpu_64(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+ uint64_t *a2 = gbl_args->a2;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a2[i] = odp_be_to_cpu_64(a1[i]);
+
+ return i;
+}
+
+static int be_to_cpu_32(void)
+{
+ int i;
+ uint32_t *b1 = gbl_args->b1;
+ uint32_t *b2 = gbl_args->b2;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ b2[i] = odp_be_to_cpu_32(b1[i]);
+
+ return i;
+}
+
+static int be_to_cpu_16(void)
+{
+ int i;
+ uint16_t *c1 = gbl_args->c1;
+ uint16_t *c2 = gbl_args->c2;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ c2[i] = odp_be_to_cpu_16(c1[i]);
+
+ return i;
+}
+
+static int cpu_to_be_64(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+ uint64_t *a2 = gbl_args->a2;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a2[i] = odp_cpu_to_be_64(a1[i]);
+
+ return i;
+}
+
+static int cpu_to_be_32(void)
+{
+ int i;
+ uint32_t *b1 = gbl_args->b1;
+ uint32_t *b2 = gbl_args->b2;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ b2[i] = odp_cpu_to_be_32(b1[i]);
+
+ return i;
+}
+
+static int cpu_to_be_16(void)
+{
+ int i;
+ uint16_t *c1 = gbl_args->c1;
+ uint16_t *c2 = gbl_args->c2;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ c2[i] = odp_cpu_to_be_16(c1[i]);
+
+ return i;
+}
+
+static int le_to_cpu_64(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+ uint64_t *a2 = gbl_args->a2;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a2[i] = odp_le_to_cpu_64(a1[i]);
+
+ return i;
+}
+
+static int le_to_cpu_32(void)
+{
+ int i;
+ uint32_t *b1 = gbl_args->b1;
+ uint32_t *b2 = gbl_args->b2;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ b2[i] = odp_le_to_cpu_32(b1[i]);
+
+ return i;
+}
+
+static int le_to_cpu_16(void)
+{
+ int i;
+ uint16_t *c1 = gbl_args->c1;
+ uint16_t *c2 = gbl_args->c2;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ c2[i] = odp_le_to_cpu_16(c1[i]);
+
+ return i;
+}
+
+static int cpu_to_le_64(void)
+{
+ int i;
+ uint64_t *a1 = gbl_args->a1;
+ uint64_t *a2 = gbl_args->a2;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ a2[i] = odp_cpu_to_le_64(a1[i]);
+
+ return i;
+}
+
+static int cpu_to_le_32(void)
+{
+ int i;
+ uint32_t *b1 = gbl_args->b1;
+ uint32_t *b2 = gbl_args->b2;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ b2[i] = odp_cpu_to_le_32(b1[i]);
+
+ return i;
+}
+
+static int cpu_to_le_16(void)
+{
+ int i;
+ uint16_t *c1 = gbl_args->c1;
+ uint16_t *c2 = gbl_args->c2;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ c2[i] = odp_cpu_to_le_16(c1[i]);
+
+ return i;
+}
+
+static int mb_release(void)
+{
+ int i;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ odp_mb_release();
+
+ return i;
+}
+
+static int mb_acquire(void)
+{
+ int i;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ odp_mb_acquire();
+
+ return i;
+}
+
+static int mb_full(void)
+{
+ int i;
+
+ for (i = 0; i < REPEAT_COUNT; i++)
+ odp_mb_full();
+
+ return i;
+}
+
+bench_info_t test_suite[] = {
+ BENCH_INFO(time_local, NULL, 0, NULL),
+ BENCH_INFO(time_local_strict, NULL, 0, NULL),
+ BENCH_INFO(time_local_ns, NULL, 0, NULL),
+ BENCH_INFO(time_local_strict_ns, NULL, 0, NULL),
+ BENCH_INFO(time_global, NULL, 0, NULL),
+ BENCH_INFO(time_global_strict, NULL, 0, NULL),
+ BENCH_INFO(time_global_ns, NULL, 0, NULL),
+ BENCH_INFO(time_global_strict_ns, NULL, 0, NULL),
+ BENCH_INFO(time_diff, init_time_global, 0, "time_diff (global)"),
+ BENCH_INFO(time_diff, init_time_local, 0, "time_diff (local)"),
+ BENCH_INFO(time_diff_ns, init_time_global, 0, NULL),
+ BENCH_INFO(time_sum, init_time_global, 0, NULL),
+ BENCH_INFO(time_to_ns, init_time_global, 0, NULL),
+ BENCH_INFO(time_local_from_ns, init_time_global, 0, NULL),
+ BENCH_INFO(time_global_from_ns, init_time_global, 0, NULL),
+ BENCH_INFO(time_cmp, init_time_global, 0, NULL),
+ BENCH_INFO(time_local_res, NULL, 0, NULL),
+ BENCH_INFO(time_global_res, NULL, 0, NULL),
+ BENCH_INFO(cpu_id, NULL, 0, NULL),
+ BENCH_INFO(cpu_count, NULL, 0, NULL),
+ BENCH_INFO(cpu_hz, NULL, 1, NULL),
+ BENCH_INFO(cpu_hz_max, NULL, 0, NULL),
+ BENCH_INFO(cpu_cycles, NULL, 0, NULL),
+ BENCH_INFO(cpu_cycles_diff, init_cpu_cycles, 0, NULL),
+ BENCH_INFO(cpu_cycles_max, NULL, 0, NULL),
+ BENCH_INFO(cpu_pause, NULL, 0, NULL),
+ BENCH_INFO(thread_id, NULL, 0, NULL),
+ BENCH_INFO(thread_count, NULL, 0, NULL),
+ BENCH_INFO(thread_count_max, NULL, 0, NULL),
+ BENCH_INFO(thread_type, NULL, 0, NULL),
+ BENCH_INFO(be_to_cpu_64, NULL, 0, NULL),
+ BENCH_INFO(be_to_cpu_32, NULL, 0, NULL),
+ BENCH_INFO(be_to_cpu_16, NULL, 0, NULL),
+ BENCH_INFO(cpu_to_be_64, NULL, 0, NULL),
+ BENCH_INFO(cpu_to_be_32, NULL, 0, NULL),
+ BENCH_INFO(cpu_to_be_16, NULL, 0, NULL),
+ BENCH_INFO(le_to_cpu_64, NULL, 0, NULL),
+ BENCH_INFO(le_to_cpu_32, NULL, 0, NULL),
+ BENCH_INFO(le_to_cpu_16, NULL, 0, NULL),
+ BENCH_INFO(cpu_to_le_64, NULL, 0, NULL),
+ BENCH_INFO(cpu_to_le_32, NULL, 0, NULL),
+ BENCH_INFO(cpu_to_le_16, NULL, 0, NULL),
+ BENCH_INFO(mb_release, NULL, 0, NULL),
+ BENCH_INFO(mb_acquire, NULL, 0, NULL),
+ BENCH_INFO(mb_full, NULL, 0, NULL),
+};
+
+/* Print usage information */
+static void usage(void)
+{
+ printf("\n"
+ "ODP miscellaneous API micro benchmarks\n"
+ "\n"
+ "Options:\n"
+ " -t, --time <opt> Time measurement. 0: measure CPU cycles (default), 1: measure time\n"
+ " -i, --index <idx> Benchmark index to run indefinitely.\n"
+ " -r, --rounds <num> Run each test case 'num' times (default %u).\n"
+ " -h, --help Display help and exit.\n\n"
+ "\n", ROUNDS);
+}
+
+/* Parse command line arguments */
+static int parse_args(int argc, char *argv[])
+{
+ int opt;
+ int long_index;
+ appl_args_t *appl_args = &gbl_args->appl;
+ static const struct option longopts[] = {
+ {"time", required_argument, NULL, 't'},
+ {"index", required_argument, NULL, 'i'},
+ {"rounds", required_argument, NULL, 'r'},
+ {"help", no_argument, NULL, 'h'},
+ {NULL, 0, NULL, 0}
+ };
+
+ static const char *shortopts = "t:i:r:h";
+
+ appl_args->time = 0; /* Measure CPU cycles */
+ appl_args->bench_idx = 0; /* Run all benchmarks */
+ appl_args->rounds = ROUNDS;
+
+ while (1) {
+ opt = getopt_long(argc, argv, shortopts, longopts, &long_index);
+
+ if (opt == -1)
+ break; /* No more options */
+
+ switch (opt) {
+ case 't':
+ appl_args->time = atoi(optarg);
+ break;
+ case 'i':
+ appl_args->bench_idx = atoi(optarg);
+ break;
+ case 'r':
+ appl_args->rounds = atoi(optarg);
+ break;
+ case 'h':
+ usage();
+ return 1;
+ default:
+ ODPH_ERR("Bad option. Use -h for help.\n");
+ return -1;
+ }
+ }
+
+ if (appl_args->rounds < 1) {
+ ODPH_ERR("Invalid test cycle repeat count: %u\n", appl_args->rounds);
+ return -1;
+ }
+
+ if (appl_args->bench_idx < 0 || appl_args->bench_idx > gbl_args->num_bench) {
+ ODPH_ERR("Bad bench index %i\n", appl_args->bench_idx);
+ return -1;
+ }
+
+ optind = 1; /* Reset 'extern optind' from the getopt lib */
+
+ return 0;
+}
+
+/* Print system and application info */
+static void print_info(void)
+{
+ odp_sys_info_print();
+
+ printf("\n"
+ "odp_bench_misc options\n"
+ "----------------------\n");
+
+ printf("CPU mask: %s\n", gbl_args->cpumask_str);
+ printf("Measurement unit: %s\n", gbl_args->appl.time ? "nsec" : "CPU cycles");
+ printf("Test rounds: %u\n", gbl_args->appl.rounds);
+ printf("\n");
+}
+
+int main(int argc, char *argv[])
+{
+ odph_helper_options_t helper_options;
+ odph_thread_t worker_thread;
+ odph_thread_common_param_t thr_common;
+ odph_thread_param_t thr_param;
+ int cpu, i;
+ odp_shm_t shm;
+ odp_cpumask_t cpumask, default_mask;
+ odp_instance_t instance;
+ odp_init_t init_param;
+ int ret = 0;
+
+ /* Let helper collect its own arguments (e.g. --odph_proc) */
+ argc = odph_parse_options(argc, argv);
+ if (odph_options(&helper_options)) {
+ ODPH_ERR("Reading ODP helper options failed\n");
+ exit(EXIT_FAILURE);
+ }
+
+ odp_init_param_init(&init_param);
+ init_param.mem_model = helper_options.mem_model;
+
+ /* Init ODP before calling anything else */
+ if (odp_init_global(&instance, &init_param, NULL)) {
+ ODPH_ERR("Global init failed\n");
+ exit(EXIT_FAILURE);
+ }
+
+ /* Init this thread */
+ if (odp_init_local(instance, ODP_THREAD_CONTROL)) {
+ ODPH_ERR("Local init failed\n");
+ exit(EXIT_FAILURE);
+ }
+
+ if (setup_sig_handler()) {
+ ODPH_ERR("Signal handler setup failed\n");
+ exit(EXIT_FAILURE);
+ }
+
+ /* Reserve memory for args from shared mem */
+ shm = odp_shm_reserve("shm_args", sizeof(gbl_args_t), ODP_CACHE_LINE_SIZE, 0);
+ if (shm == ODP_SHM_INVALID) {
+ ODPH_ERR("Shared mem reserve failed\n");
+ exit(EXIT_FAILURE);
+ }
+
+ gbl_args = odp_shm_addr(shm);
+ if (gbl_args == NULL) {
+ ODPH_ERR("Shared mem alloc failed\n");
+ exit(EXIT_FAILURE);
+ }
+
+ memset(gbl_args, 0, sizeof(gbl_args_t));
+ odp_atomic_init_u32(&gbl_args->exit_thread, 0);
+
+ gbl_args->bench = test_suite;
+ gbl_args->num_bench = sizeof(test_suite) / sizeof(test_suite[0]);
+
+ for (i = 0; i < REPEAT_COUNT; i++) {
+ gbl_args->t1[i] = ODP_TIME_NULL;
+ gbl_args->t2[i] = ODP_TIME_NULL;
+ gbl_args->t3[i] = ODP_TIME_NULL;
+ gbl_args->a1[i] = i;
+ gbl_args->a2[i] = i;
+ gbl_args->b1[i] = i;
+ gbl_args->b2[i] = i;
+ gbl_args->c1[i] = i;
+ gbl_args->c2[i] = i;
+ }
+
+ /* Parse and store the application arguments */
+ ret = parse_args(argc, argv);
+ if (ret)
+ goto exit;
+
+ /* Get default worker cpumask */
+ if (odp_cpumask_default_worker(&default_mask, 1) != 1) {
+ ODPH_ERR("Unable to allocate worker thread\n");
+ ret = -1;
+ goto exit;
+ }
+
+ (void)odp_cpumask_to_str(&default_mask, gbl_args->cpumask_str,
+ sizeof(gbl_args->cpumask_str));
+
+ print_info();
+
+ memset(&worker_thread, 0, sizeof(odph_thread_t));
+
+ /* Create worker thread */
+ cpu = odp_cpumask_first(&default_mask);
+
+ odp_cpumask_zero(&cpumask);
+ odp_cpumask_set(&cpumask, cpu);
+
+ odph_thread_common_param_init(&thr_common);
+ thr_common.instance = instance;
+ thr_common.cpumask = &cpumask;
+ thr_common.share_param = 1;
+
+ odph_thread_param_init(&thr_param);
+ thr_param.start = run_benchmarks;
+ thr_param.arg = gbl_args;
+ thr_param.thr_type = ODP_THREAD_WORKER;
+
+ odph_thread_create(&worker_thread, &thr_common, &thr_param, 1);
+
+ odph_thread_join(&worker_thread, 1);
+
+ ret = gbl_args->bench_failed;
+
+exit:
+ if (odp_shm_free(shm)) {
+ ODPH_ERR("Shared mem free failed\n");
+ exit(EXIT_FAILURE);
+ }
+
+ if (odp_term_local()) {
+ ODPH_ERR("Local term failed\n");
+ exit(EXIT_FAILURE);
+ }
+
+ if (odp_term_global(instance)) {
+ ODPH_ERR("Global term failed\n");
+ exit(EXIT_FAILURE);
+ }
+
+ if (ret < 0)
+ return EXIT_FAILURE;
+
+ return EXIT_SUCCESS;
+}
diff --git a/test/performance/odp_bench_packet.c b/test/performance/odp_bench_packet.c
index b44e92b72..aaf725b57 100644
--- a/test/performance/odp_bench_packet.c
+++ b/test/performance/odp_bench_packet.c
@@ -275,7 +275,7 @@ static int run_benchmarks(void *arg)
printf("odp_%-26s: %8.1f\n", desc, cycles);
j++;
- k = 0;
+ k = 1;
tot_cycles = 0;
}
}
diff --git a/test/performance/odp_crypto.c b/test/performance/odp_crypto.c
index b381fde31..2f79ae644 100644
--- a/test/performance/odp_crypto.c
+++ b/test/performance/odp_crypto.c
@@ -810,6 +810,8 @@ run_measure_one(crypto_args_t *cargs,
}
packets_sent += rc;
} else {
+ odp_crypto_packet_result_t result;
+
rc = odp_crypto_op(&pkt, &out_pkt,
&params, 1);
if (rc <= 0) {
@@ -820,6 +822,12 @@ run_measure_one(crypto_args_t *cargs,
}
packets_sent += rc;
packets_received++;
+ if (odp_unlikely(odp_crypto_result(&result, out_pkt) != 0) ||
+ odp_unlikely(!result.ok)) {
+ ODPH_ERR("Crypto operation failed\n");
+ odp_packet_free(out_pkt);
+ return -1;
+ }
if (cargs->debug_packets) {
mem = odp_packet_data(out_pkt);
print_mem("Immediately encrypted "
@@ -849,8 +857,12 @@ run_measure_one(crypto_args_t *cargs,
while (ev != ODP_EVENT_INVALID) {
out_pkt = odp_crypto_packet_from_event(ev);
- odp_crypto_result(&result, out_pkt);
-
+ if (odp_unlikely(odp_crypto_result(&result, out_pkt) != 0) ||
+ odp_unlikely(!result.ok)) {
+ ODPH_ERR("Crypto operation failed\n");
+ odp_packet_free(out_pkt);
+ return -1;
+ }
if (cargs->debug_packets) {
mem = odp_packet_data(out_pkt);
print_mem("Received encrypted packet",
diff --git a/test/performance/odp_ipsec.c b/test/performance/odp_ipsec.c
index c464ba0d7..50e26b1c9 100644
--- a/test/performance/odp_ipsec.c
+++ b/test/performance/odp_ipsec.c
@@ -862,7 +862,7 @@ run_measure_one_async(ipsec_args_t *cargs,
packets_sent += rc;
} else {
odp_packet_t pkt_out[max_in_flight];
- uint32_t i = 0;
+ int i = 0;
/*
* Dequeue packets until we can enqueue the next burst
@@ -895,7 +895,7 @@ run_measure_one_async(ipsec_args_t *cargs,
}
debug_packets(debug, pkt_out, i);
- if (i)
+ if (i > 0)
odp_packet_free_sp(pkt_out, i);
}
}
diff --git a/test/performance/odp_sched_perf.c b/test/performance/odp_sched_perf.c
index cd1afdf60..a97686c12 100644
--- a/test/performance/odp_sched_perf.c
+++ b/test/performance/odp_sched_perf.c
@@ -1375,6 +1375,9 @@ int main(int argc, char **argv)
if (create_queues(global))
return -1;
+ if (global->test_options.verbose)
+ odp_shm_print_all();
+
/* Start workers */
start_workers(global, instance);
diff --git a/test/performance/odp_sched_pktio.c b/test/performance/odp_sched_pktio.c
index 11be9d367..5cc07d010 100644
--- a/test/performance/odp_sched_pktio.c
+++ b/test/performance/odp_sched_pktio.c
@@ -173,7 +173,7 @@ static inline void send_packets(test_global_t *test_global,
drop = num_pkt - sent;
- if (odp_unlikely(drop))
+ if (odp_unlikely(drop > 0))
odp_packet_free_multi(&pkt[sent], drop);
if (odp_unlikely(test_global->opt.collect_stat)) {
@@ -425,6 +425,7 @@ static int worker_thread_timers(void *arg)
pktin_queue_context_t *queue_context;
odp_timer_t timer;
odp_timer_set_t ret;
+ odp_timer_start_t start_param;
worker_arg_t *worker_arg = arg;
test_global_t *test_global = worker_arg->test_global_ptr;
int worker_id = worker_arg->worker_id;
@@ -460,12 +461,17 @@ static int worker_thread_timers(void *arg)
src_pktio = queue_context->src_pktio;
src_queue = queue_context->src_queue;
timer = test_global->timer.timer[src_pktio][src_queue];
+ start_param.tick_type = ODP_TIMER_TICK_REL;
+ start_param.tick = tick;
+ start_param.tmo_ev = ODP_EVENT_INVALID;
for (i = 0; i < num; i++) {
if (odp_unlikely(odp_event_type(ev[i]) ==
ODP_EVENT_TIMEOUT)) {
tmos++;
- ret = odp_timer_set_rel(timer, tick, &ev[i]);
+
+ start_param.tmo_ev = ev[i];
+ ret = odp_timer_start(timer, &start_param);
if (odp_unlikely(ret != ODP_TIMER_SUCCESS)) {
/* Should never happen. Timeout event
@@ -487,7 +493,7 @@ static int worker_thread_timers(void *arg)
if (tmos == 0) {
/* Reset timer with existing timeout event */
- ret = odp_timer_set_rel(timer, tick, NULL);
+ ret = odp_timer_restart(timer, &start_param);
if (odp_unlikely(ret != ODP_TIMER_SUCCESS &&
ret != ODP_TIMER_FAIL)) {
@@ -1318,10 +1324,10 @@ static int create_timers(test_global_t *test_global)
static int start_timers(test_global_t *test_global)
{
int i, j;
- odp_event_t event;
odp_timeout_t timeout;
odp_timer_t timer;
odp_timer_set_t ret;
+ odp_timer_start_t start_param;
uint64_t timeout_tick = test_global->timer.timeout_tick;
int num_pktio = test_global->opt.num_pktio;
int num_queue = test_global->opt.num_pktio_queue;
@@ -1335,6 +1341,9 @@ static int start_timers(test_global_t *test_global)
timeout_tick += odp_timer_ns_to_tick(test_global->timer.timer_pool,
TIMEOUT_OFFSET_NS);
+ start_param.tick_type = ODP_TIMER_TICK_REL;
+ start_param.tick = timeout_tick;
+
for (i = 0; i < num_pktio; i++) {
for (j = 0; j < num_queue; j++) {
timer = test_global->timer.timer[i][j];
@@ -1345,10 +1354,9 @@ static int start_timers(test_global_t *test_global)
return -1;
}
- event = odp_timeout_to_event(timeout);
-
- ret = odp_timer_set_rel(timer, timeout_tick, &event);
+ start_param.tmo_ev = odp_timeout_to_event(timeout);
+ ret = odp_timer_start(timer, &start_param);
if (ret != ODP_TIMER_SUCCESS) {
printf("Timer set failed\n");
return -1;
diff --git a/test/performance/odp_timer_perf.c b/test/performance/odp_timer_perf.c
index 998db628e..ccea30a14 100644
--- a/test/performance/odp_timer_perf.c
+++ b/test/performance/odp_timer_perf.c
@@ -438,6 +438,7 @@ static int set_timers(test_global_t *global)
odp_event_t ev;
int status;
timer_ctx_t *ctx = &global->timer_ctx[i][j];
+ odp_timer_start_t start_param;
/* Set timers backwards, the last timer is set first */
if (j == 0)
@@ -456,9 +457,11 @@ static int set_timers(test_global_t *global)
tick_ns = odp_timer_ns_to_tick(tp, nsec);
nsec = nsec - period_ns;
- status = odp_timer_set_abs(timer, tick_cur + tick_ns,
- &ev);
+ start_param.tick_type = ODP_TIMER_TICK_ABS;
+ start_param.tick = tick_cur + tick_ns;
+ start_param.tmo_ev = ev;
+ status = odp_timer_start(timer, &start_param);
if (status != ODP_TIMER_SUCCESS) {
ODPH_ERR("Timer set %i/%i (ret %i)\n", i, j, status);
return -1;
@@ -656,6 +659,7 @@ static int set_cancel_mode_worker(void *arg)
odp_timer_t timer;
odp_timer_pool_t tp;
odp_timeout_t tmo;
+ odp_timer_start_t start_param;
thread_arg_t *thread_arg = arg;
test_global_t *global = thread_arg->global;
test_options_t *test_options = &global->test_options;
@@ -694,7 +698,11 @@ static int set_cancel_mode_worker(void *arg)
period_tick = global->timer_pool[i].period_tick;
tick = start_tick + j * period_tick;
- status = odp_timer_set_rel(timer, tick, &ev);
+ start_param.tick_type = ODP_TIMER_TICK_REL;
+ start_param.tick = tick;
+ start_param.tmo_ev = ev;
+
+ status = odp_timer_start(timer, &start_param);
num_tmo++;
num_set++;
@@ -746,7 +754,11 @@ static int set_cancel_mode_worker(void *arg)
if (status < 0)
continue;
- status = odp_timer_set_abs(timer, tick + j * period_tick, &ev);
+ start_param.tick_type = ODP_TIMER_TICK_ABS;
+ start_param.tick = tick + j * period_tick;
+ start_param.tmo_ev = ev;
+
+ status = odp_timer_start(timer, &start_param);
num_set++;
if (status != ODP_TIMER_SUCCESS) {
diff --git a/test/validation/api/classification/odp_classification_common.c b/test/validation/api/classification/odp_classification_common.c
index 4c24099a5..fe7677044 100644
--- a/test/validation/api/classification/odp_classification_common.c
+++ b/test/validation/api/classification/odp_classification_common.c
@@ -399,10 +399,10 @@ odp_packet_t create_packet(cls_packet_info_t pkt_info)
vlan_hdr->type = odp_cpu_to_be_16(ODPH_ETHTYPE_VLAN);
vlan_hdr++;
} else {
+ odp_packet_has_vlan_set(pkt, 1);
ethhdr->type = odp_cpu_to_be_16(ODPH_ETHTYPE_VLAN);
}
/* Default vlan header */
- odp_packet_has_vlan_set(pkt, 1);
vlan_hdr->tci = odp_cpu_to_be_16(0);
vlan_hdr->type = odp_cpu_to_be_16(eth_type);
} else {
diff --git a/test/validation/api/crypto/Makefile.am b/test/validation/api/crypto/Makefile.am
index cc4d49d60..e438ec8ca 100644
--- a/test/validation/api/crypto/Makefile.am
+++ b/test/validation/api/crypto/Makefile.am
@@ -5,3 +5,4 @@ crypto_main_SOURCES = \
odp_crypto_test_inp.c \
test_vectors.h \
test_vectors_len.h
+PRELDADD += $(LIBPACKET_COMMON)
diff --git a/test/validation/api/crypto/odp_crypto_test_inp.c b/test/validation/api/crypto/odp_crypto_test_inp.c
index 10f1b5ee2..7d5e50c71 100644
--- a/test/validation/api/crypto/odp_crypto_test_inp.c
+++ b/test/validation/api/crypto/odp_crypto_test_inp.c
@@ -1,17 +1,20 @@
/* Copyright (c) 2014-2018, Linaro Limited
- * Copyright (c) 2021-2022, Nokia
+ * Copyright (c) 2021-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
+#include <string.h>
#include <odp_api.h>
#include <odp/helper/odph_api.h>
#include <odp_cunit_common.h>
+#include <packet_common.h>
#include "test_vectors.h"
#define PKT_POOL_NUM 64
#define PKT_POOL_LEN (1 * 1024)
+#define UAREA_SIZE 8
#define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0]))
@@ -35,6 +38,7 @@ static void test_defaults(uint8_t fill)
odp_crypto_session_param_init(&param);
CU_ASSERT_EQUAL(param.op, ODP_CRYPTO_OP_ENCODE);
+ CU_ASSERT_EQUAL(param.op_type, ODP_CRYPTO_OP_TYPE_LEGACY);
CU_ASSERT_EQUAL(param.auth_cipher_text, false);
#if ODP_DEPRECATED_API
CU_ASSERT_EQUAL(param.pref_mode, ODP_CRYPTO_SYNC);
@@ -45,13 +49,6 @@ static void test_defaults(uint8_t fill)
CU_ASSERT_EQUAL(param.auth_alg, ODP_AUTH_ALG_NULL);
CU_ASSERT_EQUAL(param.auth_iv_len, 0);
CU_ASSERT_EQUAL(param.auth_aad_len, 0);
-
-#if ODP_DEPRECATED_API
- CU_ASSERT_EQUAL(param.cipher_iv.data, NULL);
- CU_ASSERT_EQUAL(param.cipher_iv.length, 0);
- CU_ASSERT_EQUAL(param.auth_iv.data, NULL);
- CU_ASSERT_EQUAL(param.auth_iv.length, 0);
-#endif
}
static void test_default_values(void)
@@ -60,53 +57,6 @@ static void test_default_values(void)
test_defaults(0xff);
}
-static int packet_cmp_mem_bits(odp_packet_t pkt, uint32_t offset,
- uint8_t *s, uint32_t len)
-{
- int rc = -1;
- uint32_t len_bytes = ((len + 7) / 8);
- uint8_t leftover_bits = len % 8;
- uint8_t buf[len_bytes];
-
- odp_packet_copy_to_mem(pkt, offset, len_bytes, buf);
-
- /* Compare till the last full byte */
- rc = memcmp(buf, s, leftover_bits ? len_bytes - 1 : len_bytes);
-
- if (rc == 0 && leftover_bits) {
- /* Do masked comparison for the leftover bits */
- uint8_t mask = 0xff << (8 - leftover_bits);
-
- rc = !((mask & buf[len_bytes - 1]) ==
- (mask & s[len_bytes - 1]));
- }
-
- return rc;
-}
-
-static int packet_cmp_mem_bytes(odp_packet_t pkt, uint32_t offset,
- uint8_t *s, uint32_t len)
-{
- uint8_t buf[len];
-
- odp_packet_copy_to_mem(pkt, offset, len, buf);
-
- return memcmp(buf, s, len);
-}
-
-static int packet_cmp_mem(odp_packet_t pkt, uint32_t offset,
- uint8_t *s, uint32_t len, odp_bool_t bit_mode)
-{
- int rc = -1;
-
- if (bit_mode)
- rc = packet_cmp_mem_bits(pkt, offset, s, len);
- else
- rc = packet_cmp_mem_bytes(pkt, offset, s, len);
-
- return rc;
-}
-
static const char *auth_alg_name(odp_auth_alg_t auth)
{
switch (auth) {
@@ -283,9 +233,12 @@ static int alg_op(odp_packet_t pkt,
}
#endif
-static int alg_packet_op(odp_packet_t pkt,
+static int alg_packet_op(odp_packet_t pkt_in,
+ odp_packet_t *pkt_out,
odp_bool_t *ok,
odp_crypto_session_t session,
+ odp_crypto_op_type_t op_type,
+ int32_t oop_shift,
uint8_t *cipher_iv_ptr,
uint8_t *auth_iv_ptr,
odp_packet_data_range_t *cipher_range,
@@ -298,7 +251,13 @@ static int alg_packet_op(odp_packet_t pkt,
odp_crypto_packet_result_t result;
odp_crypto_packet_op_param_t op_params;
odp_event_subtype_t subtype;
- odp_packet_t out_pkt = pkt;
+ odp_packet_t orig_pkt_out;
+
+ if (op_type == ODP_CRYPTO_OP_TYPE_LEGACY)
+ *pkt_out = pkt_in;
+ else if (op_type == ODP_CRYPTO_OP_TYPE_BASIC)
+ *pkt_out = ODP_PACKET_INVALID;
+ orig_pkt_out = *pkt_out;
/* Prepare input/output params */
memset(&op_params, 0, sizeof(op_params));
@@ -306,6 +265,7 @@ static int alg_packet_op(odp_packet_t pkt,
op_params.cipher_range = *cipher_range;
op_params.auth_range = *auth_range;
+ op_params.dst_offset_shift = oop_shift;
if (cipher_iv_ptr)
op_params.cipher_iv_ptr = cipher_iv_ptr;
if (auth_iv_ptr)
@@ -316,13 +276,18 @@ static int alg_packet_op(odp_packet_t pkt,
op_params.hash_result_offset = hash_result_offset;
if (suite_context.op_mode == ODP_CRYPTO_SYNC) {
- rc = odp_crypto_op(&pkt, &out_pkt, &op_params, 1);
+ rc = odp_crypto_op(&pkt_in, pkt_out, &op_params, 1);
if (rc <= 0) {
CU_FAIL("Failed odp_crypto_packet_op()");
return rc;
}
} else {
- rc = odp_crypto_op_enq(&pkt, &pkt, &op_params, 1);
+ odp_packet_t *out_param = pkt_out;
+
+ if (op_type == ODP_CRYPTO_OP_TYPE_BASIC)
+ out_param = NULL;
+
+ rc = odp_crypto_op_enq(&pkt_in, out_param, &op_params, 1);
if (rc <= 0) {
CU_FAIL("Failed odp_crypto_op_enq()");
return rc;
@@ -339,33 +304,42 @@ static int alg_packet_op(odp_packet_t pkt,
CU_ASSERT(ODP_EVENT_PACKET == odp_event_types(event, &subtype));
CU_ASSERT(ODP_EVENT_PACKET_CRYPTO == subtype);
- pkt = odp_crypto_packet_from_event(event);
+ *pkt_out = odp_crypto_packet_from_event(event);
}
- CU_ASSERT(out_pkt == pkt);
+ if (op_type != ODP_CRYPTO_OP_TYPE_BASIC)
+ CU_ASSERT(*pkt_out == orig_pkt_out);
CU_ASSERT(ODP_EVENT_PACKET ==
- odp_event_type(odp_packet_to_event(pkt)));
+ odp_event_type(odp_packet_to_event(*pkt_out)));
CU_ASSERT(ODP_EVENT_PACKET_CRYPTO ==
- odp_event_subtype(odp_packet_to_event(pkt)));
+ odp_event_subtype(odp_packet_to_event(*pkt_out)));
CU_ASSERT(ODP_EVENT_PACKET ==
- odp_event_types(odp_packet_to_event(pkt), &subtype));
+ odp_event_types(odp_packet_to_event(*pkt_out), &subtype));
CU_ASSERT(ODP_EVENT_PACKET_CRYPTO == subtype);
- CU_ASSERT(odp_packet_subtype(pkt) == ODP_EVENT_PACKET_CRYPTO);
+ CU_ASSERT(odp_packet_subtype(*pkt_out) == ODP_EVENT_PACKET_CRYPTO);
- rc = odp_crypto_result(&result, pkt);
+ rc = odp_crypto_result(&result, *pkt_out);
if (rc < 0) {
CU_FAIL("Failed odp_crypto_packet_result()");
return rc;
}
+ CU_ASSERT(rc == 0);
+
+ if (op_type == ODP_CRYPTO_OP_TYPE_OOP &&
+ suite_context.op_mode == ODP_CRYPTO_ASYNC)
+ CU_ASSERT(result.pkt_in == pkt_in);
*ok = result.ok;
return 0;
}
-static int crypto_op(odp_packet_t pkt,
+static int crypto_op(odp_packet_t pkt_in,
+ odp_packet_t *pkt_out,
odp_bool_t *ok,
odp_crypto_session_t session,
+ odp_crypto_op_type_t op_type,
+ int32_t oop_shift,
uint8_t *cipher_iv,
uint8_t *auth_iv,
odp_packet_data_range_t *cipher_range,
@@ -375,23 +349,26 @@ static int crypto_op(odp_packet_t pkt,
{
int rc;
- if (!suite_context.packet)
+ if (!suite_context.packet) {
#if ODP_DEPRECATED_API
- rc = alg_op(pkt, ok, session,
+ rc = alg_op(pkt_in, ok, session,
cipher_iv, auth_iv,
cipher_range, auth_range,
aad, hash_result_offset);
+ *pkt_out = pkt_in;
#else
rc = -1;
#endif
- else
- rc = alg_packet_op(pkt, ok, session,
+ } else {
+ rc = alg_packet_op(pkt_in, pkt_out, ok, session,
+ op_type, oop_shift,
cipher_iv, auth_iv,
cipher_range, auth_range,
aad, hash_result_offset);
+ }
if (rc < 0)
- odp_packet_free(pkt);
+ odp_packet_free(pkt_in);
return rc;
}
@@ -434,12 +411,8 @@ static void fill_with_pattern(uint8_t *buf, uint32_t len)
buf[n] = n;
}
-/*
- * Generate or verify header and trailer bytes
- */
-static void do_header_and_trailer(odp_packet_t pkt,
- uint32_t header_len, uint32_t trailer_len,
- odp_bool_t check)
+static void write_header_and_trailer(odp_packet_t pkt,
+ uint32_t header_len, uint32_t trailer_len)
{
uint32_t trailer_offset = odp_packet_len(pkt) - trailer_len;
uint32_t max_len = header_len > trailer_len ? header_len : trailer_len;
@@ -448,172 +421,470 @@ static void do_header_and_trailer(odp_packet_t pkt,
fill_with_pattern(buffer, sizeof(buffer));
- if (check) {
- CU_ASSERT(!packet_cmp_mem_bytes(pkt, 0,
- buffer, header_len));
- CU_ASSERT(!packet_cmp_mem_bytes(pkt, trailer_offset,
- buffer, trailer_len));
- } else {
- rc = odp_packet_copy_from_mem(pkt, 0,
- header_len, buffer);
- CU_ASSERT(rc == 0);
- rc = odp_packet_copy_from_mem(pkt, trailer_offset,
- trailer_len, buffer);
- CU_ASSERT(rc == 0);
- }
+ rc = odp_packet_copy_from_mem(pkt, 0, header_len, buffer);
+ CU_ASSERT(rc == 0);
+ rc = odp_packet_copy_from_mem(pkt, trailer_offset, trailer_len, buffer);
+ CU_ASSERT(rc == 0);
}
-typedef enum crypto_test {
- NORMAL_TEST = 0, /**< Plain execution */
- REPEAT_TEST, /**< Rerun without reinitializing the session */
- WRONG_DIGEST_TEST, /**< Check against wrong digest */
- MAX_TEST, /**< Final mark */
-} crypto_test;
-
typedef struct alg_test_param_t {
odp_crypto_session_t session;
odp_crypto_op_t op;
+ odp_crypto_op_type_t op_type;
+ int32_t oop_shift;
odp_cipher_alg_t cipher_alg;
odp_auth_alg_t auth_alg;
crypto_test_reference_t *ref;
uint32_t digest_offset;
- odp_bool_t override_iv;
odp_bool_t is_bit_mode_cipher;
odp_bool_t is_bit_mode_auth;
odp_bool_t adjust_segmentation;
+ odp_bool_t wrong_digest;
uint32_t first_seg_len;
uint32_t header_len;
uint32_t trailer_len;
} alg_test_param_t;
-static void alg_test_execute(const alg_test_param_t *param)
+static void prepare_crypto_ranges(const alg_test_param_t *param,
+ odp_packet_data_range_t *cipher_range,
+ odp_packet_data_range_t *auth_range)
{
- odp_bool_t ok = false;
- int iteration;
- crypto_test_reference_t *ref = param->ref;
- uint32_t reflength = ref_length_in_bytes(ref);
odp_packet_data_range_t zero_range = {.offset = 0, .length = 0};
- odp_packet_data_range_t cipher_range;
- odp_packet_data_range_t auth_range;
- uint8_t *cipher_iv = param->override_iv ? ref->cipher_iv : NULL;
- uint8_t *auth_iv = param->override_iv ? ref->auth_iv : NULL;
- cipher_range.offset = param->header_len;
- cipher_range.length = reflength;
- auth_range.offset = param->header_len;
- auth_range.length = reflength;
+ cipher_range->offset = param->header_len;
+ cipher_range->length = ref_length_in_bytes(param->ref);
+ auth_range->offset = param->header_len;
+ auth_range->length = ref_length_in_bytes(param->ref);
if (param->is_bit_mode_cipher) {
- cipher_range.offset *= 8;
- cipher_range.length = ref_length_in_bits(ref);
+ cipher_range->offset *= 8;
+ cipher_range->length = ref_length_in_bits(param->ref);
}
if (param->is_bit_mode_auth) {
- auth_range.offset *= 8;
- auth_range.length = ref_length_in_bits(ref);
+ auth_range->offset *= 8;
+ auth_range->length = ref_length_in_bits(param->ref);
}
/*
* We did not check the bit mode of the null algorithms, so let's
* not pass potentially invalid ranges to them.
*/
if (param->cipher_alg == ODP_CIPHER_ALG_NULL)
- cipher_range = zero_range;
+ *cipher_range = zero_range;
if (param->auth_alg == ODP_AUTH_ALG_NULL)
- auth_range = zero_range;
- for (iteration = NORMAL_TEST; iteration < MAX_TEST; iteration++) {
- odp_packet_t pkt;
- uint32_t digest_offset = param->digest_offset;
- uint32_t pkt_len;
+ *auth_range = zero_range;
+}
- /*
- * Test detection of wrong digest value in input packet
- * only when decoding and using non-null auth algorithm.
- */
- if (iteration == WRONG_DIGEST_TEST &&
- (param->auth_alg == ODP_AUTH_ALG_NULL ||
- param->op == ODP_CRYPTO_OP_ENCODE))
- continue;
+static int prepare_input_packet(const alg_test_param_t *param,
+ odp_packet_t *pkt_in)
+{
+ crypto_test_reference_t *ref = param->ref;
+ uint32_t reflength = ref_length_in_bytes(ref);
+ odp_packet_t pkt;
+ uint32_t digest_offset = param->digest_offset;
+ uint32_t pkt_len;
- pkt_len = param->header_len + reflength + param->trailer_len;
- if (param->digest_offset == param->header_len + reflength)
- pkt_len += ref->digest_length;
+ pkt_len = param->header_len + reflength + param->trailer_len;
+ if (param->digest_offset == param->header_len + reflength)
+ pkt_len += ref->digest_length;
- pkt = odp_packet_alloc(suite_context.pool, pkt_len);
+ pkt = odp_packet_alloc(suite_context.pool, pkt_len);
- CU_ASSERT(pkt != ODP_PACKET_INVALID);
- if (pkt == ODP_PACKET_INVALID)
- continue;
+ CU_ASSERT(pkt != ODP_PACKET_INVALID);
+ if (pkt == ODP_PACKET_INVALID)
+ return -1;
- if (param->adjust_segmentation)
- adjust_segments(&pkt, param->first_seg_len);
-
- do_header_and_trailer(pkt, param->header_len, param->trailer_len, false);
-
- if (param->op == ODP_CRYPTO_OP_ENCODE) {
- odp_packet_copy_from_mem(pkt, param->header_len,
- reflength, ref->plaintext);
- } else {
- odp_packet_copy_from_mem(pkt, param->header_len,
- reflength, ref->ciphertext);
- odp_packet_copy_from_mem(pkt, digest_offset,
- ref->digest_length,
- ref->digest);
- if (iteration == WRONG_DIGEST_TEST) {
- uint8_t byte = ~ref->digest[0];
-
- odp_packet_copy_from_mem(pkt, digest_offset,
- 1, &byte);
- }
- }
+ if (param->adjust_segmentation)
+ adjust_segments(&pkt, param->first_seg_len);
- if (crypto_op(pkt, &ok, param->session,
- cipher_iv, auth_iv,
- &cipher_range, &auth_range,
- ref->aad, digest_offset))
- break;
+ write_header_and_trailer(pkt, param->header_len, param->trailer_len);
- if (iteration == WRONG_DIGEST_TEST) {
- CU_ASSERT(!ok);
- odp_packet_free(pkt);
- continue;
+ if (param->op == ODP_CRYPTO_OP_ENCODE) {
+ odp_packet_copy_from_mem(pkt, param->header_len,
+ reflength, ref->plaintext);
+ } else {
+ odp_packet_copy_from_mem(pkt, param->header_len,
+ reflength, ref->ciphertext);
+ odp_packet_copy_from_mem(pkt, digest_offset,
+ ref->digest_length,
+ ref->digest);
+ if (param->wrong_digest) {
+ uint8_t byte = ~ref->digest[0];
+
+ odp_packet_copy_from_mem(pkt, digest_offset, 1, &byte);
}
+ }
+ *pkt_in = pkt;
+ return 0;
+}
- CU_ASSERT(ok);
+static void prepare_oop_output_packet(const alg_test_param_t *param,
+ odp_packet_t *pkt_out,
+ uint32_t pkt_len)
+{
+ uint32_t reflength = ref_length_in_bytes(param->ref);
+ const uint32_t oop_extra_len = 5;
+ uint32_t trl_len;
+ uint32_t hdr_len;
+ uint32_t oop_len;
+
+ oop_len = pkt_len + param->oop_shift + oop_extra_len;
+ *pkt_out = odp_packet_alloc(suite_context.pool, oop_len);
+ CU_ASSERT_FATAL(*pkt_out != ODP_PACKET_INVALID);
+
+ uint8_t buf[oop_len];
+
+ memset(buf, 0x55, sizeof(buf));
+ odp_packet_copy_from_mem(*pkt_out, 0, sizeof(buf), buf);
+
+ hdr_len = param->header_len + param->oop_shift;
+ trl_len = oop_len - hdr_len - reflength;
+
+ write_header_and_trailer(*pkt_out, hdr_len, trl_len);
+
+ /* have different metadata than in the input packet */
+ memset(odp_packet_user_area(*pkt_out), 0xab,
+ odp_packet_user_area_size(*pkt_out));
+}
+
+static int is_packet_data_equal(odp_packet_t pkt_1, odp_packet_t pkt_2)
+{
+ uint32_t len = odp_packet_len(pkt_1);
+ uint8_t buf_1[len];
+ uint8_t buf_2[len];
+
+ if (len != odp_packet_len(pkt_2) ||
+ odp_packet_copy_to_mem(pkt_1, 0, len, buf_1) ||
+ odp_packet_copy_to_mem(pkt_2, 0, len, buf_2))
+ return 0;
+
+ return !memcmp(buf_1, buf_2, len);
+}
+
+static int is_in_range(uint32_t offs, uint32_t range_offs, uint32_t range_len)
+{
+ return offs >= range_offs && offs < range_offs + range_len;
+}
+
+#define MAX_IGNORED_RANGES 3
+
+/*
+ * Output packet parts that we ignore since they have undefined values
+ */
+typedef struct ignore_t {
+ uint32_t byte_offset; /* offset to a byte which has bits to be ignored */
+ uint32_t byte_mask; /* mask of ignored bits in the byte */
+ struct {
+ uint32_t offset;
+ uint32_t length;
+ } ranges[MAX_IGNORED_RANGES]; /* byte ranges to be ignored */
+ uint32_t num_ranges;
+} ignore_t;
+
+static void add_ignored_range(ignore_t *ign, uint32_t offs, uint32_t len)
+{
+ if (len == 0)
+ return;
+ CU_ASSERT_FATAL(ign->num_ranges < MAX_IGNORED_RANGES);
+ ign->ranges[ign->num_ranges].offset = offs;
+ ign->ranges[ign->num_ranges].length = len;
+ ign->num_ranges++;
+}
+
+static void clear_ignored_data(const ignore_t *ign, uint8_t *data, uint32_t data_len)
+{
+ CU_ASSERT_FATAL(ign->byte_offset < data_len);
+ data[ign->byte_offset] &= ~ign->byte_mask;
+
+ for (uint32_t n = 0; n < ign->num_ranges; n++) {
+ uint32_t offset = ign->ranges[n].offset;
+ uint32_t length = ign->ranges[n].length;
+
+ CU_ASSERT(offset + length <= data_len);
+ memset(data + offset, 0, length);
+ }
+}
+
+static void prepare_ignore_info(const alg_test_param_t *param,
+ uint32_t shift,
+ uint32_t cipher_offset,
+ uint32_t cipher_len,
+ uint32_t auth_offset,
+ uint32_t auth_len,
+ ignore_t *ignore)
+{
+ memset(ignore, 0, sizeof(*ignore));
- do_header_and_trailer(pkt, param->header_len, param->trailer_len, true);
-
- if (param->op == ODP_CRYPTO_OP_ENCODE) {
- CU_ASSERT(!packet_cmp_mem(pkt, param->header_len,
- ref->ciphertext,
- ref->length,
- ref->is_length_in_bits));
- CU_ASSERT(!packet_cmp_mem_bytes(pkt, digest_offset,
- ref->digest,
- ref->digest_length));
- } else {
- /*
- * Hash result in the packet is left to undefined
- * values. Restore it from the plaintext packet
- * to make the subsequent comparison work even
- * if the hash result is within the auth_range.
- */
- odp_packet_copy_from_mem(pkt, digest_offset,
- ref->digest_length,
- ref->plaintext +
- digest_offset - param->header_len);
-
- CU_ASSERT(!packet_cmp_mem(pkt, param->header_len,
- ref->plaintext,
- ref->length,
- ref->is_length_in_bits));
+ /*
+ * Leftover bits in the last byte of the cipher range of bit mode
+ * ciphers have undefined values.
+ */
+ if (param->is_bit_mode_cipher &&
+ param->cipher_alg != ODP_CIPHER_ALG_NULL) {
+ uint8_t leftover_bits = param->ref->length % 8;
+
+ ignore->byte_offset = cipher_offset + cipher_len - 1 + shift;
+ ignore->byte_mask = ~(0xff << (8 - leftover_bits));
+ }
+
+ /*
+ * In decode sessions the bytes in the hash location have
+ * undefined values.
+ */
+ if (param->auth_alg != ODP_AUTH_ALG_NULL &&
+ param->op == ODP_CRYPTO_OP_DECODE) {
+ uint32_t offs = param->digest_offset;
+
+ if (param->op_type != ODP_CRYPTO_OP_TYPE_OOP ||
+ is_in_range(offs, cipher_offset, cipher_len) ||
+ is_in_range(offs, auth_offset, auth_len)) {
+ add_ignored_range(ignore,
+ param->digest_offset + shift,
+ param->ref->digest_length);
}
+ }
+
+ /* Decrypted bytes are undefined if authentication fails. */
+ if (param->op == ODP_CRYPTO_OP_DECODE &&
+ param->wrong_digest) {
+ add_ignored_range(ignore, cipher_offset + shift, cipher_len);
+ /* In OOP case, auth range may not get copied */
+ if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP)
+ add_ignored_range(ignore, auth_offset + shift, auth_len);
+ }
+}
+
+/* Add room for bytes that are not included in ref->length */
+#define MAX_EXP_DATA_LEN (MAX_DATA_LEN + 200)
+
+/*
+ * Expected packet data
+ */
+typedef struct expected_t {
+ uint8_t data[MAX_EXP_DATA_LEN];
+ uint32_t len;
+ ignore_t ignore;
+} expected_t;
+
+static void prepare_expected_data(const alg_test_param_t *param,
+ const odp_packet_data_range_t *cipher_range,
+ const odp_packet_data_range_t *auth_range,
+ odp_packet_t pkt_in,
+ odp_packet_t pkt_out,
+ expected_t *ex)
+{
+ uint32_t digest_offset = param->digest_offset;
+ uint32_t cipher_offset = cipher_range->offset;
+ uint32_t cipher_len = cipher_range->length;
+ uint32_t auth_offset = auth_range->offset;
+ uint32_t auth_len = auth_range->length;
+ const int32_t shift = param->op_type == ODP_CRYPTO_OP_TYPE_OOP ? param->oop_shift : 0;
+ const odp_packet_t base_pkt = param->op_type == ODP_CRYPTO_OP_TYPE_OOP ? pkt_out : pkt_in;
+ int rc;
+
+ if (param->op == ODP_CRYPTO_OP_ENCODE)
+ digest_offset += shift;
+
+ if (param->is_bit_mode_cipher) {
+ cipher_offset /= 8;
+ cipher_len = (cipher_len + 7) / 8;
+ }
+ if (param->is_bit_mode_auth) {
+ auth_offset /= 8;
+ auth_len = (auth_len + 7) / 8;
+ }
+ if (param->cipher_alg == ODP_CIPHER_ALG_NULL)
+ cipher_len = 0;
+ if (param->auth_alg == ODP_AUTH_ALG_NULL ||
+ param->auth_alg == ODP_AUTH_ALG_AES_GCM ||
+ param->auth_alg == ODP_AUTH_ALG_AES_CCM ||
+ param->auth_alg == ODP_AUTH_ALG_CHACHA20_POLY1305) {
+ /* auth range is ignored with null and AEAD algorithms */
+ auth_len = 0;
+ }
+
+ /* copy all data from base packet */
+ ex->len = odp_packet_len(base_pkt);
+ CU_ASSERT_FATAL(ex->len <= sizeof(ex->data));
+ rc = odp_packet_copy_to_mem(base_pkt, 0, ex->len, ex->data);
+ CU_ASSERT(rc == 0);
+
+ if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP && auth_len > 0) {
+ /* copy auth range from input packet */
+ rc = odp_packet_copy_to_mem(pkt_in, auth_offset, auth_len,
+ ex->data + auth_offset + shift);
+ CU_ASSERT(rc == 0);
+ }
+
+ if (param->op == ODP_CRYPTO_OP_ENCODE) {
+ /* copy hash first */
+ memcpy(ex->data + digest_offset,
+ param->ref->digest,
+ param->ref->digest_length);
+ /*
+ * Copy ciphertext, possibly overwriting hash.
+ * The other order (hash overwriting some cipher
+ * text) does not work in any real use case anyway.
+ */
+ memcpy(ex->data + cipher_offset + shift,
+ param->ref->ciphertext,
+ cipher_len);
+ } else {
+ memcpy(ex->data + cipher_offset + shift,
+ param->ref->plaintext,
+ cipher_len);
+ }
+
+ prepare_ignore_info(param, shift,
+ cipher_offset, cipher_len,
+ auth_offset, auth_len,
+ &ex->ignore);
+}
+
+static void print_data(const char *title, uint8_t *data, uint32_t len)
+{
+ static uint64_t limit;
+
+ if (limit++ > 20)
+ return;
+
+ printf("%s\n", title);
+ for (uint32_t n = 0; n < len ; n++) {
+ printf(" %02x", data[n]);
+ if ((n + 1) % 16 == 0)
+ printf("\n");
+ }
+ printf("\n");
+}
+
+static void check_output_packet_data(odp_packet_t pkt, expected_t *ex)
+{
+ int rc;
+ uint8_t pkt_data[ex->len];
+
+ CU_ASSERT(odp_packet_len(pkt) == ex->len);
+ rc = odp_packet_copy_to_mem(pkt, 0, ex->len, pkt_data);
+ CU_ASSERT(rc == 0);
+
+ clear_ignored_data(&ex->ignore, pkt_data, sizeof(pkt_data));
+ clear_ignored_data(&ex->ignore, ex->data, sizeof(ex->data));
+
+ if (memcmp(pkt_data, ex->data, ex->len)) {
+ CU_FAIL("packet data does not match expected data");
+ print_data("packet:", pkt_data, ex->len);
+ print_data("expected:", ex->data, ex->len);
+ }
+}
+
+static void alg_test_execute(const alg_test_param_t *param)
+{
+ odp_bool_t ok = false;
+ odp_packet_data_range_t cipher_range;
+ odp_packet_data_range_t auth_range;
+ odp_packet_t pkt;
+ odp_packet_t pkt_copy = ODP_PACKET_INVALID;
+ odp_packet_t pkt_out = ODP_PACKET_INVALID;
+ uint32_t digest_offset = param->digest_offset;
+ test_packet_md_t md_in, md_out, md_out_orig;
+ expected_t expected;
+
+ /*
+ * Test detection of wrong digest value in input packet
+ * only when decoding and using non-null auth algorithm.
+ */
+ if (param->wrong_digest &&
+ (param->auth_alg == ODP_AUTH_ALG_NULL ||
+ param->op == ODP_CRYPTO_OP_ENCODE))
+ return;
+
+ prepare_crypto_ranges(param, &cipher_range, &auth_range);
+ if (prepare_input_packet(param, &pkt))
+ return;
+
+ if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) {
+ prepare_oop_output_packet(param, &pkt_out, odp_packet_len(pkt));
+
+ pkt_copy = odp_packet_copy(pkt, suite_context.pool);
+ CU_ASSERT_FATAL(pkt_copy != ODP_PACKET_INVALID);
+ test_packet_get_md(pkt_out, &md_out_orig);
+ }
+
+ prepare_expected_data(param, &cipher_range, &auth_range,
+ pkt, pkt_out, &expected);
+
+ if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP &&
+ param->op == ODP_CRYPTO_OP_ENCODE) {
+ /*
+ * In this type of sessions digest offset is an offset to the output
+ * packet, so apply the shift.
+ */
+ digest_offset += param->oop_shift;
+ }
+
+ test_packet_set_md(pkt);
+ test_packet_get_md(pkt, &md_in);
+
+ if (crypto_op(pkt, &pkt_out, &ok, param->session,
+ param->op_type, param->oop_shift,
+ param->ref->cipher_iv,
+ param->ref->auth_iv,
+ &cipher_range, &auth_range,
+ param->ref->aad, digest_offset))
+ return;
+
+ /*
+ * API is not explicit about whether a failed crypto op
+ * sets the has_error packet flag or leaves it unchanged.
+ * Let's allow both behaviours.
+ */
+ test_packet_get_md(pkt_out, &md_out);
+ if (param->wrong_digest)
+ md_out.has_error = 0;
+
+ if (param->op_type == ODP_CRYPTO_OP_TYPE_OOP) {
+ test_packet_md_t md;
+
+ /* check that input packet has not changed */
+ CU_ASSERT(is_packet_data_equal(pkt, pkt_copy));
+ odp_packet_free(pkt_copy);
+ test_packet_get_md(pkt, &md);
+ CU_ASSERT(test_packet_is_md_equal(&md, &md_in));
odp_packet_free(pkt);
+
+ /* check that metadata of output packet has not changed */
+ CU_ASSERT(test_packet_is_md_equal(&md_out, &md_out_orig));
+ } else {
+ CU_ASSERT(test_packet_is_md_equal(&md_out, &md_in));
}
+
+ if (param->wrong_digest) {
+ CU_ASSERT(!ok);
+ } else {
+ CU_ASSERT(ok);
+ }
+ check_output_packet_data(pkt_out, &expected);
+ odp_packet_free(pkt_out);
}
-typedef enum {
- PACKET_IV,
- OLD_PACKET_IV,
- OLD_SESSION_IV,
-} iv_test_mode_t;
+static void alg_test_op(alg_test_param_t *param)
+{
+ int32_t oop_shifts[] = {0, 3, 130, -10};
+
+ for (uint32_t n = 0; n < ARRAY_SIZE(oop_shifts); n++) {
+ if (oop_shifts[n] != 0 &&
+ param->op_type != ODP_CRYPTO_OP_TYPE_OOP)
+ continue;
+ if ((int32_t)param->header_len + oop_shifts[n] < 0)
+ continue;
+ param->oop_shift = oop_shifts[n];
+
+ param->wrong_digest = false;
+ alg_test_execute(param);
+ alg_test_execute(param); /* rerun with the same parameters */
+ param->wrong_digest = true;
+ alg_test_execute(param);
+ }
+}
+
+static int oop_warning_shown;
typedef enum {
HASH_NO_OVERLAP,
@@ -621,10 +892,10 @@ typedef enum {
} hash_test_mode_t;
static odp_crypto_session_t session_create(odp_crypto_op_t op,
+ odp_crypto_op_type_t op_type,
odp_cipher_alg_t cipher_alg,
odp_auth_alg_t auth_alg,
crypto_test_reference_t *ref,
- iv_test_mode_t iv_mode,
hash_test_mode_t hash_mode)
{
odp_crypto_session_t session = ODP_CRYPTO_SESSION_INVALID;
@@ -641,23 +912,6 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op,
.data = auth_key_data,
.length = ref->auth_key_length
};
-#if ODP_DEPRECATED_API
- uint8_t cipher_iv_data[ref->cipher_iv_length];
- uint8_t auth_iv_data[ref->auth_iv_length];
- odp_crypto_iv_t cipher_iv = {
- .length = ref->cipher_iv_length
- };
- odp_crypto_iv_t auth_iv = {
- .length = ref->auth_iv_length
- };
-
- if (iv_mode == OLD_SESSION_IV) {
- memcpy(cipher_iv_data, ref->cipher_iv, ref->cipher_iv_length);
- memcpy(auth_iv_data, ref->auth_iv, ref->auth_iv_length);
- cipher_iv.data = cipher_iv_data;
- auth_iv.data = auth_iv_data;
- }
-#endif
memcpy(cipher_key_data, ref->cipher_key, ref->cipher_key_length);
memcpy(auth_key_data, ref->auth_key, ref->auth_key_length);
@@ -665,6 +919,7 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op,
/* Create a crypto session */
odp_crypto_session_param_init(&ses_params);
ses_params.op = op;
+ ses_params.op_type = op_type;
ses_params.auth_cipher_text = false;
ses_params.op_mode = suite_context.op_mode;
#if ODP_DEPRECATED_API
@@ -675,15 +930,8 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op,
ses_params.compl_queue = suite_context.queue;
ses_params.output_pool = suite_context.pool;
ses_params.cipher_key = cipher_key;
- if (iv_mode == PACKET_IV) {
- ses_params.cipher_iv_len = ref->cipher_iv_length;
- ses_params.auth_iv_len = ref->auth_iv_length;
- } else {
-#if ODP_DEPRECATED_API
- ses_params.cipher_iv = cipher_iv;
- ses_params.auth_iv = auth_iv;
-#endif
- }
+ ses_params.cipher_iv_len = ref->cipher_iv_length;
+ ses_params.auth_iv_len = ref->auth_iv_length;
ses_params.auth_key = auth_key;
ses_params.auth_digest_len = ref->digest_length;
ses_params.auth_aad_len = ref->aad_length;
@@ -701,14 +949,21 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op,
auth_alg_name(auth_alg));
return ODP_CRYPTO_SESSION_INVALID;
}
+
+ /* For now, allow out-of-place sessions not to be supported. */
+ if (rc < 0 && status == ODP_CRYPTO_SES_ERR_PARAMS &&
+ op_type == ODP_CRYPTO_OP_TYPE_OOP) {
+ if (!oop_warning_shown)
+ printf("\n Skipping ODP_CRYPTO_OP_TYPE_OOP tests\n");
+ oop_warning_shown = 1;
+ return ODP_CRYPTO_SESSION_INVALID;
+ }
+
/*
* We do not allow ODP_CRYPTO_SES_ERR_ALG_ORDER since we do
* not combine individual non-null crypto and auth algorithms
* with each other in the tests. Both orders should work when
* only one algorithm is used (i.e. the other one is null).
- *
- * We do not allow ODP_CRYPTO_SES_ERR_PARAMS until needed for
- * some ODP implementation.
*/
CU_ASSERT_FATAL(!rc);
CU_ASSERT(status == ODP_CRYPTO_SES_ERR_NONE);
@@ -721,23 +976,19 @@ static odp_crypto_session_t session_create(odp_crypto_op_t op,
*/
memset(cipher_key_data, 0, sizeof(cipher_key_data));
memset(auth_key_data, 0, sizeof(auth_key_data));
-#if ODP_DEPRECATED_API
- memset(cipher_iv_data, 0, sizeof(cipher_iv_data));
- memset(auth_iv_data, 0, sizeof(auth_iv_data));
-#endif
memset(&ses_params, 0, sizeof(ses_params));
return session;
}
-static void alg_test(odp_crypto_op_t op,
- odp_cipher_alg_t cipher_alg,
- odp_auth_alg_t auth_alg,
- crypto_test_reference_t *ref,
- uint32_t digest_offset,
- iv_test_mode_t iv_mode,
- odp_bool_t is_bit_mode_cipher,
- odp_bool_t is_bit_mode_auth)
+static void alg_test_ses(odp_crypto_op_t op,
+ odp_crypto_op_type_t op_type,
+ odp_cipher_alg_t cipher_alg,
+ odp_auth_alg_t auth_alg,
+ crypto_test_reference_t *ref,
+ uint32_t digest_offset,
+ odp_bool_t is_bit_mode_cipher,
+ odp_bool_t is_bit_mode_auth)
{
unsigned int initial_num_failures = CU_get_number_of_failures();
const uint32_t reflength = ref_length_in_bytes(ref);
@@ -749,22 +1000,22 @@ static void alg_test(odp_crypto_op_t op,
uint32_t max_shift;
alg_test_param_t test_param;
- session = session_create(op, cipher_alg, auth_alg, ref, iv_mode, hash_mode);
+ session = session_create(op, op_type, cipher_alg, auth_alg, ref, hash_mode);
if (session == ODP_CRYPTO_SESSION_INVALID)
return;
memset(&test_param, 0, sizeof(test_param));
test_param.session = session;
test_param.op = op;
+ test_param.op_type = op_type;
test_param.cipher_alg = cipher_alg;
test_param.auth_alg = auth_alg;
test_param.ref = ref;
- test_param.override_iv = (iv_mode != OLD_SESSION_IV);
test_param.is_bit_mode_cipher = is_bit_mode_cipher;
test_param.is_bit_mode_auth = is_bit_mode_auth;
test_param.digest_offset = digest_offset;
- alg_test_execute(&test_param);
+ alg_test_op(&test_param);
max_shift = reflength + ref->digest_length;
@@ -785,19 +1036,48 @@ static void alg_test(odp_crypto_op_t op,
test_param.header_len = 0;
test_param.trailer_len = 0;
test_param.digest_offset = digest_offset;
- alg_test_execute(&test_param);
+ alg_test_op(&test_param);
/* Test partial packet crypto with odd alignment. */
- test_param.header_len = 3;
+ test_param.header_len = 13;
test_param.trailer_len = 32;
test_param.digest_offset = test_param.header_len + digest_offset;
- alg_test_execute(&test_param);
+ alg_test_op(&test_param);
}
rc = odp_crypto_session_destroy(session);
CU_ASSERT(!rc);
}
+static void alg_test(odp_crypto_op_t op,
+ odp_cipher_alg_t cipher_alg,
+ odp_auth_alg_t auth_alg,
+ crypto_test_reference_t *ref,
+ uint32_t digest_offset,
+ odp_bool_t is_bit_mode_cipher,
+ odp_bool_t is_bit_mode_auth)
+{
+ odp_crypto_op_type_t op_types[] = {
+ ODP_CRYPTO_OP_TYPE_LEGACY,
+ ODP_CRYPTO_OP_TYPE_BASIC,
+ ODP_CRYPTO_OP_TYPE_OOP,
+ };
+
+ for (unsigned int n = 0; n < ARRAY_SIZE(op_types); n++) {
+ if (!suite_context.packet &&
+ op_types[n] != ODP_CRYPTO_OP_TYPE_LEGACY)
+ continue;
+ alg_test_ses(op,
+ op_types[n],
+ cipher_alg,
+ auth_alg,
+ ref,
+ digest_offset,
+ is_bit_mode_cipher,
+ is_bit_mode_auth);
+ }
+}
+
static odp_bool_t aad_len_ok(const odp_crypto_auth_capability_t *capa, uint32_t len)
{
if (len < capa->aad_len.min || len > capa->aad_len.max)
@@ -843,6 +1123,8 @@ static void check_alg(odp_crypto_op_t op,
memset(cipher_tested, 0, sizeof(cipher_tested));
memset(auth_tested, 0, sizeof(auth_tested));
+ oop_warning_shown = 0; /* allow OOP-unsupported warning again */
+
for (idx = 0; idx < count; idx++) {
int cipher_idx = -1, auth_idx = -1;
odp_bool_t bit_mode_needed = false;
@@ -908,18 +1190,8 @@ static void check_alg(odp_crypto_op_t op,
continue;
}
- /* test with per-packet IV */
alg_test(op, cipher_alg, auth_alg, &ref[idx], digest_offs,
- PACKET_IV, is_bit_mode_cipher, is_bit_mode_auth);
-#if ODP_DEPRECATED_API
- /* test with per-packet IV using the old API*/
- alg_test(op, cipher_alg, auth_alg, &ref[idx], digest_offs,
- OLD_PACKET_IV, is_bit_mode_cipher, is_bit_mode_auth);
-
- /* test with per-session IV */
- alg_test(op, cipher_alg, auth_alg, &ref[idx], digest_offs,
- OLD_SESSION_IV, is_bit_mode_cipher, is_bit_mode_auth);
-#endif
+ is_bit_mode_cipher, is_bit_mode_auth);
cipher_tested[cipher_idx] = true;
auth_tested[auth_idx] = true;
@@ -1223,12 +1495,15 @@ static int create_hash_test_reference(odp_auth_alg_t auth,
rc = odp_packet_copy_from_mem(pkt, 0, auth_bytes, ref->plaintext);
CU_ASSERT(rc == 0);
- session = session_create(ODP_CRYPTO_OP_ENCODE, ODP_CIPHER_ALG_NULL,
- auth, ref, PACKET_IV, HASH_NO_OVERLAP);
+ session = session_create(ODP_CRYPTO_OP_ENCODE,
+ ODP_CRYPTO_OP_TYPE_LEGACY,
+ ODP_CIPHER_ALG_NULL,
+ auth, ref, HASH_NO_OVERLAP);
if (session == ODP_CRYPTO_SESSION_INVALID)
return -1;
- rc = crypto_op(pkt, &ok, session, ref->cipher_iv, ref->auth_iv,
+ rc = crypto_op(pkt, &pkt, &ok, session, ODP_CRYPTO_OP_TYPE_LEGACY, 0,
+ ref->cipher_iv, ref->auth_iv,
&cipher_range, &auth_range, ref->aad, enc_digest_offset);
CU_ASSERT(rc == 0);
CU_ASSERT(ok);
@@ -1243,11 +1518,11 @@ static int create_hash_test_reference(odp_auth_alg_t auth,
/* copy the calculated digest in the ciphertext packet in ref */
rc = odp_packet_copy_to_mem(pkt, enc_digest_offset, ref->digest_length,
&ref->ciphertext[digest_offset]);
+ CU_ASSERT(rc == 0);
/* copy the calculated digest the digest field in ref */
rc = odp_packet_copy_to_mem(pkt, enc_digest_offset, ref->digest_length,
&ref->digest);
-
CU_ASSERT(rc == 0);
odp_packet_free(pkt);
@@ -1280,7 +1555,6 @@ static void test_auth_hash_in_auth_range(odp_auth_alg_t auth,
auth,
&ref,
digest_offset,
- PACKET_IV,
false,
capa->bit_mode);
@@ -1303,7 +1577,6 @@ static void test_auth_hash_in_auth_range(odp_auth_alg_t auth,
auth,
&ref,
digest_offset,
- PACKET_IV,
false,
capa->bit_mode);
}
@@ -2502,6 +2775,15 @@ static int crypto_init(odp_instance_t *inst)
params.pkt.num = PKT_POOL_NUM;
params.type = ODP_POOL_PACKET;
+ /*
+ * Let's have a user area so that we can check that its
+ * content gets copied along with other metadata when needed.
+ */
+ if (pool_capa.pkt.max_uarea_size >= UAREA_SIZE)
+ params.pkt.uarea_size = UAREA_SIZE;
+ else
+ printf("Warning: could not request packet user area\n");
+
if (pool_capa.pkt.max_seg_len &&
PKT_POOL_LEN > pool_capa.pkt.max_seg_len) {
fprintf(stderr, "Warning: small packet segment length\n");
diff --git a/test/validation/api/dma/dma.c b/test/validation/api/dma/dma.c
index 8eb75b172..f3e967193 100644
--- a/test/validation/api/dma/dma.c
+++ b/test/validation/api/dma/dma.c
@@ -19,6 +19,7 @@
#define TRAILER 10
#define MULTI 1
#define RESULT 1
+#define USER_DATA 0xdeadbeef
typedef struct global_t {
odp_dma_capability_t dma_capa;
@@ -385,12 +386,13 @@ static int do_transfer_async(odp_dma_t dma, const odp_dma_transfer_param_t *trs_
odp_dma_compl_param_t compl_param;
odp_event_t ev;
odp_dma_compl_t compl;
- int i, ret, done, dummy;
+ int i, ret, done;
+ uint32_t user_data = USER_DATA;
odp_dma_result_t result;
odp_dma_transfer_id_t transfer_id = ODP_DMA_TRANSFER_ID_INVALID;
uint64_t wait_ns = 500 * ODP_TIME_MSEC_IN_NS;
uint64_t sched_wait = odp_schedule_wait_time(wait_ns);
- void *user_ptr = &dummy;
+ void *user_ptr = &user_data;
odp_dma_compl_param_init(&compl_param);
compl_param.compl_mode = compl_mode;
@@ -452,6 +454,7 @@ static int do_transfer_async(odp_dma_t dma, const odp_dma_transfer_param_t *trs_
CU_ASSERT(done == 1);
CU_ASSERT(result.success);
CU_ASSERT(result.user_ptr == user_ptr);
+ CU_ASSERT(user_data == USER_DATA);
odp_dma_transfer_id_free(dma, transfer_id);
@@ -479,6 +482,7 @@ static int do_transfer_async(odp_dma_t dma, const odp_dma_transfer_param_t *trs_
CU_ASSERT(odp_dma_compl_result(compl, &result) == 0);
CU_ASSERT(result.success);
CU_ASSERT(result.user_ptr == user_ptr);
+ CU_ASSERT(user_data == USER_DATA);
/* Test also without result struct output */
CU_ASSERT(odp_dma_compl_result(compl, NULL) == 0);
diff --git a/test/validation/api/packet/packet.c b/test/validation/api/packet/packet.c
index 1ae973d79..c654aa055 100644
--- a/test/validation/api/packet/packet.c
+++ b/test/validation/api/packet/packet.c
@@ -1,5 +1,5 @@
/* Copyright (c) 2014-2018, Linaro Limited
- * Copyright (c) 2019-2022, Nokia
+ * Copyright (c) 2019-2023, Nokia
* Copyright (c) 2020, Marvell
* All rights reserved.
*
@@ -329,7 +329,8 @@ static int packet_suite_term(void)
return 0;
}
-static void packet_set_inflags(odp_packet_t pkt, int val)
+/* Set all non-conflicting metadata flags */
+static void packet_set_inflags_common(odp_packet_t pkt, int val)
{
odp_packet_has_l2_set(pkt, val);
odp_packet_has_l3_set(pkt, val);
@@ -339,23 +340,37 @@ static void packet_set_inflags(odp_packet_t pkt, int val)
odp_packet_has_eth_mcast_set(pkt, val);
odp_packet_has_jumbo_set(pkt, val);
odp_packet_has_vlan_set(pkt, val);
- odp_packet_has_vlan_qinq_set(pkt, val);
- odp_packet_has_arp_set(pkt, val);
odp_packet_has_ipv4_set(pkt, val);
- odp_packet_has_ipv6_set(pkt, val);
odp_packet_has_ip_bcast_set(pkt, val);
- odp_packet_has_ip_mcast_set(pkt, val);
odp_packet_has_ipfrag_set(pkt, val);
odp_packet_has_ipopt_set(pkt, val);
odp_packet_has_ipsec_set(pkt, val);
odp_packet_has_udp_set(pkt, val);
- odp_packet_has_tcp_set(pkt, val);
- odp_packet_has_sctp_set(pkt, val);
- odp_packet_has_icmp_set(pkt, val);
odp_packet_user_flag_set(pkt, val);
}
-static void packet_check_inflags(odp_packet_t pkt, int val)
+/* Check all non-conflicting metadata flags */
+static void packet_check_inflags_common(odp_packet_t pkt, int val)
+{
+ CU_ASSERT(odp_packet_has_l2(pkt) == !!val);
+ CU_ASSERT(odp_packet_has_l3(pkt) == !!val);
+ CU_ASSERT(odp_packet_has_l4(pkt) == !!val);
+ CU_ASSERT(odp_packet_has_eth(pkt) == !!val);
+ CU_ASSERT(odp_packet_has_eth_bcast(pkt) == !!val);
+ CU_ASSERT(odp_packet_has_eth_mcast(pkt) == !!val);
+ CU_ASSERT(odp_packet_has_jumbo(pkt) == !!val);
+ CU_ASSERT(odp_packet_has_vlan(pkt) == !!val);
+ CU_ASSERT(odp_packet_has_ipv4(pkt) == !!val);
+ CU_ASSERT(odp_packet_has_ip_bcast(pkt) == !!val);
+ CU_ASSERT(odp_packet_has_ipfrag(pkt) == !!val);
+ CU_ASSERT(odp_packet_has_ipopt(pkt) == !!val);
+ CU_ASSERT(odp_packet_has_ipsec(pkt) == !!val);
+ CU_ASSERT(odp_packet_has_udp(pkt) == !!val);
+ CU_ASSERT(odp_packet_user_flag(pkt) == !!val);
+}
+
+/* Check all metadata flags */
+static void packet_check_inflags_all(odp_packet_t pkt, int val)
{
CU_ASSERT(odp_packet_has_l2(pkt) == !!val);
CU_ASSERT(odp_packet_has_l3(pkt) == !!val);
@@ -418,7 +433,7 @@ static void packet_test_alloc_free(void)
CU_ASSERT(odp_packet_user_ptr(packet) == NULL);
/* Packet flags should be zero */
- packet_check_inflags(packet, 0);
+ packet_check_inflags_all(packet, 0);
/* Pool should have only one packet */
CU_ASSERT_FATAL(odp_packet_alloc(pool, packet_len)
@@ -937,10 +952,10 @@ static void packet_test_reset(void)
CU_ASSERT(odp_packet_reset(pkt, len) == 0);
CU_ASSERT(odp_packet_len(pkt) == len);
- packet_set_inflags(pkt, 1);
- packet_check_inflags(pkt, 1);
+ packet_set_inflags_common(pkt, 1);
+ packet_check_inflags_common(pkt, 1);
CU_ASSERT(odp_packet_reset(pkt, len) == 0);
- packet_check_inflags(pkt, 0);
+ packet_check_inflags_all(pkt, 0);
CU_ASSERT(odp_packet_reset(pkt, len - 1) == 0);
CU_ASSERT(odp_packet_len(pkt) == (len - 1));
@@ -1374,10 +1389,10 @@ static void packet_test_in_flags(void)
{
odp_packet_t pkt = test_packet;
- packet_set_inflags(pkt, 0);
- packet_check_inflags(pkt, 0);
- packet_set_inflags(pkt, 1);
- packet_check_inflags(pkt, 1);
+ packet_set_inflags_common(pkt, 0);
+ packet_check_inflags_common(pkt, 0);
+ packet_set_inflags_common(pkt, 1);
+ packet_check_inflags_common(pkt, 1);
TEST_INFLAG(pkt, has_l2);
TEST_INFLAG(pkt, has_l3);
@@ -1402,8 +1417,32 @@ static void packet_test_in_flags(void)
TEST_INFLAG(pkt, has_icmp);
TEST_INFLAG(pkt, user_flag);
- packet_set_inflags(pkt, 0);
- packet_check_inflags(pkt, 0);
+ packet_set_inflags_common(pkt, 0);
+ packet_check_inflags_common(pkt, 0);
+}
+
+static void packet_test_vlan_flags(void)
+{
+ odp_packet_t pkt = test_packet;
+
+ odp_packet_reset(pkt, odp_packet_len(test_packet));
+
+ CU_ASSERT(!odp_packet_has_vlan(pkt));
+ CU_ASSERT(!odp_packet_has_vlan_qinq(pkt));
+
+ odp_packet_has_vlan_qinq_set(pkt, 1);
+ CU_ASSERT(odp_packet_has_vlan(pkt));
+ CU_ASSERT(odp_packet_has_vlan_qinq(pkt));
+
+ odp_packet_has_vlan_qinq_set(pkt, 0);
+ CU_ASSERT(!odp_packet_has_vlan(pkt));
+ CU_ASSERT(!odp_packet_has_vlan_qinq(pkt));
+
+ odp_packet_has_vlan_set(pkt, 1);
+ CU_ASSERT(odp_packet_has_vlan(pkt));
+ CU_ASSERT(!odp_packet_has_vlan_qinq(pkt));
+
+ odp_packet_reset(pkt, odp_packet_len(test_packet));
}
static void packet_test_error_flags(void)
@@ -1628,15 +1667,15 @@ static void packet_test_meta_data_copy(void)
pkt = odp_packet_alloc(pool, packet_len);
CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID);
- packet_check_inflags(pkt, 0);
+ packet_check_inflags_all(pkt, 0);
CU_ASSERT(odp_packet_input(pkt) == ODP_PKTIO_INVALID);
CU_ASSERT(odp_packet_l3_offset(pkt) == ODP_PACKET_OFFSET_INVALID);
CU_ASSERT(odp_packet_l4_offset(pkt) == ODP_PACKET_OFFSET_INVALID);
CU_ASSERT(odp_packet_payload_offset(pkt) == ODP_PACKET_OFFSET_INVALID);
- packet_set_inflags(pkt, 1);
- packet_check_inflags(pkt, 1);
+ packet_set_inflags_common(pkt, 1);
+ packet_check_inflags_common(pkt, 1);
odp_packet_input_set(pkt, pktio);
odp_packet_user_ptr_set(pkt, (void *)(uintptr_t)0xdeadbeef);
@@ -3098,17 +3137,31 @@ static void packet_vector_test_alloc_free(void)
CU_ASSERT(odp_packet_vector_to_u64(pktv) !=
odp_packet_vector_to_u64(ODP_PACKET_VECTOR_INVALID));
- /* User flag should be initially zero */
+ /* Vector size and user flag should be initially zero */
+ CU_ASSERT(odp_packet_vector_size(pktv) == 0);
CU_ASSERT(odp_packet_vector_user_flag(pktv) == 0);
odp_packet_vector_user_flag_set(pktv, 1);
CU_ASSERT(odp_packet_vector_user_flag(pktv) != 0);
odp_packet_vector_user_flag_set(pktv, 0);
CU_ASSERT(odp_packet_vector_user_flag(pktv) == 0);
+ /* Included packet should not be freed by odp_packet_vector_free() */
+ pkt = odp_packet_alloc(default_pool, default_param.pkt.len);
+ CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID);
+
+ CU_ASSERT(odp_packet_vector_tbl(pktv, &pkts_tbl) == 0);
+ pkts_tbl[0] = pkt;
+ odp_packet_vector_size_set(pktv, 1);
+
/* Free with flag still set, alloc should clear it. */
odp_packet_vector_user_flag_set(pktv, 1);
odp_packet_vector_free(pktv);
+
+ /* Check that included packet is still valid */
+ CU_ASSERT(odp_packet_is_valid(pkt));
+
pktv = odp_packet_vector_alloc(pool);
+ CU_ASSERT(odp_packet_vector_size(pktv) == 0);
CU_ASSERT(odp_packet_vector_user_flag(pktv) == 0);
/* Since it was only one buffer pool, more vector packets can't be
@@ -3124,10 +3177,7 @@ static void packet_vector_test_alloc_free(void)
CU_ASSERT_FATAL(pktv != ODP_PACKET_VECTOR_INVALID);
CU_ASSERT(odp_packet_vector_size(pktv) == 0);
- /* Free packet vector using odp_event_free() */
- pkt = odp_packet_alloc(default_pool, default_param.pkt.len);
- CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID);
-
+ /* Free packet vector and included packet using odp_event_free() */
CU_ASSERT(odp_packet_vector_tbl(pktv, &pkts_tbl) == 0);
pkts_tbl[0] = pkt;
odp_packet_vector_size_set(pktv, 1);
@@ -4416,6 +4466,7 @@ odp_testinfo_t packet_suite[] = {
ODP_TEST_INFO(packet_test_layer_offsets),
ODP_TEST_INFO(packet_test_segment_last),
ODP_TEST_INFO(packet_test_in_flags),
+ ODP_TEST_INFO(packet_test_vlan_flags),
ODP_TEST_INFO(packet_test_error_flags),
ODP_TEST_INFO(packet_test_add_rem_data),
ODP_TEST_INFO(packet_test_meta_data_copy),
diff --git a/test/validation/api/pool/pool.c b/test/validation/api/pool/pool.c
index e8bf38c9d..cded05650 100644
--- a/test/validation/api/pool/pool.c
+++ b/test/validation/api/pool/pool.c
@@ -1,6 +1,6 @@
/* Copyright (c) 2014-2018, Linaro Limited
* Copyright (c) 2020, Marvell
- * Copyright (c) 2020-2022, Nokia
+ * Copyright (c) 2020-2023, Nokia
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -326,13 +326,19 @@ static void alloc_packet_vector(uint32_t cache_size)
num = 0;
for (i = 0; i < max_num; i++) {
- pkt_vec[num] = odp_packet_vector_alloc(pool);
- CU_ASSERT(pkt_vec[num] != ODP_PACKET_VECTOR_INVALID);
- CU_ASSERT(odp_packet_vector_valid(pkt_vec[num]) == 1);
- CU_ASSERT(odp_event_is_valid(odp_packet_vector_to_event(pkt_vec[num])) == 1);
+ odp_packet_vector_t pktv = odp_packet_vector_alloc(pool);
- if (pkt_vec[num] != ODP_PACKET_VECTOR_INVALID)
- num++;
+ CU_ASSERT(pktv != ODP_PACKET_VECTOR_INVALID);
+
+ if (pktv == ODP_PACKET_VECTOR_INVALID)
+ continue;
+
+ CU_ASSERT(odp_packet_vector_valid(pktv) == 1);
+ CU_ASSERT(odp_event_is_valid(odp_packet_vector_to_event(pktv)) == 1);
+ CU_ASSERT(odp_packet_vector_size(pktv) == 0);
+
+ pkt_vec[num] = pktv;
+ num++;
}
for (i = 0; i < num; i++)
diff --git a/test/validation/api/system/system.c b/test/validation/api/system/system.c
index f0cf48738..a98ebb634 100644
--- a/test/validation/api/system/system.c
+++ b/test/validation/api/system/system.c
@@ -135,6 +135,7 @@ static void system_test_cpu_cycles_resolution(void)
int i;
uint64_t res;
uint64_t c2, c1, max;
+ uint64_t test_cycles = odp_cpu_hz() / 100; /* CPU cycles in 10 msec */
max = odp_cpu_cycles_max();
@@ -144,11 +145,16 @@ static void system_test_cpu_cycles_resolution(void)
for (i = 0; i < RES_TRY_NUM; i++) {
c1 = odp_cpu_cycles();
- odp_time_wait_ns(100 * ODP_TIME_MSEC_IN_NS + i);
+ odp_time_wait_ns(10 * ODP_TIME_MSEC_IN_NS + i);
c2 = odp_cpu_cycles();
- CU_ASSERT(c1 % res == 0);
- CU_ASSERT(c2 % res == 0);
+ /* Diff may be zero with low resolution */
+ if (test_cycles && test_cycles > res) {
+ uint64_t diff = odp_cpu_cycles_diff(c2, c1);
+
+ CU_ASSERT(diff >= res);
+ }
+
}
}
@@ -185,13 +191,11 @@ static void system_test_cpu_cycles_diff(void)
tmp = c2 + (max - c1) + res;
diff = odp_cpu_cycles_diff(c2, c1);
CU_ASSERT(diff == tmp);
- CU_ASSERT(diff % res == 0);
/* no wrap, revert args */
tmp = c1 - c2;
diff = odp_cpu_cycles_diff(c1, c2);
CU_ASSERT(diff == tmp);
- CU_ASSERT(diff % res == 0);
}
static void system_test_cpu_cycles_long_period(void)
@@ -221,8 +225,6 @@ static void system_test_cpu_cycles_long_period(void)
c2 = odp_cpu_cycles();
CU_ASSERT(c2 != c1);
- CU_ASSERT(c1 % res == 0);
- CU_ASSERT(c2 % res == 0);
CU_ASSERT(c1 <= max && c2 <= max);
if (c2 > c1)
@@ -232,7 +234,6 @@ static void system_test_cpu_cycles_long_period(void)
diff = odp_cpu_cycles_diff(c2, c1);
CU_ASSERT(diff == tmp);
- CU_ASSERT(diff % res == 0);
/* wrap is detected and verified */
if (c2 < c1)
diff --git a/test/validation/api/time/time.c b/test/validation/api/time/time.c
index 45bfc8264..f285bf8e4 100644
--- a/test/validation/api/time/time.c
+++ b/test/validation/api/time/time.c
@@ -139,6 +139,7 @@ static void time_test_monotony(void)
lns_t2 = odp_time_local_ns();
gns_t2 = odp_time_global_ns();
+ count = 0;
while (count < BUSY_LOOP_CNT) {
count++;
};
diff --git a/test/validation/api/timer/timer.c b/test/validation/api/timer/timer.c
index 1c0a16349..0e3919b73 100644
--- a/test/validation/api/timer/timer.c
+++ b/test/validation/api/timer/timer.c
@@ -329,8 +329,10 @@ static void timer_test_capa(void)
CU_ASSERT_FATAL(ODP_CLOCK_SRC_0 + 4 == ODP_CLOCK_SRC_4);
CU_ASSERT_FATAL(ODP_CLOCK_SRC_0 + 5 == ODP_CLOCK_SRC_5);
CU_ASSERT_FATAL(ODP_CLOCK_SRC_5 + 1 == ODP_CLOCK_NUM_SRC);
+#if ODP_DEPRECATED_API
CU_ASSERT_FATAL(ODP_CLOCK_CPU == ODP_CLOCK_DEFAULT);
CU_ASSERT_FATAL(ODP_CLOCK_EXT == ODP_CLOCK_SRC_1);
+#endif
for (i = 0; i < ODP_CLOCK_NUM_SRC; i++) {
clk_src = ODP_CLOCK_SRC_0 + i;
@@ -666,6 +668,7 @@ static void timer_pool_max_res(void)
odp_queue_param_t queue_param;
odp_timer_pool_t tp;
odp_timer_t timer;
+ odp_timer_start_t start_param;
odp_pool_param_t pool_param;
odp_pool_t pool;
odp_queue_t queue;
@@ -729,7 +732,11 @@ static void timer_pool_max_res(void)
ev = odp_timeout_to_event(tmo);
CU_ASSERT_FATAL(ev != ODP_EVENT_INVALID);
- ret = odp_timer_set_rel(timer, tick, &ev);
+ start_param.tick_type = ODP_TIMER_TICK_REL;
+ start_param.tick = tick;
+ start_param.tmo_ev = ev;
+
+ ret = odp_timer_start(timer, &start_param);
CU_ASSERT(ret == ODP_TIMER_SUCCESS);
ev = ODP_EVENT_INVALID;
@@ -1145,6 +1152,7 @@ static void timer_test_event_type(odp_queue_type_t queue_type,
odp_queue_param_t queue_param;
odp_timer_pool_param_t timer_param;
odp_timer_pool_t timer_pool;
+ odp_timer_start_t start_param;
odp_queue_t queue;
odp_timeout_t tmo;
odp_buffer_t buf;
@@ -1231,7 +1239,11 @@ static void timer_test_event_type(odp_queue_type_t queue_type,
timer[i] = odp_timer_alloc(timer_pool, queue, user_ctx);
CU_ASSERT_FATAL(timer[i] != ODP_TIMER_INVALID);
- ret = odp_timer_set_rel(timer[i], (i + 1) * period_tick, &ev);
+ start_param.tick_type = ODP_TIMER_TICK_REL;
+ start_param.tick = (i + 1) * period_tick;
+ start_param.tmo_ev = ev;
+
+ ret = odp_timer_start(timer[i], &start_param);
if (ret == ODP_TIMER_TOO_NEAR)
ODPH_DBG("Timer set failed. Too near %i.\n", i);
@@ -1332,6 +1344,7 @@ static void timer_test_queue_type(odp_queue_type_t queue_type, int priv, int exp
odp_queue_param_t queue_param;
odp_timer_pool_param_t tparam;
odp_timer_pool_t tp;
+ odp_timer_start_t start_param;
odp_queue_t queue;
odp_timer_t tim;
int i, ret, num_tmo;
@@ -1411,7 +1424,12 @@ static void timer_test_queue_type(odp_queue_type_t queue_type, int priv, int exp
timer[i] = tim;
tick = tick_base + ((i + 1) * period_tick);
- ret = odp_timer_set_abs(tim, tick, &ev);
+
+ start_param.tick_type = ODP_TIMER_TICK_ABS;
+ start_param.tick = tick;
+ start_param.tmo_ev = ev;
+
+ ret = odp_timer_start(tim, &start_param);
target_tick[i] = tick;
target_nsec[i] = nsec_base + ((i + 1) * period_ns);
@@ -1541,12 +1559,12 @@ static void timer_test_cancel(void)
odp_queue_param_t queue_param;
odp_timer_capability_t capa;
odp_timer_pool_t tp;
+ odp_timer_start_t start_param;
odp_queue_t queue;
odp_timer_t tim;
odp_event_t ev;
odp_timeout_t tmo;
odp_timer_set_t rc;
- uint64_t tick;
int ret;
memset(&capa, 0, sizeof(capa));
@@ -1597,9 +1615,11 @@ static void timer_test_cancel(void)
if (ev == ODP_EVENT_INVALID)
CU_FAIL_FATAL("Failed to allocate timeout");
- tick = odp_timer_ns_to_tick(tp, tparam.max_tmo / 2);
+ start_param.tick_type = ODP_TIMER_TICK_REL;
+ start_param.tick = odp_timer_ns_to_tick(tp, tparam.max_tmo / 2);
+ start_param.tmo_ev = ev;
- rc = odp_timer_set_rel(tim, tick, &ev);
+ rc = odp_timer_start(tim, &start_param);
if (rc != ODP_TIMER_SUCCESS)
CU_FAIL_FATAL("Failed to set timer (relative time)");
@@ -1645,6 +1665,7 @@ static void timer_test_tmo_limit(odp_queue_type_t queue_type,
odp_queue_param_t queue_param;
odp_timer_pool_param_t timer_param;
odp_timer_pool_t timer_pool;
+ odp_timer_start_t start_param;
odp_queue_t queue;
odp_timeout_t tmo;
odp_event_t ev;
@@ -1731,7 +1752,12 @@ static void timer_test_tmo_limit(odp_queue_type_t queue_type,
CU_ASSERT(ev != ODP_EVENT_INVALID);
t1 = odp_time_local();
- ret = odp_timer_set_rel(timer[i], tmo_tick, &ev);
+
+ start_param.tick_type = ODP_TIMER_TICK_REL;
+ start_param.tick = tmo_tick;
+ start_param.tmo_ev = ev;
+
+ ret = odp_timer_start(timer[i], &start_param);
if (ret == ODP_TIMER_TOO_NEAR)
ODPH_DBG("Timer set failed. Too near %i.\n", i);
@@ -1943,6 +1969,7 @@ static int worker_entrypoint(void *arg ODP_UNUSED)
struct timespec ts;
uint32_t nstale;
odp_timer_set_t timer_rc;
+ odp_timer_start_t start_param;
odp_timer_pool_t tp = global_mem->tp;
odp_pool_t tbp = global_mem->tbp;
uint32_t num_timers = global_mem->timers_per_thread;
@@ -2008,7 +2035,13 @@ static int worker_entrypoint(void *arg ODP_UNUSED)
(rand_r(&seed) % RANGE_MS) * 1000000ULL;
tck = odp_timer_current_tick(tp) +
odp_timer_ns_to_tick(tp, nsec);
- timer_rc = odp_timer_set_abs(tt[i].tim, tck, &tt[i].ev);
+
+ start_param.tick_type = ODP_TIMER_TICK_ABS;
+ start_param.tick = tck;
+ start_param.tmo_ev = tt[i].ev;
+ tt[i].ev = ODP_EVENT_INVALID;
+
+ timer_rc = odp_timer_start(tt[i].tim, &start_param);
if (timer_rc == ODP_TIMER_TOO_NEAR) {
ODPH_ERR("Missed tick, setting timer\n");
} else if (timer_rc != ODP_TIMER_SUCCESS) {
@@ -2058,23 +2091,34 @@ static int worker_entrypoint(void *arg ODP_UNUSED)
odp_timer_set_t rc;
uint64_t cur_tick;
uint64_t tck;
+ int reset_timer = 0;
- if (tt[i].ev != ODP_EVENT_INVALID)
+ if (tt[i].ev != ODP_EVENT_INVALID) {
/* Timer inactive => set */
nset++;
- else if (odp_timer_current_tick(tp) >= tt[i].tick)
+ } else if (odp_timer_current_tick(tp) >= tt[i].tick) {
/* Timer just expired. */
goto sleep;
- else
+ } else {
/* Timer active => reset */
nreset++;
+ reset_timer = 1;
+ }
nsec = min_tmo + THREE_POINT_THREE_MSEC +
(rand_r(&seed) % RANGE_MS) * 1000000ULL;
tck = odp_timer_ns_to_tick(tp, nsec);
cur_tick = odp_timer_current_tick(tp);
- rc = odp_timer_set_rel(tt[i].tim, tck, &tt[i].ev);
+
+ start_param.tick_type = ODP_TIMER_TICK_REL;
+ start_param.tick = tck;
+ start_param.tmo_ev = tt[i].ev;
+
+ if (reset_timer)
+ rc = odp_timer_restart(tt[i].tim, &start_param);
+ else
+ rc = odp_timer_start(tt[i].tim, &start_param);
if (rc == ODP_TIMER_TOO_NEAR) {
CU_FAIL("Failed to set timer: TOO NEAR");