aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMaxim Uvarov <maxim.uvarov@linaro.org>2018-05-03 16:52:58 +0300
committerMaxim Uvarov <maxim.uvarov@linaro.org>2018-05-03 16:53:12 +0300
commit2c851db2a527ad0ed7cc08baa0665b1e2ca68dcf (patch)
treefeb32edd691a0763d7862cc313a01866879a519a
parent8800ca1e6dd3b3d595f4c82721ea5b131f23262a (diff)
parent19c1fb8b49e61de90ae4bab82d89e1dbcc3fa577 (diff)
Merge branch 'dev/cleanup' of git://github.com/matiaselo/odp-dpdkv1.19.0.0_DPDK_17.11_tigermothv1.19.0.0_DPDK_17.11tigermoth_lts
Reviewed-by: Bill Fischofer <bill.fischofer@linaro.org> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov@linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov@linaro.org>
-rw-r--r--.shippable.yml17
-rw-r--r--.travis.yml4
-rw-r--r--CHANGELOG105
-rw-r--r--DEPENDENCIES4
-rw-r--r--config/odp-linux-dpdk.conf17
-rw-r--r--config/odp-linux-generic.conf26
-rw-r--r--configure.ac4
-rw-r--r--doc/users-guide/users-guide-cls.adoc2
-rw-r--r--doc/users-guide/users-guide-crypto.adoc2
-rw-r--r--doc/users-guide/users-guide-ipsec.adoc4
-rw-r--r--doc/users-guide/users-guide-packet.adoc35
-rw-r--r--doc/users-guide/users-guide-pktio.adoc374
-rw-r--r--doc/users-guide/users-guide-tm.adoc8
-rw-r--r--doc/users-guide/users-guide.adoc68
-rwxr-xr-xexample/generator/generator_null_test.sh3
-rw-r--r--example/l2fwd/README7
-rw-r--r--example/l2fwd_simple/Makefile.am2
-rwxr-xr-xexample/l2fwd_simple/l2fwd_simple_run.sh6
-rw-r--r--example/l3fwd/Makefile.am4
-rw-r--r--example/l3fwd/empty.pcapbin0 -> 24 bytes
-rwxr-xr-xexample/l3fwd/odp_l3fwd_run.sh9
-rw-r--r--example/packet/Makefile.am2
-rwxr-xr-xexample/packet/pktio_run.sh11
-rw-r--r--example/switch/Makefile.am4
-rw-r--r--example/switch/empty.pcapbin0 -> 24 bytes
-rwxr-xr-xexample/switch/switch_run.sh10
-rw-r--r--include/odp/api/spec/packet.h3
-rw-r--r--m4/odp_dpdk.m49
-rw-r--r--m4/odp_libconfig.m417
-rw-r--r--platform/linux-dpdk/Makefile.am3
-rw-r--r--platform/linux-dpdk/README26
-rw-r--r--platform/linux-dpdk/include/odp/api/plat/std_clib_inlines.h2
-rw-r--r--platform/linux-dpdk/include/odp_packet_dpdk.h50
-rw-r--r--platform/linux-dpdk/include/odp_packet_internal.h9
-rw-r--r--platform/linux-dpdk/include/odp_packet_io_internal.h1
-rw-r--r--platform/linux-dpdk/odp_crypto.c2
-rw-r--r--platform/linux-dpdk/odp_init.c29
-rw-r--r--platform/linux-dpdk/odp_libconfig.c99
-rw-r--r--platform/linux-dpdk/odp_packet.c239
-rw-r--r--platform/linux-dpdk/odp_packet_dpdk.c62
-rw-r--r--platform/linux-dpdk/odp_pool.c3
-rw-r--r--platform/linux-dpdk/odp_queue_basic.c97
-rw-r--r--platform/linux-dpdk/odp_time.c4
-rwxr-xr-xplatform/linux-dpdk/test/validation/api/pktio/pktio_run.sh2
-rwxr-xr-xplatform/linux-dpdk/test/wrapper-script.sh19
-rw-r--r--platform/linux-generic/Makefile.am64
-rw-r--r--platform/linux-generic/include/odp_fdserver_internal.h (renamed from platform/linux-generic/include/_fdserver_internal.h)0
-rw-r--r--platform/linux-generic/include/odp_ishm_internal.h (renamed from platform/linux-generic/include/_ishm_internal.h)0
-rw-r--r--platform/linux-generic/include/odp_ishmphy_internal.h (renamed from platform/linux-generic/include/_ishmphy_internal.h)0
-rw-r--r--platform/linux-generic/include/odp_ishmpool_internal.h (renamed from platform/linux-generic/include/_ishmpool_internal.h)0
-rw-r--r--platform/linux-generic/include/odp_libconfig_internal.h5
-rw-r--r--platform/linux-generic/include/odp_packet_internal.h4
-rw-r--r--platform/linux-generic/include/odp_packet_netmap.h7
-rw-r--r--platform/linux-generic/include/odp_queue_internal.h22
-rw-r--r--platform/linux-generic/include/odp_schedule_scalable_ordered.h2
-rw-r--r--platform/linux-generic/odp_crypto.c214
-rw-r--r--platform/linux-generic/odp_fdserver.c (renamed from platform/linux-generic/_fdserver.c)77
-rw-r--r--platform/linux-generic/odp_ipsec.c106
-rw-r--r--platform/linux-generic/odp_ipsec_sad.c9
-rw-r--r--platform/linux-generic/odp_ishm.c (renamed from platform/linux-generic/_ishm.c)131
-rw-r--r--platform/linux-generic/odp_ishmphy.c (renamed from platform/linux-generic/_ishmphy.c)6
-rw-r--r--platform/linux-generic/odp_ishmpool.c (renamed from platform/linux-generic/_ishmpool.c)4
-rw-r--r--platform/linux-generic/odp_libconfig.c42
-rw-r--r--platform/linux-generic/odp_packet.c239
-rw-r--r--platform/linux-generic/odp_packet_io.c29
-rw-r--r--platform/linux-generic/odp_pool.c9
-rw-r--r--platform/linux-generic/odp_queue_basic.c120
-rw-r--r--platform/linux-generic/odp_queue_scalable.c4
-rw-r--r--platform/linux-generic/odp_schedule_basic.c209
-rw-r--r--platform/linux-generic/odp_schedule_scalable.c4
-rw-r--r--platform/linux-generic/odp_shared_memory.c2
-rw-r--r--platform/linux-generic/pktio/dpdk.c62
-rw-r--r--platform/linux-generic/pktio/ipc.c2
-rw-r--r--platform/linux-generic/pktio/loop.c110
-rw-r--r--platform/linux-generic/pktio/netmap.c63
-rwxr-xr-xscripts/build-pktio-dpdk2
-rwxr-xr-xscripts/ci-checkpatches.sh4
-rw-r--r--test/miscellaneous/odp_api_from_cpp.cpp6
-rw-r--r--test/performance/.gitignore2
-rw-r--r--test/performance/Makefile.am7
-rw-r--r--test/performance/odp_ipsec.c1170
-rwxr-xr-xtest/performance/odp_l2fwd_run.sh12
-rwxr-xr-xtest/performance/odp_pktio_ordered_run.sh11
-rw-r--r--test/performance/odp_sched_latency.c9
-rw-r--r--test/performance/odp_sched_pktio.c800
-rwxr-xr-xtest/performance/odp_sched_pktio_run.sh109
-rw-r--r--test/validation/api/crypto/odp_crypto_test_inp.c91
-rw-r--r--test/validation/api/crypto/test_vectors.h209
-rw-r--r--test/validation/api/crypto/test_vectors_len.h10
-rw-r--r--test/validation/api/ipsec/ipsec.c10
-rw-r--r--test/validation/api/ipsec/ipsec_test_out.c32
-rw-r--r--test/validation/api/ipsec/test_vectors.h71
-rw-r--r--test/validation/api/pktio/pktio.c465
93 files changed, 5279 insertions, 691 deletions
diff --git a/.shippable.yml b/.shippable.yml
index 82960a39e..53e094bcb 100644
--- a/.shippable.yml
+++ b/.shippable.yml
@@ -6,7 +6,7 @@ compiler:
env:
- CONF="--disable-test-perf --disable-test-perf-proc"
- # - CONF="--disable-abi-compat --disable-test-perf --disable-test-perf-proc"
+ - CONF="--disable-abi-compat --disable-test-perf --disable-test-perf-proc"
# - CONF="--enable-schedule-sp"
# - CONF="--enable-schedule-iquery"
# - CONF="--enable-dpdk-zero-copy"
@@ -15,11 +15,6 @@ env:
# - CROSS_ARCH="powerpc"
# - CROSS_ARCH="i386"
-matrix:
- allow_failures:
- - compiler: clang
- env: CONF="--disable-abi-compat --disable-test-perf --disable-test-perf-proc"
-
build:
pre_ci:
# use Dockerfile to install additional CI dependencies
@@ -34,17 +29,19 @@ build:
ci:
- mkdir -p $HOME/odp-shmdir
+ - export CI=true ODP_SHM_DIR=$HOME/odp-shmdir ODP_TEST_OUT_XML=yes
- ./bootstrap
- if [ "${CC#clang}" != "${CC}" ] ; then export CXX="${CC/clang/clang++}"; fi
+ - echo ./configure $CONF
- ./configure $CONF
- make -j $(nproc)
- - sudo env CI=true ODP_SHM_DIR=$HOME/odp-shmdir ODP_TEST_OUT_XML=yes ODP_SCHEDULER=basic make check
+ - ODP_SCHEDULER=basic make check
- ./scripts/shippable-post.sh basic
- - sudo env CI=true ODP_SHM_DIR=$HOME/odp-shmdir ODP_TEST_OUT_XML=yes ODP_SCHEDULER=sp make check
+ - ODP_SCHEDULER=sp make check
- ./scripts/shippable-post.sh sp
- - sudo env CI=true ODP_SHM_DIR=$HOME/odp-shmdir ODP_TEST_OUT_XML=yes ODP_SCHEDULER=iquery make check
+ - ODP_SCHEDULER=iquery make check
- ./scripts/shippable-post.sh iquery
- - sudo env CI=true ODP_SHM_DIR=$HOME/odp-shmdir ODP_TEST_OUT_XML=yes ODP_SCHEDULER=scalable make check
+ - ODP_SCHEDULER=scalable make check
- ./scripts/shippable-post.sh scalable
- rm -rf $HOME/odp-shmdir
diff --git a/.travis.yml b/.travis.yml
index 1be920d6a..5bc119644 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -257,7 +257,7 @@ script:
- echo "Dynamic link.."
- ${CC} ${CFLAGS} ${OLDPWD}/example/hello/odp_hello.c -o odp_hello_inst_dynamic `PKG_CONFIG_PATH=${HOME}/odp-install/lib/pkgconfig:${PKG_CONFIG_PATH} pkg-config --cflags --libs libodp-linux`
- if [ -z "$CROSS_ARCH" ] ; then
- sudo ODP_PLATFORM_PARAMS="-n 2" LD_LIBRARY_PATH="${HOME}/odp-install/lib:$LD_LIBRARY_PATH" ./odp_hello_inst_dynamic ;
+ sudo LD_LIBRARY_PATH="${HOME}/odp-install/lib:$LD_LIBRARY_PATH" ./odp_hello_inst_dynamic ;
fi
- |
# it is not possible to do static linking if we only have shared DPDK library. Compiler complains about missing -ldpdk
@@ -265,7 +265,7 @@ script:
echo "Static link.."
${CC} ${CFLAGS} ${OLDPWD}/example/hello/odp_hello.c -o odp_hello_inst_static `PKG_CONFIG_PATH=${HOME}/odp-install/lib/pkgconfig:${PKG_CONFIG_PATH} pkg-config --cflags --libs libodp-linux --static` -static || exit 1
if [ -z "$CROSS_ARCH" ] ; then
- sudo ODP_PLATFORM_PARAMS="-n 2" ./odp_hello_inst_static;
+ sudo ./odp_hello_inst_static;
fi
fi
- popd
diff --git a/CHANGELOG b/CHANGELOG
index 1dfdd0e7b..5af1777c6 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,108 @@
+== OpenDataPlane (1.19.0.0)
+=== Summary of Changes
+ODP v1.19.0.0 is the official Tiger Moth final release. It incorporates final
+implementation changes and bug fixes and completes the Tiger Moth ODP
+development cycle.
+
+==== APIs
+No functional changes for this release. The Tiger Moth API was frozen in ODP
+v1.18.0.0.
+
+===== API Documentation Update
+The specification for the `odp_packet_l4_chksum_status()` API has been
+clarified to reflect that in IPv4 UDP checksums are optional. As a result, a
+zero (nonexistent) checksum will be reported as `ODP_PACKET_CHKSUM_OK`.
+
+==== C++ Test Improvements
+The {cpp} test included in the ODP validation suite now uses `cout` instead
+of `printf()` to ensure that {cpp} is being used to compile it.
+
+==== Queue and Scheduler Configuration
+For the ODP Reference Implementation, The `config/odp-linux-generic.conf` file
+is extended with sections to control the default and maximum sizes for basic
+queues, and the priority spread used by the scheduler for scheduled queues.
+
+The configuration file is a template named `platform/odp-$platform.conf` so
+this can be easily inherited by other ODP implementations.
+
+==== Runtime Default `config` File Read Order Improvements
+For the ODP Reference Implementation, the default values of the
+application-provided `config` file (if used) override the values provided by
+the built-in `config/odp-linux-generic.conf` file.
+
+=== Implementation Improvements
+The `odp-linux` reference implementation is improved in a number of areas:
+
+==== Netmap Ring Configuration for VALE
+PktIO netmap support now uses the ODP config file to allow rings used for VALE
+processing to be specified. The supplied defaults provide optimal performance
+in typical settings.
+
+==== AES-XCBC-MAC and SHA384-HMAC
+These crypto/authentication algorithms are now implemented.
+
+==== Packet Checksum Validation and Insertion
+Proper packet checksum validation and insertion, in conformance with the
+relevant ODP APIs, is now provided.
+
+=== Dependency Changes
+
+==== DPDK 17.11 Support
+The Tiger Moth LTS release is synchronized with the most recent DPDK LTS
+release for DPDK pktio support.
+
+==== Removal of dependency on `xxd` package.
+This dependency is removed. The Reference Implementation build tools now use
+the standard `od` tool rather than the optional `xxd` package.
+
+=== Performance Tests
+
+==== `odp_sched_pktio`
+A new test has been added to test the performance of PktIO operations in
+scheduled mode. Scheduled PktIO is inherently more scalable and simpler from
+an application standpoint than direct (polled) I/O, but depending on the
+efficiency of the scheduler implementation can incur additional levels of
+overhead. This test can give insight into a given platform's scheduler
+efficiency. For the `odp-linux` reference implementation, this test has shown
+scheduled I/O to be within 10% of rates achievable via direct I/O, meaning
+that for many applications the simplicity and scalability of the event model
+is preferable.
+
+==== `odp_ipsec`
+A new test has been added that measures outbound (TX) IPsec performance with
+a variety of cipher and authentication algorithms.
+
+=== Example Changes
+
+==== `l2fwd` Example
+The `README` file associated with this example has been clarified to explain
+that this example is a throughput test and as a result does not preserve
+packet order under all conditions.
+
+=== Bug Fixes
+==== https://bugs.linaro.org/show_bug.cgi?id=3611[Bug 3611]
+ODP linux-generic fails on AArch64 in non-ABI-compat mode.
+
+==== https://bugs.linaro.org/show_bug.cgi?id=3657[Bug 3657]
+PktIO does not work with Mellanox Interfaces
+
+==== https://bugs.linaro.org/show_bug.cgi?id=3685[Bug 3685]
+RX UDP checksum offload drops valid UDP packets with Niantic
+
+==== https://bugs.linaro.org/show_bug.cgi?id=3686[Bug 3686]
+IP header checksum not inserted if L4 offset not set
+
+==== https://bugs.linaro.org/show_bug.cgi?id=3690[Bug 3690]
+fdserver process interferes with signal handling
+
+==== https://bugs.linaro.org/show_bug.cgi?id=3736[Bug 3736]
+return value not checked for some fdserver interface functions
+
+=== Known Issues
+
+==== https://bugs.linaro.org/show_bug.cgi?id=2988[Bug 2988]
+ODP exposes symbols outside of odp*/_odp* namespace
+
== OpenDataPlane (1.18.0.1)
=== Summary of Changes
ODP v1.18.0.1 is a fix level for Tiger Moth Release Candidate 2 (RC 2).
diff --git a/DEPENDENCIES b/DEPENDENCIES
index 2cd8ccb4c..48f5a8397 100644
--- a/DEPENDENCIES
+++ b/DEPENDENCIES
@@ -21,8 +21,6 @@ Prerequisites for building the OpenDataPlane (ODP) API
Libraries currently required to link: openssl, libatomic, libconfig
- Required tools: xxd
-
3.1 OpenSSL native compile
For native compilation, simply load the necessary libraries using the appropriate
@@ -174,7 +172,7 @@ Prerequisites for building the OpenDataPlane (ODP) API
3.4.1 Building DPDK and ODP with DPDK pktio support
- DPDK packet I/O has been tested to work with DPDK v17.08.
+ DPDK packet I/O has been tested to work with DPDK v17.11.
Follow steps in ./scripts/build-pktio-dpdk
diff --git a/config/odp-linux-dpdk.conf b/config/odp-linux-dpdk.conf
index 341dba0c5..33296080c 100644
--- a/config/odp-linux-dpdk.conf
+++ b/config/odp-linux-dpdk.conf
@@ -30,3 +30,20 @@ pktio_dpdk: {
rx_drop_en = 1
}
}
+
+queue_basic: {
+ # Maximum queue size. Value must be a power of two.
+ max_queue_size = 8192
+
+ # Default queue size. Value must be a power of two.
+ default_queue_size = 4096
+}
+
+sched_basic: {
+ # Priority level spread. Each priority level is spread into multiple
+ # scheduler internal queues. A higher spread value typically improves
+ # parallelism and thus is better for high thread counts, but causes
+ # uneven service level for low thread counts. Typically, optimal
+ # value is the number of threads using the scheduler.
+ prio_spread = 4
+}
diff --git a/config/odp-linux-generic.conf b/config/odp-linux-generic.conf
index 15e65d00f..f5f21b45f 100644
--- a/config/odp-linux-generic.conf
+++ b/config/odp-linux-generic.conf
@@ -30,3 +30,29 @@ pktio_dpdk: {
rx_drop_en = 1
}
}
+
+# netmap pktio options
+pktio_netmap: {
+ # Interface specific options
+ virt: {
+ nr_rx_slots = 0
+ nr_tx_slots = 0
+ }
+}
+
+queue_basic: {
+ # Maximum queue size. Value must be a power of two.
+ max_queue_size = 8192
+
+ # Default queue size. Value must be a power of two.
+ default_queue_size = 4096
+}
+
+sched_basic: {
+ # Priority level spread. Each priority level is spread into multiple
+ # scheduler internal queues. A higher spread value typically improves
+ # parallelism and thus is better for high thread counts, but causes
+ # uneven service level for low thread counts. Typically, optimal
+ # value is the number of threads using the scheduler.
+ prio_spread = 4
+}
diff --git a/configure.ac b/configure.ac
index 152c07953..ee87d9c6a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3,9 +3,9 @@ AC_PREREQ([2.5])
# Set correct API version
##########################################################################
m4_define([odpapi_generation_version], [1])
-m4_define([odpapi_major_version], [18])
+m4_define([odpapi_major_version], [19])
m4_define([odpapi_minor_version], [0])
-m4_define([odpapi_point_version], [1])
+m4_define([odpapi_point_version], [0])
m4_define([odpapi_version],
[odpapi_generation_version.odpapi_major_version.odpapi_minor_version.odpapi_point_version])
AC_INIT([OpenDataPlane],[odpapi_version],[lng-odp@lists.linaro.org])
diff --git a/doc/users-guide/users-guide-cls.adoc b/doc/users-guide/users-guide-cls.adoc
index a689826c7..359d225d8 100644
--- a/doc/users-guide/users-guide-cls.adoc
+++ b/doc/users-guide/users-guide-cls.adoc
@@ -7,7 +7,7 @@ prioritization, classification and scheduling of each packet, so that the
software application can run faster, scale better and adhere to QoS
requirements.
-The following API abstraction are not modelled after any existing product
+The following API abstraction are not modeled after any existing product
implementation, but is instead defined in terms of what a typical data-plane
application may require from such a platform, without sacrificing simplicity and
avoiding ambiguity. Certain terms that are being used within the context of
diff --git a/doc/users-guide/users-guide-crypto.adoc b/doc/users-guide/users-guide-crypto.adoc
index 029f47b17..b44402adf 100644
--- a/doc/users-guide/users-guide-crypto.adoc
+++ b/doc/users-guide/users-guide-crypto.adoc
@@ -175,7 +175,7 @@ any software generated pseudo-random data. May not be available on all
platforms.
These form a hierarchy with BASIC being the lowest kind of random and TRUE
-behing the highest. The main API for accessing random data is:
+being the highest. The main API for accessing random data is:
[source,c]
-----
diff --git a/doc/users-guide/users-guide-ipsec.adoc b/doc/users-guide/users-guide-ipsec.adoc
index ac4eae85d..6af676620 100644
--- a/doc/users-guide/users-guide-ipsec.adoc
+++ b/doc/users-guide/users-guide-ipsec.adoc
@@ -381,7 +381,7 @@ any further application involvement. Only if a problem arises will the packet
be returned to the application with an `odp_ipsec_packet_result_t` indicating
the nature of the problem.
-Note that while operating in inline mode, asychronous lookaside operations are
+Note that while operating in inline mode, asynchronous lookaside operations are
also permitted. This provide the application with additional flexibility if,
for example, some packets need additional handling that cannot be supported
directly with inline IPsec processing.
@@ -449,7 +449,7 @@ the application continues to receive and process IPsec events as normal.
Disable completion is indicated by the application seeing an event of type
`ODP_EVENT_IPSEC_STATUS` for this SA that contains an `odp_ipsec_status_id_t`
of `ODP_IPSEC_STATUS_SA_DISABLE`. For inbound SAs, receipt of this event means
-that the application has seen all IPsec packets associatd with this SA that
+that the application has seen all IPsec packets associated with this SA that
were pending at the time of the disable call. For outbound SAs, receipt of
this event means that the application has seen all result events associated
with packets sent via this SA.
diff --git a/doc/users-guide/users-guide-packet.adoc b/doc/users-guide/users-guide-packet.adoc
index 6d9e98a5a..c38c5c866 100644
--- a/doc/users-guide/users-guide-packet.adoc
+++ b/doc/users-guide/users-guide-packet.adoc
@@ -499,3 +499,38 @@ Note also that a packet may not reference itself, nor may circular reference
relationships be formed, _e.g.,_ packet A is used as a header for a reference
to packet B and B is used as a header for a reference to packet A. Results
are undefined if such circular references are attempted.
+
+=== Packet Parsing, Checksum Processing, and Overrides
+Packet parsing is normally triggered automatically as part of packet RX
+processing. However, the application can trigger parsing explicitly via the
+API:
+[source,c]
+-----
+int odp_packet_parse(odp_packet_t pkt, uint32_t offset,
+ const odp_packet_parse_param_t *param);
+-----
+This is typically done following packet decapsulation or other preprocessing
+that would prevent RX parsing from "seeing" the relevant portion of the
+packet. The `odp_packet_parse_param_t` struct that is passed to control the
+depth of the desired parse, as well as whether checksum validation should be
+performed as part of the parse, and if so which checksums require this
+processing.
+
+Packets containing Layer 3 (IPv4) and Layer 4 (TCP, UDP, SCTP) checksums
+can have these validated (on RX) and generated (on TX) automatically.
+This is normally controlled by the settings on the PktIOs that
+receive/transmit them, however they can also be controlled on an
+individual packet basis.
+
+Packets have associated `odp_packet_chksum_status_t` metadata that indicates
+the state any checksums contained in that packet. These can be queried via
+the APIs `odp_packet_l3_chksum_status()` and `odp_packet_l4_chksum_status()`,
+respectively. Checksums can either be known good, known bad, or unknown, where
+unknown means that checksum validation processing has not occurred or the
+attempt to validate the checksum failed.
+
+Similarly, the `odp_packet_l3_chksum_insert()` and
+`odp_packet_l4_chksum_insert()` APIs may be used to override default checksum
+processing for individual packets prior to transmission. If no explicit
+checksum processing is specified for a packet, then any checksum generation
+is controlled by the PktIO configuration of the interface used to transmit it.
diff --git a/doc/users-guide/users-guide-pktio.adoc b/doc/users-guide/users-guide-pktio.adoc
index 80a58d2fb..73d6e0485 100644
--- a/doc/users-guide/users-guide-pktio.adoc
+++ b/doc/users-guide/users-guide-pktio.adoc
@@ -38,6 +38,9 @@ PktIO objects begin life by being _opened_ via the call:
* errno set. Use odp_pktio_lookup() to obtain a handle to an already open
* device. Packet IO parameters provide interface level configuration options.
*
+ * Use odp_pktio_param_init() to initialize packet IO parameters into their
+ * default values. Default values are also used when 'param' pointer is NULL.
+ *
* Packet input queue configuration must be setup with
* odp_pktin_queue_config() before odp_pktio_start() is called. When packet
* input mode is ODP_PKTIN_MODE_DISABLED, odp_pktin_queue_config() call is
@@ -66,7 +69,7 @@ PktIO objects begin life by being _opened_ via the call:
* @param name Packet IO device name
* @param pool Default pool from which to allocate storage for packets
* received over this interface, must be of type ODP_POOL_PACKET
- * @param param Packet IO parameters
+ * @param param Packet IO parameters. Uses defaults when NULL.
*
* @return Packet IO handle
* @retval ODP_PKTIO_INVALID on failure
@@ -85,7 +88,7 @@ PktIO objects begin life by being _opened_ via the call:
* @see odp_pktio_start(), odp_pktio_stop(), odp_pktio_close()
*/
odp_pktio_t odp_pktio_open(const char *name, odp_pool_t pool,
- const odp_pktio_param_t *param);
+ const odp_pktio_param_t *param);
-----
`odp_pktio_open()` takes three arguments: a *name*, which is an
implementation-defined string that identifies the logical interface to be
@@ -97,25 +100,27 @@ I/O options to be associated with this PktIO instance.
/**
* Packet IO parameters
*
- * In minimum, user must select input and output modes. Use 0 for defaults.
- * Initialize entire struct with zero to maintain API compatibility.
+ * Packet IO interface level parameters. Use odp_pktio_param_init() to
+ * initialize the structure with default values.
*/
typedef struct odp_pktio_param_t {
/** Packet input mode
*
* The default value is ODP_PKTIN_MODE_DIRECT. */
odp_pktin_mode_t in_mode;
+
/** Packet output mode
*
* The default value is ODP_PKTOUT_MODE_DIRECT. */
odp_pktout_mode_t out_mode;
+
} odp_pktio_param_t;
-----
ODP defines *"loop"* as a reserved name to indicate that this PktIO represents
a loopback interface. Loopback interfaces are useful as a means of recycling
packets back for reclassification after decryption or decapsulation, as well as
-for diagnostic or testing purposes. For example, when receiving IPSEC traffic,
-the classifier is able to recognize that the traffic is IPSEC, however until
+for diagnostic or testing purposes. For example, when receiving IPsec traffic,
+the classifier is able to recognize that the traffic is IPsec, however until
the traffic is decrypted it is unable to say what that traffic contains.
So following decryption, sending the decrypted packet back to a loopback
interface allows the classifier to take a "second look" at the packet and
@@ -127,6 +132,304 @@ use for packet allocation if not overridden by the classifier due to a
specific or default Class-of-Service (CoS) match on the packet. The *param*
struct, in turn, specifies the input and output *modes* of the PktIO.
+=== PktIO Capabilities and PktIn/PktOut Configuration
+Associated with each PktIO is a set of _capabilities_ that provide information
+such as the maximum number of input/output queues it supports, its configuration
+options, and the operations is supports. These are aggregated into
+the struct:
+[source,c]
+-----
+/**
+ * Packet IO capabilities
+ */
+typedef struct odp_pktio_capability_t {
+ /** Maximum number of input queues */
+ unsigned max_input_queues;
+
+ /** Maximum number of output queues */
+ unsigned max_output_queues;
+
+ /** Supported pktio configuration options */
+ odp_pktio_config_t config;
+
+ /** Supported set operations
+ *
+ * A bit set to one indicates a supported operation. All other bits are
+ * set to zero. */
+ odp_pktio_set_op_t set_op;
+
+ /** @deprecated Use enable_loop inside odp_pktin_config_t */
+ odp_bool_t ODP_DEPRECATE(loop_supported);
+} odp_pktio_capability_t;
+-----
+That is returned by the `odp_pktio_capability()` API. This returns the
+limits and default values for these capabilities which can in turn be set
+via the `odp_pktio_config()` API, which takes as input the struct:
+[source,c]
+-----
+/**
+ * Packet IO configuration options
+ *
+ * Packet IO interface level configuration options. Use odp_pktio_capability()
+ * to see which options are supported by the implementation.
+ * Use odp_pktio_config_init() to initialize the structure with default values.
+ */
+typedef struct odp_pktio_config_t {
+ /** Packet input configuration options bit field
+ *
+ * Default value for all bits is zero. */
+ odp_pktin_config_opt_t pktin;
+
+ /** Packet output configuration options bit field
+ *
+ * Default value for all bits is zero. */
+ odp_pktout_config_opt_t pktout;
+
+ /** Packet input parser configuration */
+ odp_pktio_parser_config_t parser;
+
+ /** Interface loopback mode
+ *
+ * In this mode the packets sent out through the interface is
+ * looped back to input of the same interface. Supporting loopback mode
+ * is an optional feature per interface and should be queried in the
+ * interface capability before enabling the same. */
+ odp_bool_t enable_loop;
+
+ /** Inbound IPSEC inlined with packet input
+ *
+ * Enable/disable inline inbound IPSEC operation. When enabled packet
+ * input directs all IPSEC packets automatically to IPSEC inbound
+ * processing. IPSEC configuration is done through the IPSEC API.
+ * Packets that are not (recognized as) IPSEC are processed
+ * according to the packet input configuration.
+ *
+ * 0: Disable inbound IPSEC inline operation (default)
+ * 1: Enable inbound IPSEC inline operation
+ *
+ * @see odp_ipsec_config(), odp_ipsec_sa_create()
+ */
+ odp_bool_t inbound_ipsec;
+
+ /** Outbound IPSEC inlined with packet output
+ *
+ * Enable/disable inline outbound IPSEC operation. When enabled IPSEC
+ * outbound processing can send outgoing IPSEC packets directly
+ * to the pktio interface for output. IPSEC configuration is done
+ * through the IPSEC API.
+ *
+ * Outbound IPSEC inline operation cannot be combined with traffic
+ * manager (ODP_PKTOUT_MODE_TM).
+ *
+ * 0: Disable outbound IPSEC inline operation (default)
+ * 1: Enable outbound IPSEC inline operation
+ *
+ * @see odp_ipsec_config(), odp_ipsec_sa_create()
+ */
+ odp_bool_t outbound_ipsec;
+
+} odp_pktio_config_t;
+-----
+The IPsec related configurations will be discussed later in the IPsec chapter,
+but for now we'll focus on the PktIn/PktOut configuration and the
+parser configuration.
+
+==== PktIn Configuration
+For PktIOs that will receive packets, the `odp_pktin_config_opt_t` struct
+controls RX processing to be performed on these packets as they are received:
+[source,c]
+-----
+/**
+ * Packet input configuration options bit field
+ *
+ * Packet input configuration options listed in a bit field structure. Packet
+ * input timestamping may be enabled for all packets or at least for those that
+ * belong to time synchronization protocol (PTP).
+ *
+ * Packet input checksum checking may be enabled or disabled. When it is
+ * enabled, implementation will attempt to verify checksum correctness on
+ * incoming packets and depending on drop configuration either deliver erroneous
+ * packets with appropriate flags set (e.g. odp_packet_has_l3_error(),
+ * odp_packet_l3_chksum_status()) or drop those. When packet dropping is
+ * enabled, application will never receive a packet with the specified error
+ * and may avoid to check the error flag.
+ *
+ * If checksum checking is enabled, IPv4 header checksum checking is always
+ * done for packets that do not have IP options and L4 checksum checking
+ * is done for unfragmented packets that do not have IPv4 options or IPv6
+ * extension headers. In other cases checksum checking may or may not
+ * be done. For example, L4 checksum of fragmented packets is typically
+ * not checked.
+ *
+ * IPv4 checksum checking may be enabled only when parsing level is
+ * ODP_PROTO_LAYER_L3 or higher. Similarly, L4 level checksum checking
+ * may be enabled only with parsing level ODP_PROTO_LAYER_L4 or higher.
+ *
+ * Whether checksum checking was done and whether a checksum was correct
+ * can be queried for each received packet with odp_packet_l3_chksum_status()
+ * and odp_packet_l4_chksum_status().
+ */
+typedef union odp_pktin_config_opt_t {
+ /** Option flags */
+ struct {
+ /** Timestamp all packets on packet input */
+ uint64_t ts_all : 1;
+
+ /** Timestamp (at least) IEEE1588 / PTP packets
+ * on packet input */
+ uint64_t ts_ptp : 1;
+
+ /** Check IPv4 header checksum on packet input */
+ uint64_t ipv4_chksum : 1;
+
+ /** Check UDP checksum on packet input */
+ uint64_t udp_chksum : 1;
+
+ /** Check TCP checksum on packet input */
+ uint64_t tcp_chksum : 1;
+
+ /** Check SCTP checksum on packet input */
+ uint64_t sctp_chksum : 1;
+
+ /** Drop packets with an IPv4 error on packet input */
+ uint64_t drop_ipv4_err : 1;
+
+ /** Drop packets with an IPv6 error on packet input */
+ uint64_t drop_ipv6_err : 1;
+
+ /** Drop packets with a UDP error on packet input */
+ uint64_t drop_udp_err : 1;
+
+ /** Drop packets with a TCP error on packet input */
+ uint64_t drop_tcp_err : 1;
+
+ /** Drop packets with a SCTP error on packet input */
+ uint64_t drop_sctp_err : 1;
+
+ } bit;
+
+ /** All bits of the bit field structure
+ *
+ * This field can be used to set/clear all flags, or bitwise
+ * operations over the entire structure. */
+ uint64_t all_bits;
+} odp_pktin_config_opt_t;
+-----
+These are used to control packet timestamping as well as default packet checkum
+verification processing.
+
+==== PktIO Parsing Configuration
+For RX processing, packets may also be parsed automatically as part of
+receipt as controlled by the `odp_pktio_parser_config_t` struct:
+[source,c]
+-----
+/**
+ * Parser configuration
+ */
+typedef struct odp_pktio_parser_config_t {
+ /** Protocol parsing level in packet input
+ *
+ * Application requires that protocol headers in a packet are checked
+ * up to this layer during packet input. Use ODP_PROTO_LAYER_ALL for
+ * all layers. Packet metadata for this and all preceding layers are
+ * set. In addition, offset (and pointer) to the next layer is set.
+ * Other layer/protocol specific metadata have undefined values.
+ *
+ * The default value is ODP_PROTO_LAYER_ALL. */
+ odp_proto_layer_t layer;
+
+} odp_pktio_parser_config_t;
+-----
+Note that parsing is automatically done whenever classification is enabled
+for an RX interface (see below).
+
+==== PktOut Configuration
+For PktIOs that will transmit packets, the `odp_pktout_config_opt_t` struct
+controls TX processing to be performed on these packets as they are
+transmitted:
+[source,c]
+-----
+/**
+ * Packet output configuration options bit field
+ *
+ * Packet output configuration options listed in a bit field structure. Packet
+ * output checksum insertion may be enabled or disabled (e.g. ipv4_chksum_ena):
+ *
+ * 0: Disable checksum insertion. Application will not request checksum
+ * insertion for any packet. This is the default value for xxx_chksum_ena
+ * bits.
+ * 1: Enable checksum insertion. Application will request checksum insertion
+ * for some packets.
+ *
+ * When checksum insertion is enabled, application may use configuration options
+ * to set the default behaviour on packet output (e.g. ipv4_chksum):
+ *
+ * 0: Do not insert checksum by default. This is the default value for
+ * xxx_chksum bits.
+ * 1: Calculate and insert checksum by default.
+ *
+ * These defaults may be overridden on per packet basis using e.g.
+ * odp_packet_l4_chksum_insert().
+ *
+ * For correct operation, packet metadata must provide valid offsets for the
+ * appropriate protocols. For example, UDP checksum calculation needs both L3
+ * and L4 offsets (to access IP and UDP headers). When application
+ * (e.g. a switch) does not modify L3/L4 data and thus checksum does not need
+ * to be updated, checksum insertion should be disabled for optimal performance.
+ *
+ * Packet flags (odp_packet_has_*()) are ignored for the purpose of checksum
+ * insertion in packet output.
+ *
+ * UDP, TCP and SCTP checksum insertion must not be requested for IP fragments.
+ * Use checksum override function (odp_packet_l4_chksum_insert()) to disable
+ * checksumming when sending a fragment through a packet IO interface that has
+ * the relevant L4 checksum insertion enabled.
+ *
+ * Result of checksum insertion at packet output is undefined if the protocol
+ * headers required for checksum calculation are not well formed. Packet must
+ * contain at least as many data bytes after L3/L4 offsets as the headers
+ * indicate. Other data bytes of the packet are ignored for the checksum
+ * insertion.
+ */
+typedef union odp_pktout_config_opt_t {
+ /** Option flags for packet output */
+ struct {
+ /** Enable IPv4 header checksum insertion. */
+ uint64_t ipv4_chksum_ena : 1;
+
+ /** Enable UDP checksum insertion */
+ uint64_t udp_chksum_ena : 1;
+
+ /** Enable TCP checksum insertion */
+ uint64_t tcp_chksum_ena : 1;
+
+ /** Enable SCTP checksum insertion */
+ uint64_t sctp_chksum_ena : 1;
+
+ /** Insert IPv4 header checksum by default */
+ uint64_t ipv4_chksum : 1;
+
+ /** Insert UDP checksum on packet by default */
+ uint64_t udp_chksum : 1;
+
+ /** Insert TCP checksum on packet by default */
+ uint64_t tcp_chksum : 1;
+
+ /** Insert SCTP checksum on packet by default */
+ uint64_t sctp_chksum : 1;
+
+ } bit;
+
+ /** All bits of the bit field structure
+ *
+ * This field can be used to set/clear all flags, or bitwise
+ * operations over the entire structure. */
+ uint64_t all_bits;
+} odp_pktout_config_opt_t;
+-----
+These are used to control default checksum generation processing for
+transmitted packets.
+
=== PktIO Input and Output Modes
PktIO objects support four different Input and Output modes, that may be
specified independently at *open* time.
@@ -209,25 +512,48 @@ typedef struct odp_pktin_queue_param_t {
* applicable. */
odp_pktio_op_mode_t op_mode;
+ /** Enable classifier
+ *
+ * * 0: Classifier is disabled (default)
+ * * 1: Classifier is enabled. Use classifier to direct incoming
+ * packets into pktin event queues. Classifier can be enabled
+ * only in ODP_PKTIN_MODE_SCHED and ODP_PKTIN_MODE_QUEUE modes.
+ * Both classifier and hashing cannot be enabled simultaneously
+ * ('hash_enable' must be 0). */
+ odp_bool_t classifier_enable;
+
/** Enable flow hashing
- * 0: Do not hash flows
- * 1: Hash flows to input queues */
+ *
+ * * 0: Do not hash flows (default)
+ * * 1: Enable flow hashing. Use flow hashing to spread incoming
+ * packets into input queues. Hashing can be enabled in all
+ * modes. Both classifier and hashing cannot be enabled
+ * simultaneously ('classifier_enable' must be 0). */
odp_bool_t hash_enable;
- /** Protocol field selection for hashing. Multiple protocols can be
- * selected. */
+ /** Protocol field selection for hashing
+ *
+ * Multiple protocols can be selected. Ignored when 'hash_enable' is
+ * zero. The default value is all bits zero. */
odp_pktin_hash_proto_t hash_proto;
- /** Number of input queues to be created. More than one input queue
- * require input hashing or classifier setup. Hash_proto is ignored
- * when hash_enable is zero or num_queues is one. This value must be
- * between 1 and interface capability. Queue type is defined by the
- * input mode. The default value is 1. */
+ /** Number of input queues to be created
+ *
+ * When classifier is enabled in odp_pktin_queue_config() this
+ * value is ignored, otherwise at least one queue is required.
+ * More than one input queues require flow hashing configured.
+ * The maximum value is defined by pktio capability 'max_input_queues'.
+ * Queue type is defined by the input mode. The default value is 1. */
unsigned num_queues;
- /** Queue parameters for creating input queues in ODP_PKTIN_MODE_QUEUE
+ /** Queue parameters
+ *
+ * These are used for input queue creation in ODP_PKTIN_MODE_QUEUE
* or ODP_PKTIN_MODE_SCHED modes. Scheduler parameters are considered
- * only in ODP_PKTIN_MODE_SCHED mode. */
+ * only in ODP_PKTIN_MODE_SCHED mode. Default values are defined in
+ * odp_queue_param_t documentation.
+ * When classifier is enabled in odp_pktin_queue_config() this
+ * value is ignored. */
odp_queue_param_t queue_param;
} odp_pktin_queue_param_t;
@@ -347,8 +673,10 @@ Once started, the PktIn queue handles are used as arguments to
/**
* Receive packets directly from an interface input queue
*
- * Receives up to 'num' packets from the pktio interface input queue. When
- * input queue parameter 'op_mode' has been set to ODP_PKTIO_OP_MT_UNSAFE,
+ * Receives up to 'num' packets from the pktio interface input queue. Returns
+ * the number of packets received.
+ *
+ * When input queue parameter 'op_mode' has been set to ODP_PKTIO_OP_MT_UNSAFE,
* the operation is optimized for single thread operation per queue and the same
* queue must not be accessed simultaneously from multiple threads.
*
@@ -478,6 +806,11 @@ Once the PktIO has been configured for output and started via
* is less than 'num', the remaining packets at the end of packets[] array
* are not consumed, and the caller has to take care of them.
*
+ * Entire packet data is sent out (odp_packet_len() bytes of data, starting from
+ * odp_packet_data()). All other packet metadata is ignored unless otherwise
+ * specified e.g. for protocol offload purposes. Link protocol specific frame
+ * checksum and padding are added to frames before transmission.
+ *
* @param queue Packet output queue handle for sending packets
* @param packets[] Array of packets to send
* @param num Number of packets to send
@@ -485,7 +818,8 @@ Once the PktIO has been configured for output and started via
* @return Number of packets sent
* @retval <0 on failure
*/
-int odp_pktout_send(odp_pktout_queue_t queue, odp_packet_t packets[], int num);
+int odp_pktout_send(odp_pktout_queue_t queue, const odp_packet_t packets[],
+ int num);;
-----
Note that the argument to this call specifies the PktOut queue that the
packet is to be added to rather than the PktIO itself. This permits multiple
diff --git a/doc/users-guide/users-guide-tm.adoc b/doc/users-guide/users-guide-tm.adoc
index 251297335..55efb1b21 100644
--- a/doc/users-guide/users-guide-tm.adoc
+++ b/doc/users-guide/users-guide-tm.adoc
@@ -10,7 +10,7 @@ A given platform supporting this TM API could support one or more pure hardware
based packet scheduling systems, one or more pure software based systems or one
or more hybrid systems - where because of hardware constraints some of the
packet scheduling is done in hardware and some is done in software. In
-addition, there may also be additional API's beyond those described here for:
+addition, there may also be additional APIs beyond those described here for:
- controlling advanced capabilities supported by specific hardware, software
or hybrid subsystems
@@ -84,7 +84,7 @@ traffic, while allowing for less idle outputs.
==== Weighted Fair Queuing
-Weighted Fair Queuing (WFQ) is used to arbitrate amongst multiple input
+Weighted Fair Queuing (WFQ) is used to arbitrate among multiple input
packets with the same priority. Each input can be assigned a weight in the
range MIN_WFQ_WEIGHT..MAX_WFQ_WEIGHT (nominally 1..255) that affects the way
the algorithm chooses the next packet. If all of the weights are equal AND all
@@ -158,7 +158,7 @@ final scheduling decision is controlled by equal priority schedulers,
strict priority multiplexers, bandwidth shapers - at multiple levels - all
forming a tree rooted at a single egress object. In other words, all
tm_queues and tm_nodes have the property that their logical "output" feeds
-into one fan-in of a subsequent tm_node or egresss object - forming a proper
+into one fan-in of a subsequent tm_node or egress object - forming a proper
tree.
.Hierarchical Scheduling
@@ -178,7 +178,7 @@ choice" of what packet/tm_queue should next be serviced.
Tm_nodes are the main "entity"/object that a TM system is composed of. Each
tm_node is a mini-TM subsystem of its own, but the interconnection and
interplay of a multi-level "tree" of tm_nodes can allow the user to specify
-some very sophisticated behaviours. Each tm_node can contain a set of scheduler
+some very sophisticated behaviors. Each tm_node can contain a set of scheduler
(one per strict priority level), a strict priority multiplexer, a bandwidth
shaper and a WRED component - or a subset of these.
diff --git a/doc/users-guide/users-guide.adoc b/doc/users-guide/users-guide.adoc
index 7f2ad69e2..7914459e6 100644
--- a/doc/users-guide/users-guide.adoc
+++ b/doc/users-guide/users-guide.adoc
@@ -151,7 +151,7 @@ of the specification or other minor changes that do not affect either the
syntax or semantics of the specification. Such changes in the API specification
are expected to be rare. Increments to the minor level
represent the introduction of new APIs or functional capabilities, or changes
-to he specified syntax or functional behavior of APIs and thus may require
+to the specified syntax or functional behavior of APIs and thus may require
application source code changes. Such changes are well documented in the
release notes for each revision of the specification. Finally, increments to
the major level represent significant structural changes that most likely
@@ -247,14 +247,14 @@ polled by ODP _Threads_, or can pass through the _Classifier_ and sorted into
Queues that represent individual flows. These queues can then be dispatched
to application threads via the _Scheduler_.
-Threads, in term can invoke various ODP APIs to manipulate packet contents
+Threads, in turn can invoke various ODP APIs to manipulate packet contents
prior to disposing of them. For output processing, packets make by directly
queued to a PktIO output queue or else they may be handed to the _Traffic
Manager_ for programmatic _Quality of Service (QoS)_ processing before winding
up being transmitted (TX). Note that output interfaces may operate in
_loopback_ mode, in which case packets sent to them are re-routed back to the
-input lines for "second pass" processing. For example, an incoming IPSec packet
-cannot be properly classified (beyond being IPSec traffic) until it is
+input lines for "second pass" processing. For example, an incoming IPsec packet
+cannot be properly classified (beyond being IPsec traffic) until it is
decrypted. Once decrypted and its actual contents made visible, it can then
be classified into its real flow.
@@ -312,7 +312,7 @@ appropriate type represented by the event.
A queue is a message passing channel that holds events. Events can be
added to a queue via enqueue operations or removed from a queue via dequeue
operations. The endpoints of a queue will vary depending on how it is used.
-Queues come in two major types: polled and scheduled, which will be
+Queues come in two major types: plain and scheduled, which will be
discussed in more detail when the event model is introduced. Queues may also
have an associated context, which represents a persistent state for all
events that make use of it. These states are what permit threads to perform
@@ -576,7 +576,7 @@ values.
Calling `odp_init_global()` establishes the ODP API framework and MUST be
called before any other ODP API may be called. Note that it is only called
once per application. A successful call to `odp_init_global()` returns rc = 0
-and sets the `instance` variable supplied as input to the call to an handle
+and sets the `instance` variable supplied as input to the call to a handle
representing this unique ODP instance.
The `odp_init_t` parameter is used to specify various customizations to the
@@ -661,7 +661,7 @@ area and how best to use ODP to achieve these goals.
=== Portability and Coexistence
Because ODP offers a programming _framework_ rather than a programming
_environment_, it is designed to be able to work alongside APIs offered by
-other frameworks with minimual interference. Therefore when we speak of
+other frameworks with minimal interference. Therefore when we speak of
portability in an ODP context, we of necessity speak of portability of those
portions of the application that make use of ODP APIs. If an application uses
non-ODP APIs then those must be taken into consideration as well when
@@ -756,10 +756,10 @@ Architecture (ISA), such as x86-64 or AArch64. Binaries cannot directly port
between ISAs--that requires a recompilation.
Each ODP implementation will identify which ABI definition it supports, if any.
-When compiling against an ODP implementation in ABI compabitilty mode, the
+When compiling against an ODP implementation in ABI compatibility mode, the
resulting binary is automatically binary compatible with all other ODP
implementations that share this ABI. For example, for the x86-64 ISA, both
-the `odp-linux` and `odp-dpdk` implemtations are a common ABI.
+the `odp-linux` and `odp-dpdk` implementations are a common ABI.
== Shared memory
=== Allocating shared memory
@@ -798,16 +798,16 @@ shared_data_t *shared_data;
shared_data = odp_shm_addr(shm);
----
-The address returned by `odp_shm_addr()` is valid only in the calling ODP
-thread space: odp_shm_t handles can be shared between ODP threads and remain
-valid within any threads, whereas the address returned by `odp_shm_addr(shm)`
-may differ from ODP threads to ODP threads (for the same 'shm' block), and
-should therefore not be shared between ODP threads.
-For instance, it would be correct to send a shm handle using IPC between two
-ODP threads and let each of these thread do their own `odp_shm_addr()` to
-get the block address. Directly sending the address returned by
-`odp_shm_addr()` from one ODP thread to another would however possibly fail
-(the address may have no sense in the receiver address space).
+The address returned by `odp_shm_addr()` is normally valid only in the calling
+ODP thread space: odp_shm_t handles can be shared between ODP threads and
+remain valid within any threads, whereas the address returned by
+`odp_shm_addr(shm)` may differ from ODP threads to ODP threads (for the same
+'shm' block), and should therefore not be shared between ODP threads. For
+instance, it would be correct to send a shm handle using IPC between two ODP
+threads and let each of these thread do their own `odp_shm_addr()` to get the
+block address. Directly sending the address returned by `odp_shm_addr()` from
+one ODP thread to another would however possibly fail (the address may make no
+sense in the receiver address space).
The address returned by `odp_shm_addr()` is nevertheless guaranteed to be
aligned according to the alignment requirements provided at block creation
@@ -819,7 +819,13 @@ All shared memory blocks are contiguous in any ODP thread addressing space:
as provided in the `odp_shm_reserve()` call) is read and writeable and
mapping the shared memory block. There is no fragmentation.
-=== Memory behaviour
+The exception to this rule is if the `odp_shm_t` is created with the
+`ODP_SHM_SINGLE_VA` flag. This requests that `odp_shm_addr()` return the same
+virtual address for all ODP threads in this instance. Note that there may be a
+performance cost or shm size limit associated with providing this function in
+some implementations.
+
+=== Memory behavior
By default ODP threads are assumed to behave as cache coherent systems:
Any change performed on a shared memory block is guaranteed to eventually
become visible to other ODP threads sharing this memory block.
@@ -964,23 +970,23 @@ Queues are the fundamental event sequencing mechanism provided by ODP and all
ODP applications make use of them either explicitly or implicitly. Queues are
created via the 'odp_queue_create()' API that returns a handle of type
`odp_queue_t` that is used to refer to this queue in all subsequent APIs that
-reference it. Queues have one of two ODP-defined _types_, POLL, and SCHED that
-determine how they are used. POLL queues directly managed by the ODP
+reference it. Queues have one of two ODP-defined _types_, PLAIN, and SCHED that
+determine how they are used. PLAIN queues directly managed by the ODP
application while SCHED queues make use of the *ODP scheduler* to provide
automatic scalable dispatching and synchronization services.
-.Operations on POLL queues
+.Operations on PLAIN queues
[source,c]
----
-odp_queue_t poll_q1 = odp_queue_create("poll queue 1", ODP_QUEUE_TYPE_POLL, NULL);
-odp_queue_t poll_q2 = odp_queue_create("poll queue 2", ODP_QUEUE_TYPE_POLL, NULL);
+odp_queue_t plain_q1 = odp_queue_create("poll queue 1", ODP_QUEUE_TYPE_PLAIN, NULL);
+odp_queue_t plain_q2 = odp_queue_create("poll queue 2", ODP_QUEUE_TYPE_PLAIN, NULL);
...
-odp_event_t ev = odp_queue_deq(poll_q1);
+odp_event_t ev = odp_queue_deq(plain_q1);
...do something
-int rc = odp_queue_enq(poll_q2, ev);
+int rc = odp_queue_enq(plain_q2, ev);
----
-The key distinction is that dequeueing events from POLL queues is an
+The key distinction is that dequeueing events from PLAIN queues is an
application responsibility while dequeueing events from SCHED queues is the
responsibility of the ODP scheduler.
@@ -992,7 +998,7 @@ odp_queue_param_init(&qp);
odp_schedule_prio_t prio = ...;
odp_schedule_group_t sched_group = ...;
qp.sched.prio = prio;
-qp.sched.sync = ODP_SCHED_SYNC_[NONE|ATOMIC|ORDERED];
+qp.sched.sync = ODP_SCHED_SYNC_[PARALLEL|ATOMIC|ORDERED];
qp.sched.group = sched_group;
qp.lock_count = n; /* Only relevant for ordered queues */
odp_queue_t sched_q1 = odp_queue_create("sched queue 1", ODP_QUEUE_TYPE_SCHED, &qp);
@@ -1048,8 +1054,8 @@ Three types of queue scheduler synchronization area supported: Parallel,
Atomic, and Ordered.
==== Parallel Queues
-SCHED queues that specify a sync mode of ODP_SCHED_SYNC_NONE are unrestricted
-in how events are processed.
+SCHED queues that specify a sync mode of ODP_SCHED_SYNC_PARALLEL are
+unrestricted in how events are processed.
.Parallel Queue Scheduling
image::parallel_queue.svg[align="center"]
diff --git a/example/generator/generator_null_test.sh b/example/generator/generator_null_test.sh
index a598ffd92..3c37a99d2 100755
--- a/example/generator/generator_null_test.sh
+++ b/example/generator/generator_null_test.sh
@@ -6,7 +6,8 @@
# SPDX-License-Identifier: BSD-3-Clause
#
-if [ -n "${ODP_PLATFORM}" -a "x${ODP_PLATFORM}" != "xlinux-generic" ]
+if [ -n "${ODP_PLATFORM}" -a "x${ODP_PLATFORM}" != "xlinux-generic" ] &&
+ [ -n "${ODP_PLATFORM}" -a "x${ODP_PLATFORM}" != "xlinux-dpdk" ]
then
echo "null pktio might be unsupported on this platform, skipping"
exit 77
diff --git a/example/l2fwd/README b/example/l2fwd/README
index f6fdc01ca..091d046ea 100644
--- a/example/l2fwd/README
+++ b/example/l2fwd/README
@@ -5,3 +5,10 @@ Source code and Makefiles placed under test/performance/ directory.
This L2 forwarding application can be used as example reference as well
as performance test for different odp modes (direct, queue or scheduler
with parallel, atomic or ordered queues).
+
+Note that this example is tuned for performance. As a result, when using
+scheduled mode with direct packet I/O output or queued output with multiple
+output queues, packet order is not guaranteed. To achieve guaranteed order,
+use a single worker thread or output interfaces with single output
+queues. Other examples of scalable processing using ordered queues that
+preserve order can be seen in the odp_pktio_ordered performance test.
diff --git a/example/l2fwd_simple/Makefile.am b/example/l2fwd_simple/Makefile.am
index f082335ef..7a6d1f68c 100644
--- a/example/l2fwd_simple/Makefile.am
+++ b/example/l2fwd_simple/Makefile.am
@@ -5,7 +5,7 @@ bin_PROGRAMS = odp_l2fwd_simple
odp_l2fwd_simple_SOURCES = odp_l2fwd_simple.c
if test_example
-if HAVE_PCAP
+if HAVE_PMD_PCAP
TESTS = l2fwd_simple_run.sh
endif
endif
diff --git a/example/l2fwd_simple/l2fwd_simple_run.sh b/example/l2fwd_simple/l2fwd_simple_run.sh
index 10f4e6dc6..6ebfb9803 100755
--- a/example/l2fwd_simple/l2fwd_simple_run.sh
+++ b/example/l2fwd_simple/l2fwd_simple_run.sh
@@ -9,7 +9,11 @@
PCAP_IN=`find . ${TEST_DIR} $(dirname $0) -name udp64.pcap -print -quit`
echo "using PCAP_IN = ${PCAP_IN}"
-./odp_l2fwd_simple${EXEEXT} pcap:in=${PCAP_IN} pcap:out=pcapout.pcap \
+export ODP_PLATFORM_PARAMS="--no-pci \
+--vdev net_pcap0,rx_pcap=${PCAP_IN},tx_pcap=pcapout.pcap \
+--vdev net_pcap1,rx_pcap=${PCAP_IN},tx_pcap=pcapout.pcap"
+
+./odp_l2fwd_simple${EXEEXT} 0 1 \
02:00:00:00:00:01 02:00:00:00:00:02 &
sleep 1
diff --git a/example/l3fwd/Makefile.am b/example/l3fwd/Makefile.am
index 9a48ea173..96530b3d4 100644
--- a/example/l3fwd/Makefile.am
+++ b/example/l3fwd/Makefile.am
@@ -11,9 +11,9 @@ odp_l3fwd_SOURCES = \
if test_example
-if HAVE_PCAP
+if HAVE_PMD_PCAP
TESTS = odp_l3fwd_run.sh
endif
endif
-EXTRA_DIST = odp_l3fwd_run.sh udp64.pcap
+EXTRA_DIST = odp_l3fwd_run.sh udp64.pcap empty.pcap
diff --git a/example/l3fwd/empty.pcap b/example/l3fwd/empty.pcap
new file mode 100644
index 000000000..4f9600e90
--- /dev/null
+++ b/example/l3fwd/empty.pcap
Binary files differ
diff --git a/example/l3fwd/odp_l3fwd_run.sh b/example/l3fwd/odp_l3fwd_run.sh
index acffb8431..ffe32aa2f 100755
--- a/example/l3fwd/odp_l3fwd_run.sh
+++ b/example/l3fwd/odp_l3fwd_run.sh
@@ -8,11 +8,16 @@
PCAP_IN=`find . ${TEST_DIR} $(dirname $0) -name udp64.pcap -print -quit`
PCAP_OUT="pcapout.pcap"
+PCAP_EMPTY="empty.pcap"
PCAP_IN_SIZE=`stat -c %s ${PCAP_IN}`
echo "using PCAP_IN = ${PCAP_IN}, PCAP_OUT = ${PCAP_OUT}"
-./odp_l3fwd${EXEEXT} -i pcap:in=${PCAP_IN},pcap:out=${PCAP_OUT} \
- -r "10.0.0.0/24,pcap:out=${PCAP_OUT}" -d 30
+export ODP_PLATFORM_PARAMS="--no-pci \
+--vdev net_pcap0,rx_pcap=${PCAP_IN},tx_pcap=/dev/null \
+--vdev net_pcap1,rx_pcap=${PCAP_EMPTY},tx_pcap=${PCAP_OUT}"
+
+./odp_l3fwd${EXEEXT} -i 0,1 \
+ -r "10.0.0.0/24,1" -d 30
STATUS=$?
PCAP_OUT_SIZE=`stat -c %s ${PCAP_OUT}`
diff --git a/example/packet/Makefile.am b/example/packet/Makefile.am
index 228c3506d..eea26d3e5 100644
--- a/example/packet/Makefile.am
+++ b/example/packet/Makefile.am
@@ -5,7 +5,7 @@ bin_PROGRAMS = odp_pktio
odp_pktio_SOURCES = odp_pktio.c
if test_example
-if HAVE_PCAP
+if HAVE_PMD_PCAP
TESTS = pktio_run.sh
endif
endif
diff --git a/example/packet/pktio_run.sh b/example/packet/pktio_run.sh
index 6abaec16d..caf8649a7 100755
--- a/example/packet/pktio_run.sh
+++ b/example/packet/pktio_run.sh
@@ -11,8 +11,11 @@ PCAP_OUT="pcapout.pcap"
PCAP_IN_SIZE=`stat -c %s ${PCAP_IN}`
echo "using PCAP in=${PCAP_IN}:out=${PCAP_OUT} size %${PCAP_IN_SIZE}"
+export ODP_PLATFORM_PARAMS="--no-pci \
+--vdev net_pcap0,rx_pcap=${PCAP_IN},tx_pcap=${PCAP_OUT}"
+
# burst mode
-./odp_pktio${EXEEXT} -ipcap:in=${PCAP_IN}:out=${PCAP_OUT} -t 5 -m 0
+./odp_pktio${EXEEXT} -i 0 -t 5 -m 0
STATUS=$?
PCAP_OUT_SIZE=`stat -c %s ${PCAP_OUT}`
rm -f ${PCAP_OUT}
@@ -24,7 +27,7 @@ fi
echo "Pass -m 0: status ${STATUS}, in:${PCAP_IN_SIZE} out:${PCAP_OUT_SIZE}"
# queue mode
-./odp_pktio${EXEEXT} -ipcap:in=${PCAP_IN}:out=${PCAP_OUT} -t 5 -m 1
+./odp_pktio${EXEEXT} -i 0 -t 5 -m 1
STATUS=$?
PCAP_OUT_SIZE=`stat -c %s ${PCAP_OUT}`
rm -f ${PCAP_OUT}
@@ -36,7 +39,7 @@ fi
echo "Pass -m 1: status ${STATUS}, in:${PCAP_IN_SIZE} out:${PCAP_OUT_SIZE}"
# sched/queue mode
-./odp_pktio${EXEEXT} -ipcap:in=${PCAP_IN}:out=${PCAP_OUT} -t 5 -m 2
+./odp_pktio${EXEEXT} -i 0 -t 5 -m 2
STATUS=$?
PCAP_OUT_SIZE=`stat -c %s ${PCAP_OUT}`
rm -f ${PCAP_OUT}
@@ -48,7 +51,7 @@ fi
echo "Pass -m 2: status ${STATUS}, in:${PCAP_IN_SIZE} out:${PCAP_OUT_SIZE}"
# cpu number option test 1
-./odp_pktio${EXEEXT} -ipcap:in=${PCAP_IN}:out=${PCAP_OUT} -t 5 -m 0 -c 1
+./odp_pktio${EXEEXT} -i 0 -t 5 -m 0 -c 1
STATUS=$?
PCAP_OUT_SIZE=`stat -c %s ${PCAP_OUT}`
rm -f ${PCAP_OUT}
diff --git a/example/switch/Makefile.am b/example/switch/Makefile.am
index 2fb21bc41..cd74cd509 100644
--- a/example/switch/Makefile.am
+++ b/example/switch/Makefile.am
@@ -5,8 +5,8 @@ bin_PROGRAMS = odp_switch
odp_switch_SOURCES = odp_switch.c
if test_example
-if HAVE_PCAP
+if HAVE_PMD_PCAP
TESTS = switch_run.sh
endif
endif
-EXTRA_DIST = switch_run.sh udp64.pcap
+EXTRA_DIST = switch_run.sh udp64.pcap empty.pcap
diff --git a/example/switch/empty.pcap b/example/switch/empty.pcap
new file mode 100644
index 000000000..4f9600e90
--- /dev/null
+++ b/example/switch/empty.pcap
Binary files differ
diff --git a/example/switch/switch_run.sh b/example/switch/switch_run.sh
index 5fa1ae1cd..c14dcb22f 100755
--- a/example/switch/switch_run.sh
+++ b/example/switch/switch_run.sh
@@ -10,16 +10,22 @@ NUM_RX_PORT=3
RETVAL=0
PCAP_IN=`find . ${TEST_DIR} $(dirname $0) -name udp64.pcap -print -quit`
+PCAP_EMPTY="empty.pcap"
echo "Switch test using PCAP_IN = ${PCAP_IN}"
RX_PORTS=""
+RX_VDEVS=""
for i in `seq 1 $NUM_RX_PORT`;
do
- RX_PORTS="${RX_PORTS},pcap:out=pcapout${i}.pcap"
+ RX_PORTS="${RX_PORTS},${i}"
+ RX_VDEVS="${RX_VDEVS} --vdev net_pcap${i},rx_pcap=${PCAP_EMPTY},tx_pcap=pcapout${i}.pcap"
done
-./odp_switch${EXEEXT} -i pcap:in=${PCAP_IN}${RX_PORTS} -t 1
+export ODP_PLATFORM_PARAMS="--no-pci \
+--vdev net_pcap0,rx_pcap=${PCAP_IN},tx_pcap=/dev/null ${RX_VDEVS}"
+
+./odp_switch${EXEEXT} -i 0${RX_PORTS} -t 1
STATUS=$?
if [ "$STATUS" -ne 0 ]; then
echo "Error: status was: $STATUS, expected 0"
diff --git a/include/odp/api/spec/packet.h b/include/odp/api/spec/packet.h
index e1f2f2218..66665e121 100644
--- a/include/odp/api/spec/packet.h
+++ b/include/odp/api/spec/packet.h
@@ -1711,6 +1711,9 @@ odp_packet_chksum_status_t odp_packet_l3_chksum_status(odp_packet_t pkt);
* attempt. It depends on packet input (or IPSEC) configuration, packet content
* and implementation capabilities if checksum check is attempted for a packet.
*
+ * When a UDP packet does not have a checksum (e.g. checksum field of a UDP/IPv4
+ * packet is zero), checksum check result is ODP_PACKET_CHKSUM_OK.
+ *
* @param pkt Packet handle
*
* @return L4 checksum check status
diff --git a/m4/odp_dpdk.m4 b/m4/odp_dpdk.m4
index 53d105c8c..7425fb910 100644
--- a/m4/odp_dpdk.m4
+++ b/m4/odp_dpdk.m4
@@ -9,6 +9,8 @@ cur_driver=`basename "$filename" .a | sed -e 's/^lib//'`
AS_VAR_APPEND([DPDK_PMDS], [-l$cur_driver,])
AS_CASE([$cur_driver],
[rte_pmd_nfp], [AS_VAR_APPEND([DPDK_LIBS], [" -lm"])],
+ [rte_pmd_mlx4], [AS_VAR_APPEND([DPDK_LIBS], [" -lmlx4 -libverbs"])],
+ [rte_pmd_mlx5], [AS_VAR_APPEND([DPDK_LIBS], [" -lmlx5 -libverbs"])],
[rte_pmd_pcap], [AS_VAR_APPEND([DPDK_LIBS], [" -lpcap"])],
[rte_pmd_aesni_gcm], [AS_VAR_APPEND([DPDK_LIBS], [" -lIPSec_MB"])],
[rte_pmd_aesni_mb], [AS_VAR_APPEND([DPDK_LIBS], [" -lIPSec_MB"])],
@@ -19,6 +21,13 @@ AS_CASE([$cur_driver],
[rte_pmd_openssl], [AS_VAR_APPEND([DPDK_LIBS], [" -lcrypto"])])
done
AS_VAR_APPEND([DPDK_PMDS], [--no-whole-archive])
+have_pmd_pcap=no
+if [[ -f "$1"/librte_pmd_pcap.a ]]; then
+ have_pmd_pcap=yes
+fi
+AC_CONFIG_COMMANDS_PRE([dnl
+AM_CONDITIONAL([HAVE_PMD_PCAP], [test x$have_pmd_pcap = xyes])
+])
])
# _ODP_DPDK_SET_LIBS
diff --git a/m4/odp_libconfig.m4 b/m4/odp_libconfig.m4
index 926ceca62..302dc5066 100644
--- a/m4/odp_libconfig.m4
+++ b/m4/odp_libconfig.m4
@@ -8,12 +8,11 @@ AC_DEFUN([ODP_LIBCONFIG],
PKG_CHECK_MODULES([LIBCONFIG], [libconfig])
##########################################################################
-# Check for xxd availability
+# Check for od availability
##########################################################################
-AC_CHECK_PROGS([XXD], [xxd])
-if test -z "$XXD";
- then AC_MSG_ERROR([Could not find 'xxd'])
-fi
+AC_CHECK_PROGS([OD], [od])
+AC_PROG_SED
+AS_IF([test -z "$OD"], [AC_MSG_ERROR([Could not find 'od'])])
##########################################################################
# Create a header file odp_libconfig_config.h which containins null
@@ -21,8 +20,10 @@ fi
##########################################################################
AC_CONFIG_COMMANDS([platform/$1/include/odp_libconfig_config.h],
[mkdir -p platform/$1/include
- (cd ${srcdir}/config ; xxd -i odp-$1.conf) | \
- sed 's/\([[0-9a-f]]\)$/\0, 0x00/' > \
+ (echo "static const char config_builtin[[]] = {"; \
+ $OD -An -v -tx1 < ${srcdir}/config/odp-$1.conf | \
+ $SED -e 's/[[0-9a-f]]\+/0x\0,/g' ; \
+ echo "0x00 };") > \
platform/$1/include/odp_libconfig_config.h],
- [with_platform=$with_platform])
+ [with_platform=$with_platform OD=$OD SED=$SED])
]) # ODP_LIBCONFIG
diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am
index 0ce6ccc03..5e9f40de5 100644
--- a/platform/linux-dpdk/Makefile.am
+++ b/platform/linux-dpdk/Makefile.am
@@ -98,7 +98,6 @@ noinst_HEADERS = \
${top_srcdir}/platform/linux-generic/include/odp_libconfig_internal.h \
${top_srcdir}/platform/linux-generic/include/odp_llqueue.h \
${top_srcdir}/platform/linux-generic/include/odp_macros_internal.h \
- include/odp_packet_dpdk.h \
include/odp_packet_internal.h \
${top_srcdir}/platform/linux-generic/include/odp_name_table_internal.h \
include/odp_packet_io_internal.h \
@@ -152,7 +151,7 @@ __LIB__libodp_linux_la_SOURCES = \
../linux-generic/odp_ipsec_events.c \
../linux-generic/odp_ipsec_sad.c \
../linux-generic/odp_name_table.c \
- odp_libconfig.c \
+ ../linux-generic/odp_libconfig.c \
odp_packet.c \
odp_packet_dpdk.c \
odp_packet_flags.c \
diff --git a/platform/linux-dpdk/README b/platform/linux-dpdk/README
index 71cced161..c9e7af78e 100644
--- a/platform/linux-dpdk/README
+++ b/platform/linux-dpdk/README
@@ -4,13 +4,6 @@ All rights reserved.
SPDX-License-Identifier: BSD-3-Clause
-ERRATA:
-- DPDK 16.07 and earlier supports pool names with RTE_MEMZONE_NAMESIZE
- characters (including terminating NULL), which is 6 characters less than
- ODP_POOL_NAME_LEN. Names reaching into this interval might collide if the
- first 25 characters are not unique.
-
-
1. Rationale
=================================================
@@ -180,16 +173,15 @@ To restore the NIC's back to kernel use something like this:
5. Running ODP apps
=================================================
-ODP-DPDK applications need to be run as root. You also need to supply the DPDK
-command line parameters either as a null-terminated array of char's to
-odp_global_init()'s platform_params parameter:
+ODP-DPDK applications need to be run as root. You may also need to
+supply DPDK command line parameters either as a null-terminated array of
+char's to odp_global_init()'s platform_params parameter:
- odp_global_init([params], "-n 4");
+ odp_global_init([params], "--no-huge");
Or, if it's NULL the platform tries to read the ODP_PLATFORM_PARAMS environment
variable.
-You need to pass at least "-n [1..4]" to specify the number of memory channels.
The coremask (-c) is calculated by ODP-DPDK based on the process affinity at
startup. You can influence that with 'taskset'. DPDK init changes the affinity
of the calling thread, so after it returns the original affinity is restored.
@@ -201,7 +193,7 @@ like proper DPDK threads.
Exaple how to run an ODP-DPDK L2 forwarding application:
- sudo ODP_PLATFORM_PARAMS="-n 4" ./odp_l2fwd -i 0,1 -c 2
+ sudo ./odp_l2fwd -i 0,1 -c 2
-i 0,1 - interface numbers
-c 2 - number of worker cpus
@@ -234,7 +226,7 @@ CONFIG_RTE_LIBRTE_PMD_PCAP=y
mount -t hugetlbfs none /mnt/huge
Finally give l2fwd fake devices:
- ./l2fwd -c '0xf' -n 4 --vdev "eth_pcap0,iface=veth2-1" --vdev="eth_pcap1,iface=veth2-3" -- -p 3
+ ./l2fwd -c '0xf' --vdev "eth_pcap0,iface=veth2-1" --vdev="eth_pcap1,iface=veth2-3" -- -p 3
7. Upgrading ODP-DPDK to newer ODP API level
=================================================
@@ -270,8 +262,8 @@ http://dpdk.org/doc/guides/cryptodevs/index.html.
To build odp-dpdk with dpdk virtual crypto devices, we need to build supporting
intel multi-buffer library prior to dpdk build.
-get the Intel multi-buffer crypto from,
-https://downloadcenter.intel.com/download/22972
+Get the Intel multi-buffer crypto library from,
+https://github.com/intel/intel-ipsec-mb
and follow the README from the repo on how to build the library.
building dpdk:
@@ -294,5 +286,5 @@ AESNI_MULTI_BUFFER_LIB_PATH=/path-to/Intel-multi-buffer-crypto/ \
when building odp-dpdk application, add the multi-buffer crypto library path to make file.
Before running the application, export ODP_PLATFORM_PARAMS with corresponding
crypto vdev's.
-ex: ODP_PLATFORM_PARAMS="-n 4 --vdev cryptodev_aesni_mb_pmd,max_nb_sessions=32 \
+ex: ODP_PLATFORM_PARAMS="--vdev cryptodev_aesni_mb_pmd,max_nb_sessions=32 \
--vdev cryptodev_null_pmd,max_nb_sessions=32"
diff --git a/platform/linux-dpdk/include/odp/api/plat/std_clib_inlines.h b/platform/linux-dpdk/include/odp/api/plat/std_clib_inlines.h
index cf556e042..14072ba52 100644
--- a/platform/linux-dpdk/include/odp/api/plat/std_clib_inlines.h
+++ b/platform/linux-dpdk/include/odp/api/plat/std_clib_inlines.h
@@ -12,6 +12,8 @@ extern "C" {
#endif
#include <string.h>
+
+#include <rte_config.h>
#include <rte_memcpy.h>
_ODP_INLINE void *odp_memcpy(void *dst, const void *src, size_t num)
diff --git a/platform/linux-dpdk/include/odp_packet_dpdk.h b/platform/linux-dpdk/include/odp_packet_dpdk.h
deleted file mode 100644
index a2c9132b9..000000000
--- a/platform/linux-dpdk/include/odp_packet_dpdk.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/* Copyright (c) 2013-2018, Linaro Limited
- * All rights reserved.
- *
- * SPDX-License-Identifier: BSD-3-Clause
- */
-
-#ifndef ODP_PACKET_DPDK_H
-#define ODP_PACKET_DPDK_H
-
-#include <stdint.h>
-#include <net/if.h>
-
-#include <protocols/eth.h>
-#include <odp/api/align.h>
-#include <odp/api/debug.h>
-#include <odp/api/packet.h>
-#include <odp_packet_internal.h>
-#include <odp/api/pool.h>
-#include <odp_pool_internal.h>
-#include <odp_buffer_internal.h>
-#include <odp/api/std_types.h>
-
-#include <rte_config.h>
-#include <rte_memory.h>
-#include <rte_memzone.h>
-#include <rte_launch.h>
-#include <rte_tailq.h>
-#include <rte_eal.h>
-#include <rte_per_lcore.h>
-#include <rte_lcore.h>
-#include <rte_branch_prediction.h>
-#include <rte_prefetch.h>
-#include <rte_cycles.h>
-#include <rte_errno.h>
-#include <rte_debug.h>
-#include <rte_log.h>
-#include <rte_byteorder.h>
-#include <rte_pci.h>
-#include <rte_random.h>
-#include <rte_ether.h>
-#include <rte_ethdev.h>
-#include <rte_ip.h>
-#include <rte_ip_frag.h>
-#include <rte_udp.h>
-#include <rte_tcp.h>
-#include <rte_hash.h>
-#include <rte_jhash.h>
-#include <rte_hash_crc.h>
-
-#endif
diff --git a/platform/linux-dpdk/include/odp_packet_internal.h b/platform/linux-dpdk/include/odp_packet_internal.h
index 328b499ff..2268833f8 100644
--- a/platform/linux-dpdk/include/odp_packet_internal.h
+++ b/platform/linux-dpdk/include/odp_packet_internal.h
@@ -142,6 +142,11 @@ static inline odp_packet_hdr_t *packet_hdr(odp_packet_t pkt)
return (odp_packet_hdr_t *)(uintptr_t)pkt;
}
+static inline odp_packet_t packet_handle(odp_packet_hdr_t *pkt_hdr)
+{
+ return (odp_packet_t)pkt_hdr;
+}
+
static inline struct rte_mbuf *pkt_to_mbuf(odp_packet_t pkt)
{
return (struct rte_mbuf *)(uintptr_t)pkt;
@@ -256,6 +261,10 @@ int _odp_packet_set_data(odp_packet_t pkt, uint32_t offset,
int _odp_packet_cmp_data(odp_packet_t pkt, uint32_t offset,
const void *s, uint32_t len);
+int _odp_packet_ipv4_chksum_insert(odp_packet_t pkt);
+int _odp_packet_tcp_chksum_insert(odp_packet_t pkt);
+int _odp_packet_udp_chksum_insert(odp_packet_t pkt);
+
/* We can't enforce tailroom reservation for received packets */
ODP_STATIC_ASSERT(CONFIG_PACKET_TAILROOM == 0,
"ERROR: Tailroom has to be 0, DPDK doesn't support this");
diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h b/platform/linux-dpdk/include/odp_packet_io_internal.h
index 963ea7ff9..6160cfb66 100644
--- a/platform/linux-dpdk/include/odp_packet_io_internal.h
+++ b/platform/linux-dpdk/include/odp_packet_io_internal.h
@@ -31,7 +31,6 @@ extern "C" {
#define PKTIO_MAX_QUEUES 64
#include <linux/if_ether.h>
-#include <odp_packet_dpdk.h>
#include <odp_packet_null.h>
#define PKTIO_NAME_LEN 256
diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c
index 6b323b5e2..8be6bc146 100644
--- a/platform/linux-dpdk/odp_crypto.c
+++ b/platform/linux-dpdk/odp_crypto.c
@@ -21,6 +21,8 @@
#include <odp/api/packet.h>
#include <odp/api/plat/packet_inlines.h>
#include <odp_packet_internal.h>
+
+#include <rte_config.h>
#include <rte_crypto.h>
#include <rte_cryptodev.h>
diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c
index 7a51ff23b..2729d6a6f 100644
--- a/platform/linux-dpdk/odp_init.c
+++ b/platform/linux-dpdk/odp_init.c
@@ -7,7 +7,6 @@
#include "config.h"
#include <odp_posix_extensions.h>
-#include <odp_packet_dpdk.h>
#include <odp/api/init.h>
#include <odp_debug_internal.h>
#include <odp/api/debug.h>
@@ -25,6 +24,11 @@
#include <errno.h>
#include <sys/types.h>
#include <pwd.h>
+#include <ctype.h>
+
+#include <rte_config.h>
+#include <rte_eal.h>
+#include <rte_string_fns.h>
#define MEMPOOL_OPS(hdl) extern void mp_hdlr_init_##hdl(void)
MEMPOOL_OPS(ops_mp_mc);
@@ -50,23 +54,6 @@ void refer_constructors(void)
}
#endif
-static void print_dpdk_env_help(void)
-{
- char prgname[] = "odpdpdk";
- char help_str[] = "--help";
- char *dpdk_argv[] = {prgname, help_str};
- int dpdk_argc = 2;
-
- ODP_ERR("Neither (char *)platform_params were provided to "
- "odp_init_global(),\n");
- ODP_ERR("nor ODP_PLATFORM_PARAMS environment variable were "
- "specified.\n");
- ODP_ERR("A string of DPDK command line arguments should be provided");
- ODP_ERR("Example: export ODP_PLATFORM_PARAMS=\"-n 4 --no-huge\"\n");
- ODP_ERR("Note: -c argument substitutes automatically from odp coremask\n");
- rte_eal_init(dpdk_argc, dpdk_argv);
-}
-
static int odp_init_dpdk(const char *cmdline)
{
char **dpdk_argv;
@@ -80,10 +67,8 @@ static int odp_init_dpdk(const char *cmdline)
if (cmdline == NULL) {
cmdline = getenv("ODP_PLATFORM_PARAMS");
- if (cmdline == NULL) {
- print_dpdk_env_help();
- return -1;
- }
+ if (cmdline == NULL)
+ cmdline = "";
}
CPU_ZERO(&original_cpuset);
diff --git a/platform/linux-dpdk/odp_libconfig.c b/platform/linux-dpdk/odp_libconfig.c
deleted file mode 100644
index 6d5ee524e..000000000
--- a/platform/linux-dpdk/odp_libconfig.c
+++ /dev/null
@@ -1,99 +0,0 @@
-/* Copyright (c) 2018, Linaro Limited
- * All rights reserved.
- *
- * SPDX-License-Identifier: BSD-3-Clause
- */
-
-#include "config.h"
-
-#include <stdlib.h>
-#include <string.h>
-#include <libconfig.h>
-
-#include <odp/api/version.h>
-#include <odp_internal.h>
-#include <odp_debug_internal.h>
-#include <odp_libconfig_internal.h>
-#include <odp_libconfig_config.h>
-
-#define CONF_STR_NAME ((const char *)odp_linux_dpdk_conf)
-
-extern struct odp_global_data_s odp_global_data;
-
-int _odp_libconfig_init_global(void)
-{
- const char *filename;
- const char *vers;
- const char *vers_rt;
- const char *ipml;
- const char *ipml_rt;
- config_t *config = &odp_global_data.libconfig_default;
- config_t *config_rt = &odp_global_data.libconfig_runtime;
-
- config_init(config);
- config_init(config_rt);
-
- if (!config_read_string(config, CONF_STR_NAME)) {
- ODP_ERR("Failed to read default config: %s(%d): %s\n",
- config_error_file(config), config_error_line(config),
- config_error_text(config));
- goto fail;
- }
-
- filename = getenv("ODP_CONFIG_FILE");
- if (filename == NULL)
- return 0;
-
- if (!config_read_file(config_rt, filename)) {
- ODP_ERR("Failed to read config file: %s(%d): %s\n",
- config_error_file(config_rt),
- config_error_line(config_rt),
- config_error_text(config_rt));
- goto fail;
- }
-
- /* Check runtime configuration's implementation name and version */
- if (!config_lookup_string(config, "odp_implementation", &ipml) ||
- !config_lookup_string(config_rt, "odp_implementation", &ipml_rt)) {
- ODP_ERR("Configuration missing 'odp_implementation' field\n");
- goto fail;
- }
- if (!config_lookup_string(config, "config_file_version", &vers) ||
- !config_lookup_string(config_rt, "config_file_version", &vers_rt)) {
- ODP_ERR("Configuration missing 'config_file_version' field\n");
- goto fail;
- }
- if (strcmp(vers, vers_rt) || strcmp(ipml, ipml_rt)) {
- ODP_ERR("Runtime configuration mismatch\n");
- goto fail;
- }
-
- return 0;
-fail:
- config_destroy(config);
- config_destroy(config_rt);
- return -1;
-}
-
-int _odp_libconfig_term_global(void)
-{
- config_destroy(&odp_global_data.libconfig_default);
- config_destroy(&odp_global_data.libconfig_runtime);
-
- return 0;
-}
-
-int _odp_libconfig_lookup_int(const char *path, int *value)
-{
- int ret_def = CONFIG_FALSE;
- int ret_rt = CONFIG_FALSE;
-
- ret_def = config_lookup_int(&odp_global_data.libconfig_default, path,
- value);
-
- /* Runtime option overrides default value */
- ret_rt = config_lookup_int(&odp_global_data.libconfig_runtime, path,
- value);
-
- return (ret_def == CONFIG_TRUE || ret_rt == CONFIG_TRUE) ? 1 : 0;
-}
diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c
index e0d565dc2..2f3bea21b 100644
--- a/platform/linux-dpdk/odp_packet.c
+++ b/platform/linux-dpdk/odp_packet.c
@@ -1140,6 +1140,93 @@ int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt)
return dst_size < src_size;
}
+/* Simple implementation of ones complement sum.
+ * Based on RFC1071 and its errata.
+ */
+typedef union {
+ uint16_t w;
+ uint8_t b[2];
+} swap_buf_t;
+
+static uint32_t segment_sum16_32(const uint8_t *p,
+ uint32_t len,
+ uint32_t offset)
+
+{
+ uint32_t sum = 0;
+
+ /* Include second part of 16-bit short word split between segments */
+ if (len > 0 && (offset % 2)) {
+ swap_buf_t sw;
+
+ sw.b[0] = 0;
+ sw.b[1] = *p++;
+ sum = sw.w;
+ len--;
+ }
+
+ /*
+ * If pointer is 16-bit aligned, we can do fast path calculation.
+ * If it is not, we sum hi and lo bytes separately and then sum them.
+ */
+ if ((uintptr_t)p % 2) {
+ uint32_t sum1 = 0, sum2 = 0;
+
+ while (len > 1) {
+ sum1 += *p++;
+ sum2 += *p++;
+ len -= 2;
+ }
+#if (ODP_BYTE_ORDER == ODP_BIG_ENDIAN)
+ sum += sum2 + (sum1 << 8);
+#else
+ sum += sum1 + (sum2 << 8);
+#endif
+ } else {
+ while (len > 1) {
+ sum += *(const uint16_t *)(uintptr_t)p;
+ p += 2;
+ len -= 2;
+ }
+ }
+
+ /* Add left-over byte, if any */
+ if (len > 0) {
+ swap_buf_t sw;
+
+ sw.b[0] = *p;
+ sw.b[1] = 0;
+ sum += sw.w;
+ }
+
+ return sum;
+}
+
+static uint32_t packet_sum16_32(odp_packet_hdr_t *pkt_hdr,
+ uint32_t offset,
+ uint32_t len)
+{
+ uint32_t sum = 0;
+
+ if (offset + len > packet_len(pkt_hdr))
+ return 0;
+
+ while (len > 0) {
+ uint32_t seglen = 0; /* GCC */
+ void *mapaddr = _odp_packet_offset(packet_handle(pkt_hdr),
+ offset, &seglen, NULL);
+
+ if (seglen > len)
+ seglen = len;
+
+ sum += segment_sum16_32(mapaddr, seglen, offset);
+ len -= seglen;
+ offset += seglen;
+ }
+
+ return sum;
+}
+
/** Parser helper function for Ethernet packets */
static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr,
uint32_t *offset, uint32_t frame_len)
@@ -1496,6 +1583,158 @@ int packet_parse_common(packet_parser_t *prs, const uint8_t *ptr,
seg_len, layer, ethtype);
}
+static inline int packet_ipv4_chksum(odp_packet_t pkt,
+ uint32_t offset,
+ _odp_ipv4hdr_t *ip,
+ odp_u16sum_t *chksum)
+{
+ unsigned int nleft = _ODP_IPV4HDR_IHL(ip->ver_ihl) * 4;
+ uint16_t buf[nleft / 2];
+ int res;
+
+ if (odp_unlikely(nleft < sizeof(*ip)))
+ return -1;
+ ip->chksum = 0;
+ memcpy(buf, ip, sizeof(*ip));
+ res = odp_packet_copy_to_mem(pkt, offset + sizeof(*ip),
+ nleft - sizeof(*ip),
+ buf + sizeof(*ip) / 2);
+ if (odp_unlikely(res < 0))
+ return res;
+
+ *chksum = ~odp_chksum_ones_comp16(buf, nleft);
+
+ return 0;
+}
+
+#define _ODP_IPV4HDR_CSUM_OFFSET ODP_OFFSETOF(_odp_ipv4hdr_t, chksum)
+#define _ODP_IPV4ADDR_OFFSSET ODP_OFFSETOF(_odp_ipv4hdr_t, src_addr)
+#define _ODP_IPV6ADDR_OFFSSET ODP_OFFSETOF(_odp_ipv6hdr_t, src_addr)
+#define _ODP_IPV4HDR_CSUM_OFFSET ODP_OFFSETOF(_odp_ipv4hdr_t, chksum)
+#define _ODP_UDP_LEN_OFFSET ODP_OFFSETOF(_odp_udphdr_t, length)
+#define _ODP_UDP_CSUM_OFFSET ODP_OFFSETOF(_odp_udphdr_t, chksum)
+
+/**
+ * Calculate and fill in IPv4 checksum
+ *
+ * @param pkt ODP packet
+ *
+ * @retval 0 on success
+ * @retval <0 on failure
+ */
+int _odp_packet_ipv4_chksum_insert(odp_packet_t pkt)
+{
+ uint32_t offset;
+ _odp_ipv4hdr_t ip;
+ odp_u16sum_t chksum;
+ int res;
+
+ offset = odp_packet_l3_offset(pkt);
+ if (offset == ODP_PACKET_OFFSET_INVALID)
+ return -1;
+
+ res = odp_packet_copy_to_mem(pkt, offset, sizeof(ip), &ip);
+ if (odp_unlikely(res < 0))
+ return res;
+
+ res = packet_ipv4_chksum(pkt, offset, &ip, &chksum);
+ if (odp_unlikely(res < 0))
+ return res;
+
+ return odp_packet_copy_from_mem(pkt,
+ offset + _ODP_IPV4HDR_CSUM_OFFSET,
+ 2, &chksum);
+}
+
+static int _odp_packet_tcp_udp_chksum_insert(odp_packet_t pkt, uint16_t proto)
+{
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+ uint32_t zero = 0;
+ uint32_t sum;
+ uint16_t l3_ver;
+ uint16_t chksum;
+ uint32_t chksum_offset;
+ uint32_t frame_len = packet_len(pkt_hdr);
+
+ if (pkt_hdr->p.l3_offset == ODP_PACKET_OFFSET_INVALID)
+ return -1;
+ if (pkt_hdr->p.l4_offset == ODP_PACKET_OFFSET_INVALID)
+ return -1;
+
+ odp_packet_copy_to_mem(pkt, pkt_hdr->p.l3_offset, 2, &l3_ver);
+
+ if (_ODP_IPV4HDR_VER(l3_ver) == _ODP_IPV4)
+ sum = packet_sum16_32(pkt_hdr,
+ pkt_hdr->p.l3_offset +
+ _ODP_IPV4ADDR_OFFSSET,
+ 2 * _ODP_IPV4ADDR_LEN);
+ else
+ sum = packet_sum16_32(pkt_hdr,
+ pkt_hdr->p.l3_offset +
+ _ODP_IPV6ADDR_OFFSSET,
+ 2 * _ODP_IPV6ADDR_LEN);
+#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
+ sum += proto;
+#else
+ sum += proto << 8;
+#endif
+
+ if (proto == _ODP_IPPROTO_TCP) {
+ sum += _odp_cpu_to_be_16(frame_len - pkt_hdr->p.l4_offset);
+ chksum_offset = pkt_hdr->p.l4_offset + _ODP_UDP_CSUM_OFFSET;
+ } else {
+ sum += packet_sum16_32(pkt_hdr,
+ pkt_hdr->p.l4_offset +
+ _ODP_UDP_LEN_OFFSET,
+ 2);
+ chksum_offset = pkt_hdr->p.l4_offset + _ODP_UDP_CSUM_OFFSET;
+ }
+ odp_packet_copy_from_mem(pkt, chksum_offset, 2, &zero);
+
+ sum += packet_sum16_32(pkt_hdr,
+ pkt_hdr->p.l4_offset,
+ frame_len - pkt_hdr->p.l4_offset);
+
+ /* Not more than two additions */
+ sum = (sum & 0xffff) + (sum >> 16);
+ sum = (sum & 0xffff) + (sum >> 16);
+
+ chksum = ~sum;
+
+ if (proto == _ODP_IPPROTO_UDP && chksum == 0)
+ chksum = 0xffff;
+
+ return odp_packet_copy_from_mem(pkt,
+ chksum_offset,
+ 2, &chksum);
+}
+
+/**
+ * Calculate and fill in TCP checksum
+ *
+ * @param pkt ODP packet
+ *
+ * @retval 0 on success
+ * @retval <0 on failure
+ */
+int _odp_packet_tcp_chksum_insert(odp_packet_t pkt)
+{
+ return _odp_packet_tcp_udp_chksum_insert(pkt, _ODP_IPPROTO_TCP);
+}
+
+/**
+ * Calculate and fill in UDP checksum
+ *
+ * @param pkt ODP packet
+ *
+ * @retval 0 on success
+ * @retval <0 on failure
+ */
+int _odp_packet_udp_chksum_insert(odp_packet_t pkt)
+{
+ return _odp_packet_tcp_udp_chksum_insert(pkt, _ODP_IPPROTO_UDP);
+}
+
/**
* Simple packet parser
*/
diff --git a/platform/linux-dpdk/odp_packet_dpdk.c b/platform/linux-dpdk/odp_packet_dpdk.c
index 1c85577fa..eae2da1dc 100644
--- a/platform/linux-dpdk/odp_packet_dpdk.c
+++ b/platform/linux-dpdk/odp_packet_dpdk.c
@@ -32,9 +32,17 @@
#include <odp_classification_internal.h>
#include <odp_packet_io_internal.h>
#include <odp_libconfig_internal.h>
-#include <odp_packet_dpdk.h>
+#include <odp/api/plat/packet_inlines.h>
#include <net/if.h>
+#include <protocols/udp.h>
+
+#include <rte_config.h>
+#include <rte_ethdev.h>
+#include <rte_ip_frag.h>
+#include <rte_udp.h>
+#include <rte_tcp.h>
+
/* DPDK poll mode drivers requiring minimum RX burst size DPDK_MIN_RX_BURST */
#define IXGBE_DRV_NAME "net_ixgbe"
#define I40E_DRV_NAME "net_i40e"
@@ -57,22 +65,16 @@ extern void *pktio_entry_ptr[ODP_CONFIG_PKTIO_ENTRIES];
static uint32_t mtu_get_pkt_dpdk(pktio_entry_t *pktio_entry);
-static int lookup_opt(const char *path, const char *drv_name, int *val)
+static int lookup_opt(const char *opt_name, const char *drv_name, int *val)
{
const char *base = "pktio_dpdk";
- char opt_path[256];
- int ret = 0;
-
- /* Default option */
- snprintf(opt_path, sizeof(opt_path), "%s.%s", base, path);
- ret += _odp_libconfig_lookup_int(opt_path, val);
-
- /* Driver specific option overrides default option */
- snprintf(opt_path, sizeof(opt_path), "%s.%s.%s", base, drv_name, path);
- ret += _odp_libconfig_lookup_int(opt_path, val);
+ int ret;
+ ret = _odp_libconfig_lookup_ext_int(base, drv_name, opt_name, val);
if (ret == 0)
- ODP_ERR("Unable to find DPDK configuration option: %s\n", path);
+ ODP_ERR("Unable to find DPDK configuration option: %s\n",
+ opt_name);
+
return ret;
}
@@ -385,8 +387,9 @@ static int setup_pkt_dpdk(odp_pktio_t pktio ODP_UNUSED,
return 0;
}
-static int close_pkt_dpdk(pktio_entry_t *pktio_entry ODP_UNUSED)
+static int close_pkt_dpdk(pktio_entry_t *pktio_entry)
{
+ rte_eth_dev_stop(pktio_entry->s.pkt_dpdk.port_id);
return 0;
}
@@ -441,6 +444,10 @@ static int start_pkt_dpdk(pktio_entry_t *pktio_entry)
struct rte_eth_txconf *txconf = NULL;
uint32_t txq_flags = 0;
+ if (pktio_entry->s.state == PKTIO_STATE_STOPPED ||
+ pktio_entry->s.state == PKTIO_STATE_STOP_PENDING)
+ rte_eth_dev_stop(pkt_dpdk->port_id);
+
/* DPDK doesn't support nb_rx_q/nb_tx_q being 0 */
if (!pktio_entry->s.num_in_queue)
pktio_entry->s.num_in_queue = 1;
@@ -548,7 +555,14 @@ static int start_pkt_dpdk(pktio_entry_t *pktio_entry)
static int stop_pkt_dpdk(pktio_entry_t *pktio_entry)
{
- rte_eth_dev_stop(pktio_entry->s.pkt_dpdk.port_id);
+ unsigned int i;
+ uint16_t port_id = pktio_entry->s.pkt_dpdk.port_id;
+
+ for (i = 0; i < pktio_entry->s.num_in_queue; i++)
+ rte_eth_dev_rx_queue_stop(port_id, i);
+ for (i = 0; i < pktio_entry->s.num_out_queue; i++)
+ rte_eth_dev_tx_queue_stop(port_id, i);
+
return 0;
}
@@ -590,6 +604,7 @@ static void _odp_pktio_send_completion(pktio_entry_t *pktio_entry)
#define IP4_CSUM_RESULT(m) (m->ol_flags & PKT_RX_IP_CKSUM_MASK)
#define L4_CSUM_RESULT(m) (m->ol_flags & PKT_RX_L4_CKSUM_MASK)
#define HAS_L4_PROTO(m, proto) ((m->packet_type & RTE_PTYPE_L4_MASK) == proto)
+#define UDP4_CSUM(_p) (((_odp_udphdr_t *)_odp_packet_l4_ptr(_p, NULL))->chksum)
#define PKTIN_CSUM_BITS 0x1C
@@ -622,6 +637,13 @@ static inline int pkt_set_ol_rx(odp_pktin_config_opt_t *pktin_cfg,
if (packet_csum_result == PKT_RX_L4_CKSUM_GOOD) {
pkt_hdr->p.input_flags.l4_chksum_done = 1;
} else if (packet_csum_result != PKT_RX_L4_CKSUM_UNKNOWN) {
+ if (pkt_hdr->p.input_flags.ipv4 &&
+ pkt_hdr->p.input_flags.udp &&
+ !UDP4_CSUM(packet_handle(pkt_hdr))) {
+ pkt_hdr->p.input_flags.l4_chksum_done = 1;
+ return 0;
+ }
+
if (pktin_cfg->bit.drop_udp_err)
return -1;
@@ -860,11 +882,7 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg,
if (!ipv4_chksum_pkt && !udp_chksum_pkt && !tcp_chksum_pkt)
return;
- if (pkt_p->l4_offset == ODP_PACKET_OFFSET_INVALID)
- return;
-
mbuf->l2_len = pkt_p->l3_offset - pkt_p->l2_offset;
- mbuf->l3_len = pkt_p->l4_offset - pkt_p->l3_offset;
if (l3_proto_v4)
mbuf->ol_flags = PKT_TX_IPV4;
@@ -875,8 +893,14 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg,
mbuf->ol_flags |= PKT_TX_IP_CKSUM;
((struct ipv4_hdr *)l3_hdr)->hdr_checksum = 0;
+ mbuf->l3_len = _ODP_IPV4HDR_IHL(*(uint8_t *)l3_hdr) * 4;
}
+ if (pkt_p->l4_offset == ODP_PACKET_OFFSET_INVALID)
+ return;
+
+ mbuf->l3_len = pkt_p->l4_offset - pkt_p->l3_offset;
+
l4_hdr = (void *)(mbuf_data + pkt_p->l4_offset);
if (udp_chksum_pkt) {
diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c
index 463d7aedb..d5b3195a9 100644
--- a/platform/linux-dpdk/odp_pool.c
+++ b/platform/linux-dpdk/odp_pool.c
@@ -29,8 +29,7 @@
#include <odp/api/plat/pool_inline_types.h>
-/* for DPDK */
-#include <odp_packet_dpdk.h>
+#include <rte_config.h>
#include <rte_version.h>
#ifdef POOL_USE_TICKETLOCK
diff --git a/platform/linux-dpdk/odp_queue_basic.c b/platform/linux-dpdk/odp_queue_basic.c
index 5df48ecc2..67303c185 100644
--- a/platform/linux-dpdk/odp_queue_basic.c
+++ b/platform/linux-dpdk/odp_queue_basic.c
@@ -25,6 +25,7 @@
#include <odp/api/hints.h>
#include <odp/api/sync.h>
#include <odp/api/traffic_mngr.h>
+#include <odp_libconfig_internal.h>
#define NUM_INTERNAL_QUEUES 64
@@ -36,6 +37,9 @@
#include <string.h>
#include <inttypes.h>
+#define MIN_QUEUE_SIZE 8
+#define MAX_QUEUE_SIZE (1 * 1024 * 1024)
+
static int queue_init(queue_entry_t *queue, const char *name,
const odp_queue_param_t *param);
@@ -60,11 +64,11 @@ static int queue_capa(odp_queue_capability_t *capa, int sched)
/* Reserve some queues for internal use */
capa->max_queues = ODP_CONFIG_QUEUES - NUM_INTERNAL_QUEUES;
capa->plain.max_num = capa->max_queues;
- capa->plain.max_size = CONFIG_QUEUE_SIZE - 1;
+ capa->plain.max_size = queue_glb->config.max_queue_size - 1;
capa->plain.lockfree.max_num = queue_glb->queue_lf_num;
capa->plain.lockfree.max_size = queue_glb->queue_lf_size;
capa->sched.max_num = capa->max_queues;
- capa->sched.max_size = CONFIG_QUEUE_SIZE - 1;
+ capa->sched.max_size = queue_glb->config.max_queue_size - 1;
if (sched) {
capa->max_ordered_locks = sched_fn->max_ordered_locks();
@@ -75,6 +79,52 @@ static int queue_capa(odp_queue_capability_t *capa, int sched)
return 0;
}
+static int read_config_file(queue_global_t *queue_glb)
+{
+ const char *str;
+ uint32_t val_u32;
+ int val = 0;
+
+ ODP_PRINT("Queue config:\n");
+
+ str = "queue_basic.max_queue_size";
+ if (!_odp_libconfig_lookup_int(str, &val)) {
+ ODP_ERR("Config option '%s' not found.\n", str);
+ return -1;
+ }
+
+ val_u32 = val;
+
+ if (val_u32 > MAX_QUEUE_SIZE || val_u32 < MIN_QUEUE_SIZE ||
+ !CHECK_IS_POWER2(val_u32)) {
+ ODP_ERR("Bad value %s = %u\n", str, val_u32);
+ return -1;
+ }
+
+ queue_glb->config.max_queue_size = val_u32;
+ ODP_PRINT(" %s: %u\n", str, val_u32);
+
+ str = "queue_basic.default_queue_size";
+ if (!_odp_libconfig_lookup_int(str, &val)) {
+ ODP_ERR("Config option '%s' not found.\n", str);
+ return -1;
+ }
+
+ val_u32 = val;
+
+ if (val_u32 > queue_glb->config.max_queue_size ||
+ val_u32 < MIN_QUEUE_SIZE ||
+ !CHECK_IS_POWER2(val_u32)) {
+ ODP_ERR("Bad value %s = %u\n", str, val_u32);
+ return -1;
+ }
+
+ queue_glb->config.default_queue_size = val_u32;
+ ODP_PRINT(" %s: %u\n\n", str, val_u32);
+
+ return 0;
+}
+
static int queue_init_global(void)
{
uint32_t i;
@@ -85,7 +135,7 @@ static int queue_init_global(void)
ODP_DBG("Starts...\n");
- shm = odp_shm_reserve("odp_queues",
+ shm = odp_shm_reserve("_odp_queue_gbl",
sizeof(queue_global_t),
sizeof(queue_entry_t), 0);
@@ -105,6 +155,15 @@ static int queue_init_global(void)
queue->s.handle = queue_from_index(i);
}
+ if (read_config_file(queue_glb)) {
+ odp_shm_free(shm);
+ return -1;
+ }
+
+ queue_glb->queue_gbl_shm = shm;
+ queue_glb->queue_ring_shm = ODP_SHM_INVALID;
+ queue_glb->ring_data = NULL;
+
lf_func = &queue_glb->queue_lf_func;
queue_glb->queue_lf_num = queue_lf_init_global(&lf_size, lf_func);
queue_glb->queue_lf_size = lf_size;
@@ -134,7 +193,6 @@ static int queue_term_local(void)
static int queue_term_global(void)
{
int ret = 0;
- int rc = 0;
queue_entry_t *queue;
int i;
@@ -143,20 +201,19 @@ static int queue_term_global(void)
LOCK(queue);
if (queue->s.status != QUEUE_STATUS_FREE) {
ODP_ERR("Not destroyed queue: %s\n", queue->s.name);
- rc = -1;
+ ret = -1;
}
UNLOCK(queue);
}
queue_lf_term_global();
- ret = odp_shm_free(odp_shm_lookup("odp_queues"));
- if (ret < 0) {
- ODP_ERR("shm free failed for odp_queues");
- rc = -1;
+ if (odp_shm_free(queue_glb->queue_gbl_shm)) {
+ ODP_ERR("shm free failed");
+ ret = -1;
}
- return rc;
+ return ret;
}
static int queue_capability(odp_queue_capability_t *capa)
@@ -208,7 +265,7 @@ static odp_queue_t queue_create(const char *name,
}
if (param->nonblocking == ODP_BLOCKING) {
- if (param->size > CONFIG_QUEUE_SIZE)
+ if (param->size > queue_glb->config.max_queue_size)
return ODP_QUEUE_INVALID;
} else if (param->nonblocking == ODP_NONBLOCKING_LF) {
/* Only plain type lock-free queues supported */
@@ -558,6 +615,8 @@ static odp_event_t queue_deq(odp_queue_t handle)
static int queue_init(queue_entry_t *queue, const char *name,
const odp_queue_param_t *param)
{
+ uint32_t queue_size;
+
if (name == NULL) {
queue->s.name[0] = 0;
} else {
@@ -581,7 +640,21 @@ static int queue_init(queue_entry_t *queue, const char *name,
queue->s.pktin = PKTIN_INVALID;
queue->s.pktout = PKTOUT_INVALID;
- queue->s.ring_st = ring_st_create(queue->s.name, CONFIG_QUEUE_SIZE);
+ /* Use default size for all small queues to quarantee performance
+ * level. */
+ queue_size = queue_glb->config.default_queue_size;
+ if (param->size > queue_glb->config.default_queue_size)
+ queue_size = param->size;
+
+ /* Round up if not already a power of two */
+ queue_size = ROUNDUP_POWER2_U32(queue_size);
+
+ if (queue_size > queue_glb->config.max_queue_size) {
+ ODP_ERR("Too large queue size %u\n", queue_size);
+ return -1;
+ }
+
+ queue->s.ring_st = ring_st_create(queue->s.name, queue_size);
if (queue->s.ring_st == NULL)
return -1;
diff --git a/platform/linux-dpdk/odp_time.c b/platform/linux-dpdk/odp_time.c
index 3cbb771d8..e4c82b756 100644
--- a/platform/linux-dpdk/odp_time.c
+++ b/platform/linux-dpdk/odp_time.c
@@ -11,10 +11,12 @@
#include <odp/api/hints.h>
#include <odp_debug_internal.h>
#include <odp_arch_time_internal.h>
-#include <rte_cycles.h>
#include <string.h>
#include <inttypes.h>
+#include <rte_config.h>
+#include <rte_cycles.h>
+
typedef uint64_t (*time_to_ns_fn) (odp_time_t time);
typedef odp_time_t (*time_cur_fn)(void);
typedef odp_time_t (*time_from_ns_fn) (uint64_t ns);
diff --git a/platform/linux-dpdk/test/validation/api/pktio/pktio_run.sh b/platform/linux-dpdk/test/validation/api/pktio/pktio_run.sh
index 6007195e1..d051fa801 100755
--- a/platform/linux-dpdk/test/validation/api/pktio/pktio_run.sh
+++ b/platform/linux-dpdk/test/validation/api/pktio/pktio_run.sh
@@ -85,7 +85,7 @@ run()
echo "Failed to setup test environment, skipping test."
exit $TEST_SKIPPED
fi
- export ODP_PLATFORM_PARAMS="-n 4 --no-pci --vdev net_pcap0,iface=$IF0 --vdev net_pcap1,iface=$IF1"
+ export ODP_PLATFORM_PARAMS="--no-pci --vdev net_pcap0,iface=$IF0 --vdev net_pcap1,iface=$IF1"
export ODP_PKTIO_IF0=0
export ODP_PKTIO_IF1=1
fi
diff --git a/platform/linux-dpdk/test/wrapper-script.sh b/platform/linux-dpdk/test/wrapper-script.sh
index 23ebec202..b092e1064 100755
--- a/platform/linux-dpdk/test/wrapper-script.sh
+++ b/platform/linux-dpdk/test/wrapper-script.sh
@@ -1,6 +1,6 @@
#!/bin/bash
-export ODP_PLATFORM_PARAMS=${ODP_PLATFORM_PARAMS:--n 4 --vdev "crypto_openssl" --vdev crypto_null}
+export ODP_PLATFORM_PARAMS=${ODP_PLATFORM_PARAMS:---vdev="crypto_openssl" --vdev="crypto_null"}
# where to mount huge pages
export HUGEPAGEDIR=${HUGEPAGEDIR:-/mnt/huge}
# exit codes expected by automake for skipped tests
@@ -52,21 +52,14 @@ if [ "$(id -u)" != "0" ]; then
fi
echo "Mounting hugetlbfs"
-export SIZE=1G
-export SIZE_KB=1048576
-export RESERVE=1
+export SIZE=2MB
+export SIZE_KB=2048
+export RESERVE=512
mount_and_reserve
res=$?
if [ $res -ne 0 ]; then
- export SIZE=2MB
- export SIZE_KB=2048
- export RESERVE=1024
- mount_and_reserve
- res=$?
- if [ $res -ne 0 ]; then
- echo "ERROR: can't mount hugepages with any size"
- exit $res
- fi
+ echo "ERROR: can't mount hugepages"
+ exit $res
fi
echo "running $1!"
$1
diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am
index 7e57994d1..4f03eab15 100644
--- a/platform/linux-generic/Makefile.am
+++ b/platform/linux-generic/Makefile.am
@@ -82,15 +82,11 @@ endif
noinst_HEADERS = \
arch/odp_arch_time_internal.h \
- include/_fdserver_internal.h \
- include/_ishm_internal.h \
- include/_ishmphy_internal.h \
- include/_ishmpool_internal.h \
include/odp_align_internal.h \
include/odp_atomic_internal.h \
- include/odp_buffer_inlines.h \
include/odp_bitmap_internal.h \
include/odp_bitset.h \
+ include/odp_buffer_inlines.h \
include/odp_buffer_internal.h \
include/odp_classification_datamodel.h \
include/odp_classification_inlines.h \
@@ -98,37 +94,41 @@ noinst_HEADERS = \
include/odp_config_internal.h \
include/odp_debug_internal.h \
include/odp_errno_define.h \
+ include/odp_fdserver_internal.h \
include/odp_forward_typedefs_internal.h \
include/odp_internal.h \
include/odp_ipsec_internal.h \
+ include/odp_ishm_internal.h \
+ include/odp_ishmphy_internal.h \
+ include/odp_ishmpool_internal.h \
include/odp_libconfig_internal.h \
include/odp_llqueue.h \
include/odp_macros_internal.h \
include/odp_name_table_internal.h \
+ include/odp_packet_dpdk.h \
include/odp_packet_internal.h \
include/odp_packet_io_internal.h \
include/odp_packet_io_ipc_internal.h \
include/odp_packet_io_ring_internal.h \
include/odp_packet_netmap.h \
- include/odp_packet_dpdk.h \
+ include/odp_packet_null.h \
include/odp_packet_socket.h \
include/odp_packet_tap.h \
- include/odp_packet_null.h \
include/odp_pkt_queue_internal.h \
include/odp_pool_internal.h \
include/odp_posix_extensions.h \
- include/odp_queue_internal.h \
- include/odp_queue_scalable_internal.h \
include/odp_queue_if.h \
+ include/odp_queue_internal.h \
include/odp_queue_lf.h \
+ include/odp_queue_scalable_internal.h \
include/odp_ring_internal.h \
include/odp_ring_st_internal.h \
include/odp_schedule_if.h \
- include/odp_schedule_scalable.h \
include/odp_schedule_scalable_config.h \
+ include/odp_schedule_scalable.h \
include/odp_schedule_scalable_ordered.h \
- include/odp_sorted_list_internal.h \
include/odp_shm_internal.h \
+ include/odp_sorted_list_internal.h \
include/odp_timer_internal.h \
include/odp_timer_wheel_internal.h \
include/odp_traffic_mngr_internal.h \
@@ -138,15 +138,10 @@ noinst_HEADERS = \
include/protocols/tcp.h \
include/protocols/thash.h \
include/protocols/udp.h
-
nodist_noinst_HEADERS = \
include/odp_libconfig_config.h
__LIB__libodp_linux_la_SOURCES = \
- _fdserver.c \
- _ishm.c \
- _ishmphy.c \
- _ishmpool.c \
odp_atomic.c \
odp_barrier.c \
odp_bitmap.c \
@@ -159,30 +154,21 @@ __LIB__libodp_linux_la_SOURCES = \
odp_crypto.c \
odp_errno.c \
odp_event.c \
+ odp_fdserver.c \
odp_hash.c \
- odp_init.c \
odp_impl.c \
+ odp_init.c \
odp_ipsec.c \
odp_ipsec_events.c \
odp_ipsec_sad.c \
+ odp_ishm.c \
+ odp_ishmphy.c \
+ odp_ishmpool.c \
odp_libconfig.c \
odp_name_table.c \
odp_packet.c \
odp_packet_flags.c \
odp_packet_io.c \
- pktio/ethtool.c \
- pktio/io_ops.c \
- pktio/ipc.c \
- pktio/pktio_common.c \
- pktio/loop.c \
- pktio/netmap.c \
- pktio/null.c \
- pktio/dpdk.c \
- pktio/socket.c \
- pktio/socket_mmap.c \
- pktio/sysfs.c \
- pktio/tap.c \
- pktio/ring.c \
odp_pkt_queue.c \
odp_pool.c \
odp_queue_basic.c \
@@ -193,10 +179,10 @@ __LIB__libodp_linux_la_SOURCES = \
odp_rwlock_recursive.c \
odp_schedule_basic.c \
odp_schedule_if.c \
- odp_schedule_sp.c \
odp_schedule_iquery.c \
odp_schedule_scalable.c \
odp_schedule_scalable_ordered.c \
+ odp_schedule_sp.c \
odp_shared_memory.c \
odp_sorted_list.c \
odp_spinlock.c \
@@ -209,8 +195,20 @@ __LIB__libodp_linux_la_SOURCES = \
odp_timer_wheel.c \
odp_traffic_mngr.c \
odp_version.c \
- odp_weak.c
-
+ odp_weak.c \
+ pktio/dpdk.c \
+ pktio/ethtool.c \
+ pktio/io_ops.c \
+ pktio/ipc.c \
+ pktio/loop.c \
+ pktio/netmap.c \
+ pktio/null.c \
+ pktio/pktio_common.c \
+ pktio/ring.c \
+ pktio/socket.c \
+ pktio/socket_mmap.c \
+ pktio/sysfs.c \
+ pktio/tap.c
if ODP_ABI_COMPAT
__LIB__libodp_linux_la_SOURCES += \
odp_atomic_api.c \
diff --git a/platform/linux-generic/include/_fdserver_internal.h b/platform/linux-generic/include/odp_fdserver_internal.h
index 8518a5b7e..8518a5b7e 100644
--- a/platform/linux-generic/include/_fdserver_internal.h
+++ b/platform/linux-generic/include/odp_fdserver_internal.h
diff --git a/platform/linux-generic/include/_ishm_internal.h b/platform/linux-generic/include/odp_ishm_internal.h
index 56c7f5a93..56c7f5a93 100644
--- a/platform/linux-generic/include/_ishm_internal.h
+++ b/platform/linux-generic/include/odp_ishm_internal.h
diff --git a/platform/linux-generic/include/_ishmphy_internal.h b/platform/linux-generic/include/odp_ishmphy_internal.h
index 05e3fcec7..05e3fcec7 100644
--- a/platform/linux-generic/include/_ishmphy_internal.h
+++ b/platform/linux-generic/include/odp_ishmphy_internal.h
diff --git a/platform/linux-generic/include/_ishmpool_internal.h b/platform/linux-generic/include/odp_ishmpool_internal.h
index 94bcddaeb..94bcddaeb 100644
--- a/platform/linux-generic/include/_ishmpool_internal.h
+++ b/platform/linux-generic/include/odp_ishmpool_internal.h
diff --git a/platform/linux-generic/include/odp_libconfig_internal.h b/platform/linux-generic/include/odp_libconfig_internal.h
index 042917755..727f68863 100644
--- a/platform/linux-generic/include/odp_libconfig_internal.h
+++ b/platform/linux-generic/include/odp_libconfig_internal.h
@@ -22,6 +22,11 @@ int _odp_libconfig_term_global(void);
int _odp_libconfig_lookup_int(const char *path, int *value);
+int _odp_libconfig_lookup_ext_int(const char *base_path,
+ const char *local_path,
+ const char *name,
+ int *value);
+
#ifdef __cplusplus
}
#endif
diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h
index 75b4ce9e5..be17cac86 100644
--- a/platform/linux-generic/include/odp_packet_internal.h
+++ b/platform/linux-generic/include/odp_packet_internal.h
@@ -303,6 +303,10 @@ int _odp_packet_set_data(odp_packet_t pkt, uint32_t offset,
int _odp_packet_cmp_data(odp_packet_t pkt, uint32_t offset,
const void *s, uint32_t len);
+int _odp_packet_ipv4_chksum_insert(odp_packet_t pkt);
+int _odp_packet_tcp_chksum_insert(odp_packet_t pkt);
+int _odp_packet_udp_chksum_insert(odp_packet_t pkt);
+
#ifdef __cplusplus
}
#endif
diff --git a/platform/linux-generic/include/odp_packet_netmap.h b/platform/linux-generic/include/odp_packet_netmap.h
index bb81f5f85..bd3efdcf5 100644
--- a/platform/linux-generic/include/odp_packet_netmap.h
+++ b/platform/linux-generic/include/odp_packet_netmap.h
@@ -19,6 +19,12 @@
#define NM_MAX_DESC 64
+/** netmap runtime configuration options */
+typedef struct {
+ int nr_rx_slots;
+ int nr_tx_slots;
+} netmap_opt_t;
+
/** Ring for mapping pktin/pktout queues to netmap descriptors */
struct netmap_ring_t {
unsigned first; /**< Index of first netmap descriptor */
@@ -61,6 +67,7 @@ typedef struct {
netmap_ring_t rx_desc_ring[PKTIO_MAX_QUEUES];
/** mapping of pktout queues to netmap tx descriptors */
netmap_ring_t tx_desc_ring[PKTIO_MAX_QUEUES];
+ netmap_opt_t opt; /**< options */
} pkt_netmap_t;
#endif
diff --git a/platform/linux-generic/include/odp_queue_internal.h b/platform/linux-generic/include/odp_queue_internal.h
index 3aec3fe9d..98e86fa0e 100644
--- a/platform/linux-generic/include/odp_queue_internal.h
+++ b/platform/linux-generic/include/odp_queue_internal.h
@@ -63,17 +63,19 @@ union queue_entry_u {
uint8_t pad[ROUNDUP_CACHE_LINE(sizeof(struct queue_entry_s))];
};
-typedef struct ODP_ALIGNED_CACHE {
- /* Storage space for ring data */
- uint32_t data[CONFIG_QUEUE_SIZE];
-} queue_ring_data_t;
-
typedef struct queue_global_t {
- queue_entry_t queue[ODP_CONFIG_QUEUES];
- queue_ring_data_t ring_data[ODP_CONFIG_QUEUES];
- uint32_t queue_lf_num;
- uint32_t queue_lf_size;
- queue_lf_func_t queue_lf_func;
+ queue_entry_t queue[ODP_CONFIG_QUEUES];
+ uint32_t *ring_data;
+ uint32_t queue_lf_num;
+ uint32_t queue_lf_size;
+ queue_lf_func_t queue_lf_func;
+ odp_shm_t queue_gbl_shm;
+ odp_shm_t queue_ring_shm;
+
+ struct {
+ uint32_t max_queue_size;
+ uint32_t default_queue_size;
+ } config;
} queue_global_t;
diff --git a/platform/linux-generic/include/odp_schedule_scalable_ordered.h b/platform/linux-generic/include/odp_schedule_scalable_ordered.h
index fb4720a51..17d4f7eab 100644
--- a/platform/linux-generic/include/odp_schedule_scalable_ordered.h
+++ b/platform/linux-generic/include/odp_schedule_scalable_ordered.h
@@ -14,7 +14,7 @@
#include <odp_internal.h>
#include <odp_align_internal.h>
#include <odp_bitset.h>
-#include <_ishmpool_internal.h>
+#include <odp_ishmpool_internal.h>
/* High level functioning of reordering
* Datastructures -
diff --git a/platform/linux-generic/odp_crypto.c b/platform/linux-generic/odp_crypto.c
index a7fddb5b4..852f02125 100644
--- a/platform/linux-generic/odp_crypto.c
+++ b/platform/linux-generic/odp_crypto.c
@@ -35,6 +35,8 @@
#endif
#define MAX_SESSIONS 32
+#define AES_BLOCK_SIZE 16
+#define AES_KEY_LENGTH 16
/*
* Cipher algorithm capabilities
@@ -95,10 +97,18 @@ static const odp_crypto_auth_capability_t auth_capa_sha256_hmac[] = {
{.digest_len = 16, .key_len = 32, .aad_len = {.min = 0, .max = 0, .inc = 0} },
{.digest_len = 32, .key_len = 32, .aad_len = {.min = 0, .max = 0, .inc = 0} } };
+static const odp_crypto_auth_capability_t auth_capa_sha384_hmac[] = {
+{.digest_len = 24, .key_len = 48, .aad_len = {.min = 0, .max = 0, .inc = 0} },
+{.digest_len = 48, .key_len = 48, .aad_len = {.min = 0, .max = 0, .inc = 0} } };
+
static const odp_crypto_auth_capability_t auth_capa_sha512_hmac[] = {
{.digest_len = 32, .key_len = 64, .aad_len = {.min = 0, .max = 0, .inc = 0} },
{.digest_len = 64, .key_len = 64, .aad_len = {.min = 0, .max = 0, .inc = 0} } };
+static const odp_crypto_auth_capability_t auth_capa_aes_xcbc[] = {
+{.digest_len = 12, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0} },
+{.digest_len = 16, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0} } };
+
static const odp_crypto_auth_capability_t auth_capa_aes_gcm[] = {
{.digest_len = 16, .key_len = 0, .aad_len = {.min = 8, .max = 12, .inc = 4} } };
@@ -308,6 +318,169 @@ void packet_hmac(odp_packet_t pkt,
HMAC_Final(ctx, hash, NULL);
}
+static void xor_block(uint8_t *res, const uint8_t *op)
+{
+ int i;
+
+ for (i = 0; i < AES_BLOCK_SIZE; i++)
+ res[i] ^= op[i];
+}
+
+static void memxor(uint8_t *res, const uint8_t *op, size_t len)
+{
+ for (size_t i = 0; i < len; i++)
+ res[i] ^= op[i];
+}
+
+static
+void packet_aes_xcbc_mac(odp_packet_t pkt,
+ const odp_crypto_packet_op_param_t *param,
+ odp_crypto_generic_session_t *session,
+ uint8_t *hash)
+{
+ uint8_t e[AES_BLOCK_SIZE] = {0};
+ size_t eoff = 0;
+ uint32_t offset = param->auth_range.offset;
+ uint32_t len = param->auth_range.length;
+ uint32_t seglen = 0;
+ uint32_t datalen = 0;
+ int dummy_len = 0;
+ EVP_CIPHER_CTX *ctx;
+ void *mapaddr;
+ uint8_t *data = NULL;
+
+ ODP_ASSERT(offset + len <= odp_packet_len(pkt));
+ ODP_ASSERT(session != NULL);
+ ODP_ASSERT(sizeof(session->auth.key) >= 3 * AES_KEY_LENGTH);
+
+ ctx = EVP_CIPHER_CTX_new();
+ EVP_EncryptInit_ex(ctx, session->auth.evp_cipher,
+ NULL, session->auth.key, NULL);
+
+ while (len > 0) {
+ mapaddr = odp_packet_offset(pkt, offset, &seglen, NULL);
+ datalen = seglen >= len ? len : seglen;
+ data = (uint8_t *)mapaddr;
+ offset += datalen;
+ len -= datalen;
+ if (eoff != 0) {
+ if (eoff + datalen > AES_BLOCK_SIZE) {
+ memxor(e + eoff, data, AES_BLOCK_SIZE - eoff);
+ datalen -= (AES_BLOCK_SIZE - eoff);
+ eoff = 0;
+ EVP_EncryptUpdate(ctx,
+ e, &dummy_len, e, sizeof(e));
+ } else {
+ memxor(e + eoff, data, datalen);
+ eoff += datalen;
+ continue;
+ }
+ }
+ while (datalen > AES_BLOCK_SIZE) {
+ xor_block(e, data);
+ EVP_EncryptUpdate(ctx, e, &dummy_len, e, sizeof(e));
+ data += AES_BLOCK_SIZE;
+ datalen -= AES_BLOCK_SIZE;
+ }
+ /* Segmentation handle */
+ if (datalen > 0) {
+ memxor(e, data, datalen);
+ eoff = datalen;
+ }
+ }
+
+ if (eoff == AES_BLOCK_SIZE) {
+ xor_block(e, session->auth.key + AES_KEY_LENGTH);
+ } else {
+ e[eoff] ^= 0x80;
+ xor_block(e, session->auth.key + AES_KEY_LENGTH * 2);
+ }
+ EVP_EncryptUpdate(ctx, hash, &dummy_len, e, sizeof(e));
+ EVP_CIPHER_CTX_free(ctx);
+}
+
+static
+odp_crypto_alg_err_t auth_xcbcmac_gen(odp_packet_t pkt,
+ const odp_crypto_packet_op_param_t *param,
+ odp_crypto_generic_session_t *session)
+{
+ uint8_t hash[EVP_MAX_MD_SIZE];
+
+ /* Hash it */
+ packet_aes_xcbc_mac(pkt, param, session, hash);
+
+ /* Copy to the output location */
+ odp_packet_copy_from_mem(pkt,
+ param->hash_result_offset,
+ session->p.auth_digest_len,
+ hash);
+
+ return ODP_CRYPTO_ALG_ERR_NONE;
+}
+
+static odp_crypto_alg_err_t
+auth_xcbcmac_check(odp_packet_t pkt,
+ const odp_crypto_packet_op_param_t *param,
+ odp_crypto_generic_session_t *session)
+{
+ uint32_t bytes = session->p.auth_digest_len;
+ uint8_t hash_in[EVP_MAX_MD_SIZE];
+ uint8_t hash_out[EVP_MAX_MD_SIZE];
+
+ /* Copy current value out and clear it before authentication */
+ odp_packet_copy_to_mem(pkt, param->hash_result_offset,
+ bytes, hash_in);
+
+ _odp_packet_set_data(pkt, param->hash_result_offset,
+ 0, bytes);
+
+ /* Hash it */
+ packet_aes_xcbc_mac(pkt, param, session, hash_out);
+
+ /* Verify match */
+ if (0 != memcmp(hash_in, hash_out, bytes))
+ return ODP_CRYPTO_ALG_ERR_ICV_CHECK;
+
+ /* Matched */
+ return ODP_CRYPTO_ALG_ERR_NONE;
+}
+
+static int process_aesxcbc_param(odp_crypto_generic_session_t *session,
+ const EVP_CIPHER *cipher)
+{
+ uint32_t k1[4] = { 0x01010101, 0x01010101, 0x01010101, 0x01010101 };
+ uint32_t k2[4] = { 0x02020202, 0x02020202, 0x02020202, 0x02020202 };
+ uint32_t k3[4] = { 0x03030303, 0x03030303, 0x03030303, 0x03030303 };
+ EVP_CIPHER_CTX *ctx;
+ int dummy_len = 0;
+
+ /* Set function */
+ if (ODP_CRYPTO_OP_ENCODE == session->p.op)
+ session->auth.func = auth_xcbcmac_gen;
+ else
+ session->auth.func = auth_xcbcmac_check;
+ session->auth.init = null_crypto_init_routine;
+
+ session->auth.evp_cipher = cipher;
+ ctx = EVP_CIPHER_CTX_new();
+ EVP_EncryptInit_ex(ctx, session->auth.evp_cipher, NULL,
+ session->p.auth_key.data, NULL);
+ /* K1 = 0x01010101010101010101010101010101 encrypted with Key K */
+ EVP_EncryptUpdate(ctx, session->auth.key,
+ &dummy_len, (uint8_t *)k1, AES_BLOCK_SIZE);
+
+ /* K2 = 0x02020202020202020202020202020202 encrypted with Key K */
+ EVP_EncryptUpdate(ctx, session->auth.key + AES_KEY_LENGTH,
+ &dummy_len, (uint8_t *)k2, AES_BLOCK_SIZE);
+
+ /* K3 = 0x03030303030303030303030303030303 encrypted with Key K */
+ EVP_EncryptUpdate(ctx, session->auth.key + AES_KEY_LENGTH * 2,
+ &dummy_len, (uint8_t *)k3, AES_BLOCK_SIZE);
+
+ EVP_CIPHER_CTX_free(ctx);
+ return 0;
+}
+
static
odp_crypto_alg_err_t auth_hmac_gen(odp_packet_t pkt,
const odp_crypto_packet_op_param_t *param,
@@ -661,9 +834,8 @@ static int process_cipher_param(odp_crypto_generic_session_t *session,
return -1;
/* Verify IV len is correct */
- if (!((0 == session->p.cipher_iv.length) ||
- ((uint32_t)EVP_CIPHER_iv_length(cipher) ==
- session->p.cipher_iv.length)))
+ if ((uint32_t)EVP_CIPHER_iv_length(cipher) !=
+ session->p.cipher_iv.length)
return -1;
session->cipher.evp_cipher = cipher;
@@ -791,6 +963,10 @@ static int process_aes_gcm_param(odp_crypto_generic_session_t *session,
session->p.cipher_key.length)
return -1;
+ /* Verify IV len is correct */
+ if (12 != session->p.cipher_iv.length)
+ return -1;
+
memcpy(session->cipher.key_data, session->p.cipher_key.data,
session->p.cipher_key.length);
@@ -902,6 +1078,10 @@ static int process_aes_gmac_param(odp_crypto_generic_session_t *session,
session->p.auth_key.length)
return -1;
+ /* Verify IV len is correct */
+ if (12 != session->p.auth_iv.length)
+ return -1;
+
memcpy(session->auth.key, session->p.auth_key.data,
session->p.auth_key.length);
@@ -1058,6 +1238,11 @@ static int process_aes_ccm_param(odp_crypto_generic_session_t *session,
session->p.cipher_key.length)
return -1;
+ /* Verify IV len is correct */
+ if (11 != session->p.cipher_iv.length &&
+ 13 != session->p.cipher_iv.length)
+ return -1;
+
memcpy(session->cipher.key_data, session->p.cipher_key.data,
session->p.cipher_key.length);
@@ -1078,6 +1263,10 @@ static int process_aes_ccm_param(odp_crypto_generic_session_t *session,
static int process_auth_hmac_param(odp_crypto_generic_session_t *session,
const EVP_MD *evp_md)
{
+ /* Verify IV len is correct */
+ if (0 != session->p.auth_iv.length)
+ return -1;
+
/* Set function */
if (ODP_CRYPTO_OP_ENCODE == session->p.op)
session->auth.func = auth_hmac_gen;
@@ -1106,6 +1295,9 @@ static int process_auth_cmac_param(odp_crypto_generic_session_t *session,
session->p.auth_key.length)
return -1;
+ if (0 != session->p.auth_iv.length)
+ return -1;
+
/* Set function */
if (ODP_CRYPTO_OP_ENCODE == session->p.op)
session->auth.func = auth_cmac_gen;
@@ -1152,7 +1344,9 @@ int odp_crypto_capability(odp_crypto_capability_t *capa)
capa->auths.bit.md5_hmac = 1;
capa->auths.bit.sha1_hmac = 1;
capa->auths.bit.sha256_hmac = 1;
+ capa->auths.bit.sha384_hmac = 1;
capa->auths.bit.sha512_hmac = 1;
+ capa->auths.bit.aes_xcbc_mac = 1;
capa->auths.bit.aes_gcm = 1;
capa->auths.bit.aes_ccm = 1;
capa->auths.bit.aes_gmac = 1;
@@ -1249,10 +1443,18 @@ int odp_crypto_auth_capability(odp_auth_alg_t auth,
src = auth_capa_sha256_hmac;
num = sizeof(auth_capa_sha256_hmac) / size;
break;
+ case ODP_AUTH_ALG_SHA384_HMAC:
+ src = auth_capa_sha384_hmac;
+ num = sizeof(auth_capa_sha384_hmac) / size;
+ break;
case ODP_AUTH_ALG_SHA512_HMAC:
src = auth_capa_sha512_hmac;
num = sizeof(auth_capa_sha512_hmac) / size;
break;
+ case ODP_AUTH_ALG_AES_XCBC_MAC:
+ src = auth_capa_aes_xcbc;
+ num = sizeof(auth_capa_aes_xcbc) / size;
+ break;
case ODP_AUTH_ALG_AES_GCM:
src = auth_capa_aes_gcm;
num = sizeof(auth_capa_aes_gcm) / size;
@@ -1462,9 +1664,15 @@ odp_crypto_session_create(odp_crypto_session_param_t *param,
case ODP_AUTH_ALG_SHA256_HMAC:
rc = process_auth_hmac_param(session, EVP_sha256());
break;
+ case ODP_AUTH_ALG_SHA384_HMAC:
+ rc = process_auth_hmac_param(session, EVP_sha384());
+ break;
case ODP_AUTH_ALG_SHA512_HMAC:
rc = process_auth_hmac_param(session, EVP_sha512());
break;
+ case ODP_AUTH_ALG_AES_XCBC_MAC:
+ rc = process_aesxcbc_param(session, EVP_aes_128_ecb());
+ break;
#if ODP_DEPRECATED_API
case ODP_AUTH_ALG_AES128_GCM:
if (param->cipher_alg == ODP_CIPHER_ALG_AES128_GCM)
diff --git a/platform/linux-generic/_fdserver.c b/platform/linux-generic/odp_fdserver.c
index 065736f01..0e9fb0e4d 100644
--- a/platform/linux-generic/_fdserver.c
+++ b/platform/linux-generic/odp_fdserver.c
@@ -42,7 +42,7 @@
#include <odp/api/spinlock.h>
#include <odp_internal.h>
#include <odp_debug_internal.h>
-#include <_fdserver_internal.h>
+#include <odp_fdserver_internal.h>
#include <sys/prctl.h>
#include <signal.h>
@@ -70,6 +70,14 @@
#define MAP_ANONYMOUS MAP_ANON
#endif
+#define FD_ODP_DEBUG_PRINT 0
+
+#define FD_ODP_DBG(fmt, ...) \
+ do { \
+ if (FD_ODP_DEBUG_PRINT == 1) \
+ ODP_DBG(fmt, ##__VA_ARGS__);\
+ } while (0)
+
/* when accessing the client functions, clients should be mutexed: */
static odp_spinlock_t *client_lock;
@@ -257,7 +265,9 @@ static int get_socket(void)
remote.sun_family = AF_UNIX;
strcpy(remote.sun_path, sockpath);
len = strlen(remote.sun_path) + sizeof(remote.sun_family);
- if (connect(s_sock, (struct sockaddr *)&remote, len) == -1) {
+ while (connect(s_sock, (struct sockaddr *)&remote, len) == -1) {
+ if (errno == EINTR)
+ continue;
ODP_ERR("cannot connect to server: %s\n", strerror(errno));
close(s_sock);
return -1;
@@ -280,8 +290,8 @@ int _odp_fdserver_register_fd(fd_server_context_e context, uint64_t key,
odp_spinlock_lock(client_lock);
- ODP_DBG("FD client register: pid=%d key=%" PRIu64 ", fd=%d\n",
- getpid(), key, fd_to_send);
+ FD_ODP_DBG("FD client register: pid=%d key=%" PRIu64 ", fd=%d\n",
+ getpid(), key, fd_to_send);
s_sock = get_socket();
if (s_sock < 0) {
@@ -326,8 +336,8 @@ int _odp_fdserver_deregister_fd(fd_server_context_e context, uint64_t key)
odp_spinlock_lock(client_lock);
- ODP_DBG("FD client deregister: pid=%d key=%" PRIu64 "\n",
- getpid(), key);
+ FD_ODP_DBG("FD client deregister: pid=%d key=%" PRIu64 "\n",
+ getpid(), key);
s_sock = get_socket();
if (s_sock < 0) {
@@ -413,7 +423,7 @@ static int stop_server(void)
odp_spinlock_lock(client_lock);
- ODP_DBG("FD sending server stop request\n");
+ FD_ODP_DBG("FD sending server stop request\n");
s_sock = get_socket();
if (s_sock < 0) {
@@ -464,8 +474,8 @@ static int handle_request(int client_sock)
fd_table[fd_table_nb_entries].context = context;
fd_table[fd_table_nb_entries].key = key;
fd_table[fd_table_nb_entries++].fd = fd;
- ODP_DBG("storing {ctx=%d, key=%" PRIu64 "}->fd=%d\n",
- context, key, fd);
+ FD_ODP_DBG("storing {ctx=%d, key=%" PRIu64 "}->fd=%d\n",
+ context, key, fd);
} else {
ODP_ERR("FD table full\n");
send_fdserver_msg(client_sock, FD_REGISTER_NACK,
@@ -517,9 +527,9 @@ static int handle_request(int client_sock)
for (i = 0; i < fd_table_nb_entries; i++) {
if ((fd_table[i].context == context) &&
(fd_table[i].key == key)) {
- ODP_DBG("drop {ctx=%d,"
- " key=%" PRIu64 "}->fd=%d\n",
- context, key, fd_table[i].fd);
+ FD_ODP_DBG("drop {ctx=%d,"
+ " key=%" PRIu64 "}->fd=%d\n",
+ context, key, fd_table[i].fd);
close(fd_table[i].fd);
fd_table[i] = fd_table[--fd_table_nb_entries];
send_fdserver_msg(client_sock,
@@ -535,7 +545,7 @@ static int handle_request(int client_sock)
break;
case FD_SERVERSTOP_REQ:
- ODP_DBG("Stoping FD server\n");
+ FD_ODP_DBG("Stoping FD server\n");
return 1;
default:
@@ -559,8 +569,11 @@ static void wait_requests(int sock)
addr_sz = sizeof(remote);
c_socket = accept(sock, (struct sockaddr *)&remote, &addr_sz);
if (c_socket == -1) {
- ODP_ERR("wait_requests: %s\n", strerror(errno));
- return;
+ if (errno == EINTR)
+ continue;
+
+ ODP_ERR("wait_requests: %s\n", strerror(errno));
+ return;
}
if (handle_request(c_socket))
@@ -636,6 +649,34 @@ int _odp_fdserver_init_global(void)
}
if (server_pid == 0) { /*child */
+ sigset_t sigset;
+ struct sigaction action;
+
+ sigfillset(&sigset);
+ /* undefined if these are ignored, as per POSIX */
+ sigdelset(&sigset, SIGFPE);
+ sigdelset(&sigset, SIGILL);
+ sigdelset(&sigset, SIGSEGV);
+ /* can not be masked */
+ sigdelset(&sigset, SIGKILL);
+ sigdelset(&sigset, SIGSTOP);
+ /* these we want to handle */
+ sigdelset(&sigset, SIGTERM);
+ if (sigprocmask(SIG_SETMASK, &sigset, NULL) == -1) {
+ ODP_ERR("Could not set signal mask");
+ exit(1);
+ }
+
+ /* set default handlers for those signals we can handle */
+ memset(&action, 0, sizeof(action));
+ action.sa_handler = SIG_DFL;
+ sigemptyset(&action.sa_mask);
+ action.sa_flags = 0;
+ sigaction(SIGFPE, &action, NULL);
+ sigaction(SIGILL, &action, NULL);
+ sigaction(SIGSEGV, &action, NULL);
+ sigaction(SIGTERM, &action, NULL);
+
/* TODO: pin the server on appropriate service cpu mask */
/* when (if) we can agree on the usage of service mask */
@@ -643,6 +684,12 @@ int _odp_fdserver_init_global(void)
/* orphans being "adopted" by the init process... */
prctl(PR_SET_PDEATHSIG, SIGTERM);
+ res = setsid();
+ if (res == -1) {
+ ODP_ERR("Could not setsid()");
+ exit(1);
+ }
+
/* allocate the space for the file descriptor<->key table: */
fd_table = malloc(FDSERVER_MAX_ENTRIES * sizeof(fdentry_t));
if (!fd_table) {
diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c
index 1e90cea0f..782cbf45c 100644
--- a/platform/linux-generic/odp_ipsec.c
+++ b/platform/linux-generic/odp_ipsec.c
@@ -162,67 +162,10 @@ static odp_ipsec_packet_result_t *ipsec_pkt_result(odp_packet_t packet)
return &packet_hdr(packet)->ipsec_ctx;
}
-static inline int _odp_ipv4_csum(odp_packet_t pkt,
- uint32_t offset,
- _odp_ipv4hdr_t *ip,
- odp_u16sum_t *chksum)
-{
- unsigned nleft = _ODP_IPV4HDR_IHL(ip->ver_ihl) * 4;
- uint16_t buf[nleft / 2];
- int res;
-
- if (odp_unlikely(nleft < sizeof(*ip)))
- return -1;
- ip->chksum = 0;
- memcpy(buf, ip, sizeof(*ip));
- res = odp_packet_copy_to_mem(pkt, offset + sizeof(*ip),
- nleft - sizeof(*ip),
- buf + sizeof(*ip) / 2);
- if (odp_unlikely(res < 0))
- return res;
-
- *chksum = ~odp_chksum_ones_comp16(buf, nleft);
-
- return 0;
-}
-
-#define _ODP_IPV4HDR_CSUM_OFFSET ODP_OFFSETOF(_odp_ipv4hdr_t, chksum)
#define _ODP_IPV4HDR_PROTO_OFFSET ODP_OFFSETOF(_odp_ipv4hdr_t, proto)
#define _ODP_IPV6HDR_NHDR_OFFSET ODP_OFFSETOF(_odp_ipv6hdr_t, next_hdr)
#define _ODP_IPV6HDREXT_NHDR_OFFSET ODP_OFFSETOF(_odp_ipv6hdr_ext_t, next_hdr)
-/**
- * Calculate and fill in IPv4 checksum
- *
- * @param pkt ODP packet
- *
- * @retval 0 on success
- * @retval <0 on failure
- */
-static inline int _odp_ipv4_csum_update(odp_packet_t pkt)
-{
- uint32_t offset;
- _odp_ipv4hdr_t ip;
- odp_u16sum_t chksum;
- int res;
-
- offset = odp_packet_l3_offset(pkt);
- if (offset == ODP_PACKET_OFFSET_INVALID)
- return -1;
-
- res = odp_packet_copy_to_mem(pkt, offset, sizeof(ip), &ip);
- if (odp_unlikely(res < 0))
- return res;
-
- res = _odp_ipv4_csum(pkt, offset, &ip, &chksum);
- if (odp_unlikely(res < 0))
- return res;
-
- return odp_packet_copy_from_mem(pkt,
- offset + _ODP_IPV4HDR_CSUM_OFFSET,
- 2, &chksum);
-}
-
#define ipv4_hdr_len(ip) (_ODP_IPV4HDR_IHL((ip)->ver_ihl) * 4)
static const uint8_t ipsec_padding[255] = {
@@ -815,7 +758,7 @@ static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt,
ipv4hdr->tot_len = _odp_cpu_to_be_16(state.ip_tot_len);
else
ipv4hdr->ttl -= ipsec_sa->dec_ttl;
- _odp_ipv4_csum_update(pkt);
+ _odp_packet_ipv4_chksum_insert(pkt);
} else if (state.is_ipv6 && odp_packet_len(pkt) > _ODP_IPV6HDR_LEN) {
_odp_ipv6hdr_t *ipv6hdr = odp_packet_l3_ptr(pkt, NULL);
@@ -883,6 +826,7 @@ static int ipsec_out_tunnel_parse_ipv4(ipsec_state_t *state,
state->out_tunnel.ip_tos = ipv4hdr->tos;
state->out_tunnel.ip_df = _ODP_IPV4HDR_FLAGS_DONT_FRAG(flags);
state->out_tunnel.ip_flabel = 0;
+ state->ip_next_hdr = ipv4hdr->proto;
return 0;
}
@@ -1238,7 +1182,7 @@ static int ipsec_out_esp(odp_packet_t *pkt,
static void ipsec_out_esp_post(ipsec_state_t *state, odp_packet_t pkt)
{
if (state->is_ipv4)
- _odp_ipv4_csum_update(pkt);
+ _odp_packet_ipv4_chksum_insert(pkt);
}
static int ipsec_out_ah(odp_packet_t *pkt,
@@ -1343,7 +1287,7 @@ static void ipsec_out_ah_post(ipsec_state_t *state, odp_packet_t pkt)
ipv4hdr->tos = state->ah_ipv4.tos;
ipv4hdr->frag_offset = state->ah_ipv4.frag_offset;
- _odp_ipv4_csum_update(pkt);
+ _odp_packet_ipv4_chksum_insert(pkt);
} else {
_odp_ipv6hdr_t *ipv6hdr = odp_packet_l3_ptr(pkt, NULL);
@@ -1352,6 +1296,41 @@ static void ipsec_out_ah_post(ipsec_state_t *state, odp_packet_t pkt)
}
}
+#define OL_TX_CHKSUM_PKT(_cfg, _proto, _ovr_set, _ovr) \
+ (_proto && (_ovr_set ? _ovr : _cfg))
+
+static void ipsec_out_checksums(odp_packet_t pkt,
+ ipsec_state_t *state)
+{
+ odp_bool_t ipv4_chksum_pkt, udp_chksum_pkt, tcp_chksum_pkt;
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+ odp_ipsec_outbound_config_t outbound = ipsec_config.outbound;
+
+ ipv4_chksum_pkt = OL_TX_CHKSUM_PKT(outbound.chksum.inner_ipv4,
+ state->is_ipv4,
+ pkt_hdr->p.flags.l3_chksum_set,
+ pkt_hdr->p.flags.l3_chksum);
+ udp_chksum_pkt = OL_TX_CHKSUM_PKT(outbound.chksum.inner_udp,
+ state->ip_next_hdr ==
+ _ODP_IPPROTO_UDP,
+ pkt_hdr->p.flags.l4_chksum_set,
+ pkt_hdr->p.flags.l4_chksum);
+ tcp_chksum_pkt = OL_TX_CHKSUM_PKT(outbound.chksum.inner_tcp,
+ state->ip_next_hdr ==
+ _ODP_IPPROTO_TCP,
+ pkt_hdr->p.flags.l4_chksum_set,
+ pkt_hdr->p.flags.l4_chksum);
+
+ if (ipv4_chksum_pkt)
+ _odp_packet_ipv4_chksum_insert(pkt);
+
+ if (tcp_chksum_pkt)
+ _odp_packet_tcp_chksum_insert(pkt);
+
+ if (udp_chksum_pkt)
+ _odp_packet_udp_chksum_insert(pkt);
+}
+
static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt,
odp_ipsec_sa_t sa,
odp_packet_t *pkt_out,
@@ -1411,6 +1390,9 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt,
if (state.ip_tot_len + state.ip_offset != odp_packet_len(pkt))
rc = -1;
+
+ if (rc == 0)
+ ipsec_out_checksums(pkt, &state);
} else {
if (state.is_ipv4)
rc = ipsec_out_tunnel_parse_ipv4(&state, ipsec_sa);
@@ -1428,6 +1410,8 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt,
goto err;
}
+ ipsec_out_checksums(pkt, &state);
+
if (ipsec_sa->tun_ipv4)
rc = ipsec_out_tunnel_ipv4(&pkt, &state, ipsec_sa,
opt->flag.ip_param ?
@@ -1500,7 +1484,7 @@ static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt,
else if (ODP_IPSEC_AH == ipsec_sa->proto)
ipsec_out_ah_post(&state, pkt);
- _odp_ipv4_csum_update(pkt);
+ _odp_packet_ipv4_chksum_insert(pkt);
*pkt_out = pkt;
return ipsec_sa;
diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c
index 05865eb3a..c21269694 100644
--- a/platform/linux-generic/odp_ipsec_sad.c
+++ b/platform/linux-generic/odp_ipsec_sad.c
@@ -220,6 +220,8 @@ uint32_t _odp_ipsec_cipher_iv_len(odp_cipher_alg_t cipher)
#endif
case ODP_CIPHER_ALG_AES_GCM:
return 12;
+ case ODP_CIPHER_ALG_AES_CCM:
+ return 11;
case ODP_CIPHER_ALG_CHACHA20_POLY1305:
return 12;
default:
@@ -244,14 +246,20 @@ uint32_t _odp_ipsec_auth_digest_len(odp_auth_alg_t auth)
#endif
case ODP_AUTH_ALG_SHA256_HMAC:
return 16;
+ case ODP_AUTH_ALG_SHA384_HMAC:
+ return 24;
case ODP_AUTH_ALG_SHA512_HMAC:
return 32;
+ case ODP_AUTH_ALG_AES_XCBC_MAC:
+ return 12;
#if ODP_DEPRECATED_API
case ODP_AUTH_ALG_AES128_GCM:
#endif
case ODP_AUTH_ALG_AES_GCM:
case ODP_AUTH_ALG_AES_GMAC:
return 16;
+ case ODP_AUTH_ALG_AES_CCM:
+ return 16;
case ODP_AUTH_ALG_CHACHA20_POLY1305:
return 16;
default:
@@ -411,6 +419,7 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param)
case ODP_CIPHER_ALG_AES128_GCM:
#endif
case ODP_CIPHER_ALG_AES_GCM:
+ case ODP_CIPHER_ALG_AES_CCM:
ipsec_sa->use_counter_iv = 1;
ipsec_sa->esp_iv_len = 8;
ipsec_sa->esp_block_len = 16;
diff --git a/platform/linux-generic/_ishm.c b/platform/linux-generic/odp_ishm.c
index ab112acea..babf51b91 100644
--- a/platform/linux-generic/_ishm.c
+++ b/platform/linux-generic/odp_ishm.c
@@ -57,10 +57,10 @@
#include <odp_shm_internal.h>
#include <odp_debug_internal.h>
#include <odp_align_internal.h>
-#include <_fdserver_internal.h>
-#include <_ishm_internal.h>
-#include <_ishmphy_internal.h>
-#include <_ishmpool_internal.h>
+#include <odp_fdserver_internal.h>
+#include <odp_ishm_internal.h>
+#include <odp_ishmphy_internal.h>
+#include <odp_ishmpool_internal.h>
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
@@ -456,10 +456,7 @@ static int create_file(int block_index, huge_flag_t huge, uint64_t len,
fd = open(filename, oflag, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
if (fd < 0) {
- if (huge == HUGE)
- ODP_DBG("open failed for %s: %s.\n",
- filename, strerror(errno));
- else
+ if (huge != HUGE)
ODP_ERR("open failed for %s: %s.\n",
filename, strerror(errno));
return -1;
@@ -569,7 +566,7 @@ static void *do_map(int block_index, uint64_t len, uint32_t align,
}
return NULL;
}
- ishm_tbl->block[block_index].fragment = fragment;
+ new_block->fragment = fragment;
}
/* try to mmap: */
@@ -771,6 +768,65 @@ static void procsync(void)
}
/*
+ * Free a block as described in block_free(), but
+ * considering whether to close the file descriptor or not, and
+ * whether to deregister from the fdserver.
+ */
+static int block_free_internal(int block_index, int close_fd, int deregister)
+{
+ int proc_index;
+ ishm_block_t *block; /* entry in the main block table*/
+ int last;
+ int ret = 0;
+
+ if ((block_index < 0) ||
+ (block_index >= ISHM_MAX_NB_BLOCKS) ||
+ (ishm_tbl->block[block_index].len == 0)) {
+ ODP_ERR("Request to free an invalid block\n");
+ return -1;
+ }
+
+ block = &ishm_tbl->block[block_index];
+
+ proc_index = procfind_block(block_index);
+ if (proc_index >= 0) {
+ /* remove the mapping and possible fragment */
+ do_unmap(ishm_proctable->entry[proc_index].start,
+ block->len,
+ ishm_proctable->entry[proc_index].flags,
+ block_index);
+
+ /* close the related fd */
+ if (close_fd)
+ close(ishm_proctable->entry[proc_index].fd);
+
+ /* remove entry from process local table: */
+ last = ishm_proctable->nb_entries - 1;
+ ishm_proctable->entry[proc_index] = ishm_proctable->entry[last];
+ ishm_proctable->nb_entries = last;
+ } else {
+ /* just possibly free the fragment as no mapping exist here: */
+ do_unmap(NULL, 0, block->flags, block_index);
+ }
+
+ /* remove all files related to this block: */
+ if (close_fd)
+ delete_file(block);
+
+ /* deregister the file descriptor from the file descriptor server. */
+ if (deregister)
+ ret = _odp_fdserver_deregister_fd(FD_SRV_CTX_ISHM, block_index);
+
+ /* mark the block as free in the main block table: */
+ block->len = 0;
+
+ /* mark the change so other processes see this entry as obsolete: */
+ block->seq++;
+
+ return ret;
+}
+
+/*
* Allocate and map internal shared memory, or other objects:
* If a name is given, check that this name is not already in use.
* If ok, allocate a new shared memory block and map the
@@ -834,7 +890,6 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd,
/* If a file descriptor is provided, get the real size and map: */
if (fd >= 0) {
if (fstat(fd, &statbuf) < 0) {
- close(fd);
odp_spinlock_unlock(&ishm_tbl->lock);
ODP_ERR("_ishm_reserve failed (fstat failed: %s).\n",
strerror(errno));
@@ -842,17 +897,16 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd,
return -1;
}
len = statbuf.st_size;
+ new_block->external_fd = 1;
/* note that the huge page flag is meningless here as huge
* page is determined by the provided file descriptor: */
addr = do_map(new_index, len, align, flags, EXTERNAL, &fd);
if (addr == NULL) {
- close(fd);
odp_spinlock_unlock(&ishm_tbl->lock);
ODP_ERR("_ishm_reserve failed.\n");
return -1;
}
new_block->huge = EXTERNAL;
- new_block->external_fd = 1;
} else {
new_block->external_fd = 0;
}
@@ -933,7 +987,10 @@ int _odp_ishm_reserve(const char *name, uint64_t size, int fd,
ishm_proctable->entry[new_proc_entry].fd = fd;
/* register the file descriptor to the file descriptor server. */
- _odp_fdserver_register_fd(FD_SRV_CTX_ISHM, new_index, fd);
+ if (_odp_fdserver_register_fd(FD_SRV_CTX_ISHM, new_index, fd) == -1) {
+ block_free_internal(new_index, !new_block->external_fd, 0);
+ new_index = -1;
+ }
odp_spinlock_unlock(&ishm_tbl->lock);
return new_index;
@@ -1039,53 +1096,7 @@ error_exp_file:
*/
static int block_free(int block_index)
{
- int proc_index;
- ishm_block_t *block; /* entry in the main block table*/
- int last;
-
- if ((block_index < 0) ||
- (block_index >= ISHM_MAX_NB_BLOCKS) ||
- (ishm_tbl->block[block_index].len == 0)) {
- ODP_ERR("Request to free an invalid block\n");
- return -1;
- }
-
- block = &ishm_tbl->block[block_index];
-
- proc_index = procfind_block(block_index);
- if (proc_index >= 0) {
- /* close the related fd */
- close(ishm_proctable->entry[proc_index].fd);
-
- /* remove the mapping and possible fragment */
- do_unmap(ishm_proctable->entry[proc_index].start,
- block->len,
- ishm_proctable->entry[proc_index].flags,
- block_index);
-
- /* remove entry from process local table: */
- last = ishm_proctable->nb_entries - 1;
- ishm_proctable->entry[proc_index] =
- ishm_proctable->entry[last];
- ishm_proctable->nb_entries = last;
- } else {
- /* just possibly free the fragment as no mapping exist here: */
- do_unmap(NULL, 0, block->flags, block_index);
- }
-
- /* remove all files related to this block: */
- delete_file(block);
-
- /* deregister the file descriptor from the file descriptor server. */
- _odp_fdserver_deregister_fd(FD_SRV_CTX_ISHM, block_index);
-
- /* mark the block as free in the main block table: */
- block->len = 0;
-
- /* mark the change so other processes see this entry as obsolete: */
- block->seq++;
-
- return 0;
+ return block_free_internal(block_index, 1, 1);
}
/*
diff --git a/platform/linux-generic/_ishmphy.c b/platform/linux-generic/odp_ishmphy.c
index 8dde2831e..6207ce757 100644
--- a/platform/linux-generic/_ishmphy.c
+++ b/platform/linux-generic/odp_ishmphy.c
@@ -18,8 +18,8 @@
#include <odp/api/debug.h>
#include <odp_debug_internal.h>
#include <odp_align_internal.h>
-#include <_ishm_internal.h>
-#include <_ishmphy_internal.h>
+#include <odp_ishm_internal.h>
+#include <odp_ishmphy_internal.h>
#include <stdlib.h>
#include <stdio.h>
@@ -31,7 +31,7 @@
#include <fcntl.h>
#include <sys/types.h>
#include <sys/wait.h>
-#include <_ishmphy_internal.h>
+#include <odp_ishmphy_internal.h>
static void *common_va_address;
static uint64_t common_va_len;
diff --git a/platform/linux-generic/_ishmpool.c b/platform/linux-generic/odp_ishmpool.c
index 4ff24c026..04a0e535a 100644
--- a/platform/linux-generic/_ishmpool.c
+++ b/platform/linux-generic/odp_ishmpool.c
@@ -51,8 +51,8 @@
#include <odp_shm_internal.h>
#include <odp_debug_internal.h>
#include <odp_align_internal.h>
-#include <_ishm_internal.h>
-#include <_ishmpool_internal.h>
+#include <odp_ishm_internal.h>
+#include <odp_ishmpool_internal.h>
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
diff --git a/platform/linux-generic/odp_libconfig.c b/platform/linux-generic/odp_libconfig.c
index 3b3b31703..316ed067a 100644
--- a/platform/linux-generic/odp_libconfig.c
+++ b/platform/linux-generic/odp_libconfig.c
@@ -16,8 +16,6 @@
#include <odp_libconfig_internal.h>
#include <odp_libconfig_config.h>
-#define CONF_STR_NAME ((const char *)odp_linux_generic_conf)
-
extern struct odp_global_data_s odp_global_data;
int _odp_libconfig_init_global(void)
@@ -33,7 +31,7 @@ int _odp_libconfig_init_global(void)
config_init(config);
config_init(config_rt);
- if (!config_read_string(config, CONF_STR_NAME)) {
+ if (!config_read_string(config, config_builtin)) {
ODP_ERR("Failed to read default config: %s(%d): %s\n",
config_error_file(config), config_error_line(config),
config_error_text(config));
@@ -97,3 +95,41 @@ int _odp_libconfig_lookup_int(const char *path, int *value)
return (ret_def == CONFIG_TRUE || ret_rt == CONFIG_TRUE) ? 1 : 0;
}
+
+static int lookup_int(config_t *cfg,
+ const char *base_path,
+ const char *local_path,
+ const char *name,
+ int *value)
+{
+ char path[256];
+
+ if (local_path) {
+ snprintf(path, sizeof(path), "%s.%s.%s", base_path,
+ local_path, name);
+ if (config_lookup_int(cfg, path, value) == CONFIG_TRUE)
+ return 1;
+ }
+
+ snprintf(path, sizeof(path), "%s.%s", base_path, name);
+ if (config_lookup_int(cfg, path, value) == CONFIG_TRUE)
+ return 1;
+
+ return 0;
+}
+
+int _odp_libconfig_lookup_ext_int(const char *base_path,
+ const char *local_path,
+ const char *name,
+ int *value)
+{
+ if (lookup_int(&odp_global_data.libconfig_runtime,
+ base_path, local_path, name, value))
+ return 1;
+
+ if (lookup_int(&odp_global_data.libconfig_default,
+ base_path, local_path, name, value))
+ return 1;
+
+ return 0;
+}
diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c
index 462c8a4c7..d4b19feeb 100644
--- a/platform/linux-generic/odp_packet.c
+++ b/platform/linux-generic/odp_packet.c
@@ -1910,6 +1910,92 @@ int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt)
return dst_uarea_size < src_uarea_size;
}
+/* Simple implementation of ones complement sum.
+ * Based on RFC1071 and its errata.
+ */
+typedef union {
+ uint16_t w;
+ uint8_t b[2];
+} swap_buf_t;
+
+static uint32_t segment_sum16_32(const uint8_t *p,
+ uint32_t len,
+ uint32_t offset)
+
+{
+ uint32_t sum = 0;
+
+ /* Include second part of 16-bit short word split between segments */
+ if (len > 0 && (offset % 2)) {
+ swap_buf_t sw;
+
+ sw.b[0] = 0;
+ sw.b[1] = *p++;
+ sum = sw.w;
+ len--;
+ }
+
+ /*
+ * If pointer is 16-bit aligned, we can do fast path calculation.
+ * If it is not, we sum hi and lo bytes separately and then sum them.
+ */
+ if ((uintptr_t)p % 2) {
+ uint32_t sum1 = 0, sum2 = 0;
+
+ while (len > 1) {
+ sum1 += *p++;
+ sum2 += *p++;
+ len -= 2;
+ }
+#if (ODP_BYTE_ORDER == ODP_BIG_ENDIAN)
+ sum += sum2 + (sum1 << 8);
+#else
+ sum += sum1 + (sum2 << 8);
+#endif
+ } else {
+ while (len > 1) {
+ sum += *(const uint16_t *)(uintptr_t)p;
+ p += 2;
+ len -= 2;
+ }
+ }
+
+ /* Add left-over byte, if any */
+ if (len > 0) {
+ swap_buf_t sw;
+
+ sw.b[0] = *p;
+ sw.b[1] = 0;
+ sum += sw.w;
+ }
+
+ return sum;
+}
+
+static uint32_t packet_sum16_32(odp_packet_hdr_t *pkt_hdr,
+ uint32_t offset,
+ uint32_t len)
+{
+ uint32_t sum = 0;
+
+ if (offset + len > pkt_hdr->frame_len)
+ return 0;
+
+ while (len > 0) {
+ uint32_t seglen = 0; /* GCC */
+ void *mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL);
+
+ if (seglen > len)
+ seglen = len;
+
+ sum += segment_sum16_32(mapaddr, seglen, offset);
+ len -= seglen;
+ offset += seglen;
+ }
+
+ return sum;
+}
+
/** Parser helper function for Ethernet packets */
static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr,
uint32_t *offset, uint32_t frame_len)
@@ -2266,6 +2352,159 @@ int packet_parse_common(packet_parser_t *prs, const uint8_t *ptr,
seg_len, layer, ethtype);
}
+static inline int packet_ipv4_chksum(odp_packet_t pkt,
+ uint32_t offset,
+ _odp_ipv4hdr_t *ip,
+ odp_u16sum_t *chksum)
+{
+ unsigned int nleft = _ODP_IPV4HDR_IHL(ip->ver_ihl) * 4;
+ uint16_t buf[nleft / 2];
+ int res;
+
+ if (odp_unlikely(nleft < sizeof(*ip)))
+ return -1;
+ ip->chksum = 0;
+ memcpy(buf, ip, sizeof(*ip));
+ res = odp_packet_copy_to_mem(pkt, offset + sizeof(*ip),
+ nleft - sizeof(*ip),
+ buf + sizeof(*ip) / 2);
+ if (odp_unlikely(res < 0))
+ return res;
+
+ *chksum = ~odp_chksum_ones_comp16(buf, nleft);
+
+ return 0;
+}
+
+#define _ODP_IPV4HDR_CSUM_OFFSET ODP_OFFSETOF(_odp_ipv4hdr_t, chksum)
+#define _ODP_IPV4ADDR_OFFSSET ODP_OFFSETOF(_odp_ipv4hdr_t, src_addr)
+#define _ODP_IPV6ADDR_OFFSSET ODP_OFFSETOF(_odp_ipv6hdr_t, src_addr)
+#define _ODP_IPV4HDR_CSUM_OFFSET ODP_OFFSETOF(_odp_ipv4hdr_t, chksum)
+#define _ODP_UDP_LEN_OFFSET ODP_OFFSETOF(_odp_udphdr_t, length)
+#define _ODP_UDP_CSUM_OFFSET ODP_OFFSETOF(_odp_udphdr_t, chksum)
+
+/**
+ * Calculate and fill in IPv4 checksum
+ *
+ * @param pkt ODP packet
+ *
+ * @retval 0 on success
+ * @retval <0 on failure
+ */
+int _odp_packet_ipv4_chksum_insert(odp_packet_t pkt)
+{
+ uint32_t offset;
+ _odp_ipv4hdr_t ip;
+ odp_u16sum_t chksum;
+ int res;
+
+ offset = odp_packet_l3_offset(pkt);
+ if (offset == ODP_PACKET_OFFSET_INVALID)
+ return -1;
+
+ res = odp_packet_copy_to_mem(pkt, offset, sizeof(ip), &ip);
+ if (odp_unlikely(res < 0))
+ return res;
+
+ res = packet_ipv4_chksum(pkt, offset, &ip, &chksum);
+ if (odp_unlikely(res < 0))
+ return res;
+
+ return odp_packet_copy_from_mem(pkt,
+ offset + _ODP_IPV4HDR_CSUM_OFFSET,
+ 2, &chksum);
+}
+
+static int _odp_packet_tcp_udp_chksum_insert(odp_packet_t pkt, uint16_t proto)
+{
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+ uint32_t zero = 0;
+ uint32_t sum;
+ uint16_t l3_ver;
+ uint16_t chksum;
+ uint32_t chksum_offset;
+
+ if (pkt_hdr->p.l3_offset == ODP_PACKET_OFFSET_INVALID)
+ return -1;
+ if (pkt_hdr->p.l4_offset == ODP_PACKET_OFFSET_INVALID)
+ return -1;
+
+ odp_packet_copy_to_mem(pkt, pkt_hdr->p.l3_offset, 2, &l3_ver);
+
+ if (_ODP_IPV4HDR_VER(l3_ver) == _ODP_IPV4)
+ sum = packet_sum16_32(pkt_hdr,
+ pkt_hdr->p.l3_offset +
+ _ODP_IPV4ADDR_OFFSSET,
+ 2 * _ODP_IPV4ADDR_LEN);
+ else
+ sum = packet_sum16_32(pkt_hdr,
+ pkt_hdr->p.l3_offset +
+ _ODP_IPV6ADDR_OFFSSET,
+ 2 * _ODP_IPV6ADDR_LEN);
+#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
+ sum += proto;
+#else
+ sum += proto << 8;
+#endif
+
+ if (proto == _ODP_IPPROTO_TCP) {
+ sum += _odp_cpu_to_be_16(pkt_hdr->frame_len -
+ pkt_hdr->p.l4_offset);
+ chksum_offset = pkt_hdr->p.l4_offset + _ODP_UDP_CSUM_OFFSET;
+ } else {
+ sum += packet_sum16_32(pkt_hdr,
+ pkt_hdr->p.l4_offset +
+ _ODP_UDP_LEN_OFFSET,
+ 2);
+ chksum_offset = pkt_hdr->p.l4_offset + _ODP_UDP_CSUM_OFFSET;
+ }
+ odp_packet_copy_from_mem(pkt, chksum_offset, 2, &zero);
+
+ sum += packet_sum16_32(pkt_hdr,
+ pkt_hdr->p.l4_offset,
+ pkt_hdr->frame_len -
+ pkt_hdr->p.l4_offset);
+
+ /* Not more than two additions */
+ sum = (sum & 0xffff) + (sum >> 16);
+ sum = (sum & 0xffff) + (sum >> 16);
+
+ chksum = ~sum;
+
+ if (proto == _ODP_IPPROTO_UDP && chksum == 0)
+ chksum = 0xffff;
+
+ return odp_packet_copy_from_mem(pkt,
+ chksum_offset,
+ 2, &chksum);
+}
+
+/**
+ * Calculate and fill in TCP checksum
+ *
+ * @param pkt ODP packet
+ *
+ * @retval 0 on success
+ * @retval <0 on failure
+ */
+int _odp_packet_tcp_chksum_insert(odp_packet_t pkt)
+{
+ return _odp_packet_tcp_udp_chksum_insert(pkt, _ODP_IPPROTO_TCP);
+}
+
+/**
+ * Calculate and fill in UDP checksum
+ *
+ * @param pkt ODP packet
+ *
+ * @retval 0 on success
+ * @retval <0 on failure
+ */
+int _odp_packet_udp_chksum_insert(odp_packet_t pkt)
+{
+ return _odp_packet_tcp_udp_chksum_insert(pkt, _ODP_IPPROTO_UDP);
+}
+
/**
* Simple packet parser
*/
diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c
index 56e9f6860..bf7aef148 100644
--- a/platform/linux-generic/odp_packet_io.c
+++ b/platform/linux-generic/odp_packet_io.c
@@ -376,8 +376,10 @@ int odp_pktio_close(odp_pktio_t hdl)
int res;
entry = get_pktio_entry(hdl);
- if (entry == NULL)
+ if (entry == NULL) {
+ ODP_ERR("Bad handle\n");
return -1;
+ }
if (entry->s.state == PKTIO_STATE_STARTED) {
ODP_DBG("Missing odp_pktio_stop() before close.\n");
@@ -416,8 +418,10 @@ int odp_pktio_config(odp_pktio_t hdl, const odp_pktio_config_t *config)
int res = 0;
entry = get_pktio_entry(hdl);
- if (!entry)
+ if (!entry) {
+ ODP_ERR("Bad handle\n");
return -1;
+ }
if (config == NULL) {
odp_pktio_config_init(&default_config);
@@ -466,12 +470,15 @@ int odp_pktio_start(odp_pktio_t hdl)
int res = 0;
entry = get_pktio_entry(hdl);
- if (!entry)
+ if (!entry) {
+ ODP_ERR("Bad handle\n");
return -1;
+ }
lock_entry(entry);
if (entry->s.state == PKTIO_STATE_STARTED) {
unlock_entry(entry);
+ ODP_ERR("Already started\n");
return -1;
}
if (entry->s.ops->start)
@@ -513,8 +520,10 @@ static int _pktio_stop(pktio_entry_t *entry)
int res = 0;
odp_pktin_mode_t mode = entry->s.param.in_mode;
- if (entry->s.state != PKTIO_STATE_STARTED)
+ if (entry->s.state != PKTIO_STATE_STARTED) {
+ ODP_ERR("Not started\n");
return -1;
+ }
if (entry->s.ops->stop)
res = entry->s.ops->stop(entry);
@@ -536,8 +545,10 @@ int odp_pktio_stop(odp_pktio_t hdl)
int res;
entry = get_pktio_entry(hdl);
- if (!entry)
+ if (!entry) {
+ ODP_ERR("Bad handle\n");
return -1;
+ }
lock_entry(entry);
res = _pktio_stop(entry);
@@ -1159,13 +1170,15 @@ void odp_pktio_print(odp_pktio_t hdl)
len += snprintf(&str[len], n - len,
"pktio\n");
len += snprintf(&str[len], n - len,
- " handle %" PRIu64 "\n",
- odp_pktio_to_u64(hdl));
- len += snprintf(&str[len], n - len,
" name %s\n", entry->s.name);
len += snprintf(&str[len], n - len,
" type %s\n", entry->s.ops->name);
len += snprintf(&str[len], n - len,
+ " index %i\n", _odp_pktio_index(hdl));
+ len += snprintf(&str[len], n - len,
+ " handle (u64) %" PRIu64 "\n",
+ odp_pktio_to_u64(hdl));
+ len += snprintf(&str[len], n - len,
" state %s\n",
entry->s.state == PKTIO_STATE_STARTED ? "start" :
(entry->s.state == PKTIO_STATE_STOPPED ? "stop" :
diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c
index 998fc649e..955e14f9b 100644
--- a/platform/linux-generic/odp_pool.c
+++ b/platform/linux-generic/odp_pool.c
@@ -718,12 +718,13 @@ int buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int max_num)
buf_hdr[i] = buf_hdr_from_index(pool, cache->buf_index[j]);
}
+ /* Declare variable here to fix clang compilation bug */
+ uint32_t data[burst];
+
/* If needed, get more from the global pool */
if (odp_unlikely(num_deq)) {
- /* Temporary copy needed since odp_buffer_t is uintptr_t
- * and not uint32_t. */
- uint32_t data[burst];
-
+ /* Temporary copy to data[] needed since odp_buffer_t is
+ * uintptr_t and not uint32_t. */
ring = &pool->ring->hdr;
mask = pool->ring_mask;
burst = ring_deq_multi(ring, mask, data, burst);
diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c
index 1218987b1..89a0cd907 100644
--- a/platform/linux-generic/odp_queue_basic.c
+++ b/platform/linux-generic/odp_queue_basic.c
@@ -25,6 +25,7 @@
#include <odp/api/hints.h>
#include <odp/api/sync.h>
#include <odp/api/traffic_mngr.h>
+#include <odp_libconfig_internal.h>
#define NUM_INTERNAL_QUEUES 64
@@ -36,6 +37,9 @@
#include <string.h>
#include <inttypes.h>
+#define MIN_QUEUE_SIZE 8
+#define MAX_QUEUE_SIZE (1 * 1024 * 1024)
+
static int queue_init(queue_entry_t *queue, const char *name,
const odp_queue_param_t *param);
@@ -60,11 +64,11 @@ static int queue_capa(odp_queue_capability_t *capa, int sched)
/* Reserve some queues for internal use */
capa->max_queues = ODP_CONFIG_QUEUES - NUM_INTERNAL_QUEUES;
capa->plain.max_num = capa->max_queues;
- capa->plain.max_size = CONFIG_QUEUE_SIZE;
+ capa->plain.max_size = queue_glb->config.max_queue_size;
capa->plain.lockfree.max_num = queue_glb->queue_lf_num;
capa->plain.lockfree.max_size = queue_glb->queue_lf_size;
capa->sched.max_num = capa->max_queues;
- capa->sched.max_size = CONFIG_QUEUE_SIZE;
+ capa->sched.max_size = queue_glb->config.max_queue_size;
if (sched) {
capa->max_ordered_locks = sched_fn->max_ordered_locks();
@@ -75,6 +79,52 @@ static int queue_capa(odp_queue_capability_t *capa, int sched)
return 0;
}
+static int read_config_file(queue_global_t *queue_glb)
+{
+ const char *str;
+ uint32_t val_u32;
+ int val = 0;
+
+ ODP_PRINT("Queue config:\n");
+
+ str = "queue_basic.max_queue_size";
+ if (!_odp_libconfig_lookup_int(str, &val)) {
+ ODP_ERR("Config option '%s' not found.\n", str);
+ return -1;
+ }
+
+ val_u32 = val;
+
+ if (val_u32 > MAX_QUEUE_SIZE || val_u32 < MIN_QUEUE_SIZE ||
+ !CHECK_IS_POWER2(val_u32)) {
+ ODP_ERR("Bad value %s = %u\n", str, val_u32);
+ return -1;
+ }
+
+ queue_glb->config.max_queue_size = val_u32;
+ ODP_PRINT(" %s: %u\n", str, val_u32);
+
+ str = "queue_basic.default_queue_size";
+ if (!_odp_libconfig_lookup_int(str, &val)) {
+ ODP_ERR("Config option '%s' not found.\n", str);
+ return -1;
+ }
+
+ val_u32 = val;
+
+ if (val_u32 > queue_glb->config.max_queue_size ||
+ val_u32 < MIN_QUEUE_SIZE ||
+ !CHECK_IS_POWER2(val_u32)) {
+ ODP_ERR("Bad value %s = %u\n", str, val_u32);
+ return -1;
+ }
+
+ queue_glb->config.default_queue_size = val_u32;
+ ODP_PRINT(" %s: %u\n\n", str, val_u32);
+
+ return 0;
+}
+
static int queue_init_global(void)
{
uint32_t i;
@@ -82,10 +132,11 @@ static int queue_init_global(void)
uint32_t lf_size = 0;
queue_lf_func_t *lf_func;
odp_queue_capability_t capa;
+ uint64_t mem_size;
ODP_DBG("Starts...\n");
- shm = odp_shm_reserve("odp_queues",
+ shm = odp_shm_reserve("_odp_queue_gbl",
sizeof(queue_global_t),
sizeof(queue_entry_t), 0);
@@ -104,6 +155,26 @@ static int queue_init_global(void)
queue->s.handle = queue_from_index(i);
}
+ if (read_config_file(queue_glb)) {
+ odp_shm_free(shm);
+ return -1;
+ }
+
+ queue_glb->queue_gbl_shm = shm;
+ mem_size = sizeof(uint32_t) * ODP_CONFIG_QUEUES *
+ (uint64_t)queue_glb->config.max_queue_size;
+
+ shm = odp_shm_reserve("_odp_queue_rings", mem_size,
+ ODP_CACHE_LINE_SIZE, 0);
+
+ if (shm == ODP_SHM_INVALID) {
+ odp_shm_free(queue_glb->queue_gbl_shm);
+ return -1;
+ }
+
+ queue_glb->queue_ring_shm = shm;
+ queue_glb->ring_data = odp_shm_addr(shm);
+
lf_func = &queue_glb->queue_lf_func;
queue_glb->queue_lf_num = queue_lf_init_global(&lf_size, lf_func);
queue_glb->queue_lf_size = lf_size;
@@ -133,7 +204,6 @@ static int queue_term_local(void)
static int queue_term_global(void)
{
int ret = 0;
- int rc = 0;
queue_entry_t *queue;
int i;
@@ -142,20 +212,24 @@ static int queue_term_global(void)
LOCK(queue);
if (queue->s.status != QUEUE_STATUS_FREE) {
ODP_ERR("Not destroyed queue: %s\n", queue->s.name);
- rc = -1;
+ ret = -1;
}
UNLOCK(queue);
}
queue_lf_term_global();
- ret = odp_shm_free(odp_shm_lookup("odp_queues"));
- if (ret < 0) {
- ODP_ERR("shm free failed for odp_queues");
- rc = -1;
+ if (odp_shm_free(queue_glb->queue_ring_shm)) {
+ ODP_ERR("shm free failed");
+ ret = -1;
}
- return rc;
+ if (odp_shm_free(queue_glb->queue_gbl_shm)) {
+ ODP_ERR("shm free failed");
+ ret = -1;
+ }
+
+ return ret;
}
static int queue_capability(odp_queue_capability_t *capa)
@@ -207,7 +281,7 @@ static odp_queue_t queue_create(const char *name,
}
if (param->nonblocking == ODP_BLOCKING) {
- if (param->size > CONFIG_QUEUE_SIZE)
+ if (param->size > queue_glb->config.max_queue_size)
return ODP_QUEUE_INVALID;
} else if (param->nonblocking == ODP_NONBLOCKING_LF) {
/* Only plain type lock-free queues supported */
@@ -586,6 +660,9 @@ static odp_event_t queue_deq(odp_queue_t handle)
static int queue_init(queue_entry_t *queue, const char *name,
const odp_queue_param_t *param)
{
+ uint64_t offset;
+ uint32_t queue_size;
+
if (name == NULL) {
queue->s.name[0] = 0;
} else {
@@ -609,9 +686,24 @@ static int queue_init(queue_entry_t *queue, const char *name,
queue->s.pktin = PKTIN_INVALID;
queue->s.pktout = PKTOUT_INVALID;
- ring_st_init(&queue->s.ring_st,
- queue_glb->ring_data[queue->s.index].data,
- CONFIG_QUEUE_SIZE);
+ /* Use default size for all small queues to quarantee performance
+ * level. */
+ queue_size = queue_glb->config.default_queue_size;
+ if (param->size > queue_glb->config.default_queue_size)
+ queue_size = param->size;
+
+ /* Round up if not already a power of two */
+ queue_size = ROUNDUP_POWER2_U32(queue_size);
+
+ if (queue_size > queue_glb->config.max_queue_size) {
+ ODP_ERR("Too large queue size %u\n", queue_size);
+ return -1;
+ }
+
+ offset = queue->s.index * (uint64_t)queue_glb->config.max_queue_size;
+
+ ring_st_init(&queue->s.ring_st, &queue_glb->ring_data[offset],
+ queue_size);
return 0;
}
diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c
index 895133cd5..2a5e01d47 100644
--- a/platform/linux-generic/odp_queue_scalable.c
+++ b/platform/linux-generic/odp_queue_scalable.c
@@ -25,8 +25,8 @@
#include <odp_pool_internal.h>
#include <odp_queue_scalable_internal.h>
#include <odp_schedule_if.h>
-#include <_ishm_internal.h>
-#include <_ishmpool_internal.h>
+#include <odp_ishm_internal.h>
+#include <odp_ishmpool_internal.h>
#include <string.h>
#include <inttypes.h>
diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c
index cd20b39dd..b50462a55 100644
--- a/platform/linux-generic/odp_schedule_basic.c
+++ b/platform/linux-generic/odp_schedule_basic.c
@@ -27,6 +27,7 @@
#include <odp_timer_internal.h>
#include <odp_queue_internal.h>
#include <odp_buffer_inlines.h>
+#include <odp_libconfig_internal.h>
/* Number of priority levels */
#define NUM_PRIO 8
@@ -41,15 +42,21 @@ ODP_STATIC_ASSERT((ODP_SCHED_PRIO_NORMAL > 0) &&
/* Number of scheduling groups */
#define NUM_SCHED_GRPS 32
-/* Priority queues per priority */
-#define QUEUES_PER_PRIO 4
+/* Group weight table size */
+#define GRP_WEIGHT_TBL_SIZE NUM_SCHED_GRPS
+
+/* Maximum priority queue spread */
+#define MAX_SPREAD 8
+
+/* Minimum priority queue spread */
+#define MIN_SPREAD 1
/* A thread polls a non preferred sched queue every this many polls
* of the prefer queue. */
#define PREFER_RATIO 64
-/* Size of poll weight table */
-#define WEIGHT_TBL_SIZE ((QUEUES_PER_PRIO - 1) * PREFER_RATIO)
+/* Spread weight table */
+#define SPREAD_TBL_SIZE ((MAX_SPREAD - 1) * PREFER_RATIO)
/* Maximum number of packet IO interfaces */
#define NUM_PKTIO ODP_CONFIG_PKTIO_ENTRIES
@@ -60,14 +67,10 @@ ODP_STATIC_ASSERT((ODP_SCHED_PRIO_NORMAL > 0) &&
/* Not a valid index */
#define NULL_INDEX ((uint32_t)-1)
-/* Priority queue ring size. In worst case, all event queues are scheduled
- * queues and have the same priority. The ring size must be larger than or
- * equal to ODP_CONFIG_QUEUES / QUEUES_PER_PRIO, so that it can hold all
- * queues in the worst case. */
-#define PRIO_QUEUE_RING_SIZE (ODP_CONFIG_QUEUES / QUEUES_PER_PRIO)
-
-/* Mask for wrapping around priority queue index */
-#define RING_MASK (PRIO_QUEUE_RING_SIZE - 1)
+/* Maximum priority queue ring size. A ring must be large enough to store all
+ * queues in the worst case (all queues are scheduled, have the same priority
+ * and no spreading). */
+#define MAX_RING_SIZE ODP_CONFIG_QUEUES
/* Priority queue empty, not a valid queue index. */
#define PRIO_QUEUE_EMPTY NULL_INDEX
@@ -76,14 +79,14 @@ ODP_STATIC_ASSERT((ODP_SCHED_PRIO_NORMAL > 0) &&
ODP_STATIC_ASSERT(CHECK_IS_POWER2(ODP_CONFIG_QUEUES),
"Number_of_queues_is_not_power_of_two");
-/* Ring size must be power of two, so that MAX_QUEUE_IDX_MASK can be used. */
-ODP_STATIC_ASSERT(CHECK_IS_POWER2(PRIO_QUEUE_RING_SIZE),
+/* Ring size must be power of two, so that mask can be used. */
+ODP_STATIC_ASSERT(CHECK_IS_POWER2(MAX_RING_SIZE),
"Ring_size_is_not_power_of_two");
/* Mask of queues per priority */
typedef uint8_t pri_mask_t;
-ODP_STATIC_ASSERT((8 * sizeof(pri_mask_t)) >= QUEUES_PER_PRIO,
+ODP_STATIC_ASSERT((8 * sizeof(pri_mask_t)) >= MAX_SPREAD,
"pri_mask_t_is_too_small");
/* Start of named groups in group mask arrays */
@@ -116,17 +119,18 @@ typedef struct {
int thr;
uint16_t stash_num;
uint16_t stash_index;
- uint16_t pause;
- uint16_t round;
+ uint16_t grp_round;
+ uint16_t spread_round;
uint32_t stash_qi;
odp_queue_t stash_queue;
odp_event_t stash_ev[MAX_DEQ];
uint32_t grp_epoch;
- int num_grp;
+ uint16_t num_grp;
+ uint16_t pause;
uint8_t grp[NUM_SCHED_GRPS];
- uint8_t weight_tbl[WEIGHT_TBL_SIZE];
- uint8_t grp_weight[WEIGHT_TBL_SIZE];
+ uint8_t spread_tbl[SPREAD_TBL_SIZE];
+ uint8_t grp_weight[GRP_WEIGHT_TBL_SIZE];
struct {
/* Source queue index */
@@ -147,7 +151,7 @@ typedef struct ODP_ALIGNED_CACHE {
ring_t ring;
/* Ring data: queue indexes */
- uint32_t queue_index[PRIO_QUEUE_RING_SIZE];
+ uint32_t queue_index[MAX_RING_SIZE];
} prio_queue_t;
@@ -168,14 +172,21 @@ typedef struct {
pri_mask_t pri_mask[NUM_PRIO];
odp_spinlock_t mask_lock;
- prio_queue_t prio_q[NUM_SCHED_GRPS][NUM_PRIO][QUEUES_PER_PRIO];
+ prio_queue_t prio_q[NUM_SCHED_GRPS][NUM_PRIO][MAX_SPREAD];
odp_shm_t shm;
- uint32_t pri_count[NUM_PRIO][QUEUES_PER_PRIO];
+
+ struct {
+ uint8_t num_spread;
+ } config;
+
+ uint32_t pri_count[NUM_PRIO][MAX_SPREAD];
odp_thrmask_t mask_all;
odp_spinlock_t grp_lock;
odp_atomic_u32_t grp_epoch;
+ uint32_t ring_mask;
+ uint16_t max_spread;
struct {
char name[ODP_SCHED_GROUP_NAME_LEN];
@@ -186,7 +197,7 @@ typedef struct {
struct {
uint8_t grp;
uint8_t prio;
- uint8_t queue_per_prio;
+ uint8_t spread;
uint8_t sync;
uint8_t order_lock_count;
uint8_t poll_pktin;
@@ -206,11 +217,11 @@ typedef struct {
/* Check that queue[] variables are large enough */
ODP_STATIC_ASSERT(NUM_SCHED_GRPS <= 256, "Group_does_not_fit_8_bits");
ODP_STATIC_ASSERT(NUM_PRIO <= 256, "Prio_does_not_fit_8_bits");
-ODP_STATIC_ASSERT(QUEUES_PER_PRIO <= 256,
- "Queues_per_prio_does_not_fit_8_bits");
+ODP_STATIC_ASSERT(MAX_SPREAD <= 256, "Spread_does_not_fit_8_bits");
ODP_STATIC_ASSERT(CONFIG_QUEUE_MAX_ORD_LOCKS <= 256,
"Ordered_lock_count_does_not_fit_8_bits");
ODP_STATIC_ASSERT(NUM_PKTIO <= 256, "Pktio_index_does_not_fit_8_bits");
+ODP_STATIC_ASSERT(CHECK_IS_POWER2(GRP_WEIGHT_TBL_SIZE), "Not_power_of_2");
/* Global scheduler context */
static sched_global_t *sched;
@@ -221,11 +232,41 @@ static __thread sched_local_t sched_local;
/* Function prototypes */
static inline void schedule_release_context(void);
+static int read_config_file(sched_global_t *sched)
+{
+ const char *str;
+ int val = 0;
+
+ ODP_PRINT("Scheduler config:\n");
+
+ str = "sched_basic.prio_spread";
+ if (!_odp_libconfig_lookup_int(str, &val)) {
+ ODP_ERR("Config option '%s' not found.\n", str);
+ return -1;
+ }
+
+ if (val > MAX_SPREAD || val < MIN_SPREAD) {
+ ODP_ERR("Bad value %s = %u\n", str, val);
+ return -1;
+ }
+
+ sched->config.num_spread = val;
+ ODP_PRINT(" %s: %i\n\n", str, val);
+
+ return 0;
+}
+
+static inline uint8_t prio_spread_index(uint32_t index)
+{
+ return index % sched->config.num_spread;
+}
+
static void sched_local_init(void)
{
int i;
- uint8_t id;
- uint8_t offset = 0;
+ uint8_t spread;
+ uint8_t num_spread = sched->config.num_spread;
+ uint8_t offset = 1;
memset(&sched_local, 0, sizeof(sched_local_t));
@@ -234,17 +275,17 @@ static void sched_local_init(void)
sched_local.stash_qi = PRIO_QUEUE_EMPTY;
sched_local.ordered.src_queue = NULL_INDEX;
- id = sched_local.thr & (QUEUES_PER_PRIO - 1);
+ spread = prio_spread_index(sched_local.thr);
- for (i = 0; i < WEIGHT_TBL_SIZE; i++) {
- sched_local.weight_tbl[i] = id;
+ for (i = 0; i < SPREAD_TBL_SIZE; i++) {
+ sched_local.spread_tbl[i] = spread;
- if (i % PREFER_RATIO == 0) {
+ if (num_spread > 1 && (i % PREFER_RATIO) == 0) {
+ sched_local.spread_tbl[i] = prio_spread_index(spread +
+ offset);
offset++;
- sched_local.weight_tbl[i] = (id + offset) &
- (QUEUES_PER_PRIO - 1);
- if (offset == QUEUES_PER_PRIO - 1)
- offset = 0;
+ if (offset == num_spread)
+ offset = 1;
}
}
}
@@ -269,19 +310,26 @@ static int schedule_init_global(void)
memset(sched, 0, sizeof(sched_global_t));
+ if (read_config_file(sched)) {
+ odp_shm_free(shm);
+ return -1;
+ }
+
+ /* When num_spread == 1, only spread_tbl[0] is used. */
+ sched->max_spread = (sched->config.num_spread - 1) * PREFER_RATIO;
sched->shm = shm;
odp_spinlock_init(&sched->mask_lock);
for (grp = 0; grp < NUM_SCHED_GRPS; grp++) {
for (i = 0; i < NUM_PRIO; i++) {
- for (j = 0; j < QUEUES_PER_PRIO; j++) {
+ for (j = 0; j < MAX_SPREAD; j++) {
prio_queue_t *prio_q;
int k;
prio_q = &sched->prio_q[grp][i][j];
ring_init(&prio_q->ring);
- for (k = 0; k < PRIO_QUEUE_RING_SIZE; k++) {
+ for (k = 0; k < MAX_RING_SIZE; k++) {
prio_q->queue_index[k] =
PRIO_QUEUE_EMPTY;
}
@@ -322,14 +370,15 @@ static int schedule_term_global(void)
int ret = 0;
int rc = 0;
int i, j, grp;
+ uint32_t ring_mask = sched->ring_mask;
for (grp = 0; grp < NUM_SCHED_GRPS; grp++) {
for (i = 0; i < NUM_PRIO; i++) {
- for (j = 0; j < QUEUES_PER_PRIO; j++) {
+ for (j = 0; j < MAX_SPREAD; j++) {
ring_t *ring = &sched->prio_q[grp][i][j].ring;
uint32_t qi;
- while ((qi = ring_deq(ring, RING_MASK)) !=
+ while ((qi = ring_deq(ring, ring_mask)) !=
RING_EMPTY) {
odp_event_t events[1];
int num;
@@ -401,7 +450,7 @@ static inline int grp_update_tbl(void)
odp_spinlock_unlock(&sched->grp_lock);
/* Update group weights. Round robin over all thread's groups. */
- for (i = 0; i < WEIGHT_TBL_SIZE; i++)
+ for (i = 0; i < GRP_WEIGHT_TBL_SIZE; i++)
sched_local.grp_weight[i] = i % num;
sched_local.num_grp = num;
@@ -413,11 +462,6 @@ static uint32_t schedule_max_ordered_locks(void)
return CONFIG_QUEUE_MAX_ORD_LOCKS;
}
-static inline int queue_per_prio(uint32_t queue_index)
-{
- return ((QUEUES_PER_PRIO - 1) & queue_index);
-}
-
static void pri_set(int id, int prio)
{
odp_spinlock_lock(&sched->mask_lock);
@@ -441,33 +485,39 @@ static void pri_clr(int id, int prio)
static void pri_set_queue(uint32_t queue_index, int prio)
{
- int id = queue_per_prio(queue_index);
+ uint8_t id = prio_spread_index(queue_index);
return pri_set(id, prio);
}
static void pri_clr_queue(uint32_t queue_index, int prio)
{
- int id = queue_per_prio(queue_index);
+ uint8_t id = prio_spread_index(queue_index);
pri_clr(id, prio);
}
static int schedule_init_queue(uint32_t queue_index,
const odp_schedule_param_t *sched_param)
{
+ uint32_t ring_size;
int i;
int prio = sched_param->prio;
pri_set_queue(queue_index, prio);
sched->queue[queue_index].grp = sched_param->group;
sched->queue[queue_index].prio = prio;
- sched->queue[queue_index].queue_per_prio = queue_per_prio(queue_index);
+ sched->queue[queue_index].spread = prio_spread_index(queue_index);
sched->queue[queue_index].sync = sched_param->sync;
sched->queue[queue_index].order_lock_count = sched_param->lock_count;
sched->queue[queue_index].poll_pktin = 0;
sched->queue[queue_index].pktio_index = 0;
sched->queue[queue_index].pktin_index = 0;
+ ring_size = MAX_RING_SIZE / sched->config.num_spread;
+ ring_size = ROUNDUP_POWER2_U32(ring_size);
+ ODP_ASSERT(ring_size <= MAX_RING_SIZE);
+ sched->ring_mask = ring_size - 1;
+
odp_atomic_init_u64(&sched->order[queue_index].ctx, 0);
odp_atomic_init_u64(&sched->order[queue_index].next_ctx, 0);
@@ -492,9 +542,9 @@ static void schedule_destroy_queue(uint32_t queue_index)
int prio = sched->queue[queue_index].prio;
pri_clr_queue(queue_index, prio);
- sched->queue[queue_index].grp = 0;
- sched->queue[queue_index].prio = 0;
- sched->queue[queue_index].queue_per_prio = 0;
+ sched->queue[queue_index].grp = 0;
+ sched->queue[queue_index].prio = 0;
+ sched->queue[queue_index].spread = 0;
if (queue_is_ordered(queue_index) &&
odp_atomic_load_u64(&sched->order[queue_index].ctx) !=
@@ -504,12 +554,12 @@ static void schedule_destroy_queue(uint32_t queue_index)
static int schedule_sched_queue(uint32_t queue_index)
{
- int grp = sched->queue[queue_index].grp;
- int prio = sched->queue[queue_index].prio;
- int queue_per_prio = sched->queue[queue_index].queue_per_prio;
- ring_t *ring = &sched->prio_q[grp][prio][queue_per_prio].ring;
+ int grp = sched->queue[queue_index].grp;
+ int prio = sched->queue[queue_index].prio;
+ int spread = sched->queue[queue_index].spread;
+ ring_t *ring = &sched->prio_q[grp][prio][spread].ring;
- ring_enq(ring, RING_MASK, queue_index);
+ ring_enq(ring, sched->ring_mask, queue_index);
return 0;
}
@@ -540,13 +590,14 @@ static void schedule_release_atomic(void)
uint32_t qi = sched_local.stash_qi;
if (qi != PRIO_QUEUE_EMPTY && sched_local.stash_num == 0) {
- int grp = sched->queue[qi].grp;
- int prio = sched->queue[qi].prio;
- int queue_per_prio = sched->queue[qi].queue_per_prio;
- ring_t *ring = &sched->prio_q[grp][prio][queue_per_prio].ring;
+ int grp = sched->queue[qi].grp;
+ int prio = sched->queue[qi].prio;
+ int spread = sched->queue[qi].spread;
+ ring_t *ring = &sched->prio_q[grp][prio][spread].ring;
/* Release current atomic queue */
- ring_enq(ring, RING_MASK, qi);
+ ring_enq(ring, sched->ring_mask, qi);
+
sched_local.stash_qi = PRIO_QUEUE_EMPTY;
}
}
@@ -773,8 +824,10 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[],
int prio, i;
int ret;
int id;
- unsigned int max_deq = MAX_DEQ;
uint32_t qi;
+ unsigned int max_deq = MAX_DEQ;
+ int num_spread = sched->config.num_spread;
+ uint32_t ring_mask = sched->ring_mask;
/* Schedule events */
for (prio = 0; prio < NUM_PRIO; prio++) {
@@ -785,14 +838,14 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[],
/* Select the first ring based on weights */
id = first;
- for (i = 0; i < QUEUES_PER_PRIO;) {
+ for (i = 0; i < num_spread;) {
int num;
int ordered;
odp_queue_t handle;
ring_t *ring;
int pktin;
- if (id >= QUEUES_PER_PRIO)
+ if (id >= num_spread)
id = 0;
/* No queues created for this priority queue */
@@ -805,7 +858,7 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[],
/* Get queue index from the priority queue */
ring = &sched->prio_q[grp][prio][id].ring;
- qi = ring_deq(ring, RING_MASK);
+ qi = ring_deq(ring, ring_mask);
/* Priority queue empty */
if (qi == RING_EMPTY) {
@@ -854,7 +907,7 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[],
continue;
if (num_pkt == 0 || !stash) {
- ring_enq(ring, RING_MASK, qi);
+ ring_enq(ring, ring_mask, qi);
break;
}
@@ -880,14 +933,14 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[],
sched_local.ordered.src_queue = qi;
/* Continue scheduling ordered queues */
- ring_enq(ring, RING_MASK, qi);
+ ring_enq(ring, ring_mask, qi);
} else if (queue_is_atomic(qi)) {
/* Hold queue during atomic access */
sched_local.stash_qi = qi;
} else {
/* Continue scheduling the queue */
- ring_enq(ring, RING_MASK, qi);
+ ring_enq(ring, ring_mask, qi);
}
handle = queue_from_index(qi);
@@ -916,7 +969,7 @@ static inline int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[],
int i, num_grp;
int ret;
int first, grp_id;
- uint16_t round;
+ uint16_t spread_round, grp_round;
uint32_t epoch;
if (sched_local.stash_num) {
@@ -933,15 +986,17 @@ static inline int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[],
if (odp_unlikely(sched_local.pause))
return 0;
- /* Each thread prefers a priority queue. Poll weight table avoids
+ /* Each thread prefers a priority queue. Spread weight table avoids
* starvation of other priority queues on low thread counts. */
- round = sched_local.round + 1;
+ spread_round = sched_local.spread_round;
+ grp_round = (sched_local.grp_round++) & (GRP_WEIGHT_TBL_SIZE - 1);
- if (odp_unlikely(round == WEIGHT_TBL_SIZE))
- round = 0;
+ if (odp_unlikely(spread_round + 1 >= sched->max_spread))
+ sched_local.spread_round = 0;
+ else
+ sched_local.spread_round = spread_round + 1;
- sched_local.round = round;
- first = sched_local.weight_tbl[round];
+ first = sched_local.spread_tbl[spread_round];
epoch = odp_atomic_load_acq_u32(&sched->grp_epoch);
num_grp = sched_local.num_grp;
@@ -951,7 +1006,7 @@ static inline int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[],
sched_local.grp_epoch = epoch;
}
- grp_id = sched_local.grp_weight[round];
+ grp_id = sched_local.grp_weight[grp_round];
/* Schedule queues per group and priority */
for (i = 0; i < num_grp; i++) {
diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c
index f5974442d..023ab1edd 100644
--- a/platform/linux-generic/odp_schedule_scalable.c
+++ b/platform/linux-generic/odp_schedule_scalable.c
@@ -21,8 +21,8 @@
#include <odp_internal.h>
#include <odp_config_internal.h>
#include <odp_debug_internal.h>
-#include <_ishm_internal.h>
-#include <_ishmpool_internal.h>
+#include <odp_ishm_internal.h>
+#include <odp_ishmpool_internal.h>
#include <odp_align_internal.h>
#include <odp_buffer_inlines.h>
diff --git a/platform/linux-generic/odp_shared_memory.c b/platform/linux-generic/odp_shared_memory.c
index c9b04dfdb..edf261af7 100644
--- a/platform/linux-generic/odp_shared_memory.c
+++ b/platform/linux-generic/odp_shared_memory.c
@@ -11,7 +11,7 @@
#include <odp/api/std_types.h>
#include <odp/api/shared_memory.h>
#include <odp/api/plat/strong_types.h>
-#include <_ishm_internal.h>
+#include <odp_ishm_internal.h>
#include <odp_internal.h>
#include <string.h>
diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c
index 7b9fed72d..0550416b0 100644
--- a/platform/linux-generic/pktio/dpdk.c
+++ b/platform/linux-generic/pktio/dpdk.c
@@ -27,6 +27,7 @@
#include <odp_libconfig_internal.h>
#include <protocols/eth.h>
+#include <protocols/udp.h>
#include <rte_config.h>
#include <rte_malloc.h>
@@ -48,6 +49,13 @@
#include <rte_string_fns.h>
#include <rte_version.h>
+/* NUMA is not supported on all platforms */
+#ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
+#include <numa.h>
+#else
+#define numa_num_configured_nodes() 1
+#endif
+
#if RTE_VERSION < RTE_VERSION_NUM(17, 5, 0, 0)
#define rte_log_set_global_level rte_set_log_level
#endif
@@ -94,22 +102,16 @@ void refer_constructors(void)
}
#endif
-static int lookup_opt(const char *path, const char *drv_name, int *val)
+static int lookup_opt(const char *opt_name, const char *drv_name, int *val)
{
const char *base = "pktio_dpdk";
- char opt_path[256];
- int ret = 0;
-
- /* Default option */
- snprintf(opt_path, sizeof(opt_path), "%s.%s", base, path);
- ret += _odp_libconfig_lookup_int(opt_path, val);
-
- /* Driver specific option overrides default option */
- snprintf(opt_path, sizeof(opt_path), "%s.%s.%s", base, drv_name, path);
- ret += _odp_libconfig_lookup_int(opt_path, val);
+ int ret;
+ ret = _odp_libconfig_lookup_ext_int(base, drv_name, opt_name, val);
if (ret == 0)
- ODP_ERR("Unable to find DPDK configuration option: %s\n", path);
+ ODP_ERR("Unable to find DPDK configuration option: %s\n",
+ opt_name);
+
return ret;
}
@@ -419,6 +421,7 @@ MEMPOOL_REGISTER_OPS(ops_stack);
#define IP4_CSUM_RESULT(m) (m->ol_flags & PKT_RX_IP_CKSUM_MASK)
#define L4_CSUM_RESULT(m) (m->ol_flags & PKT_RX_L4_CKSUM_MASK)
#define HAS_L4_PROTO(m, proto) ((m->packet_type & RTE_PTYPE_L4_MASK) == proto)
+#define UDP4_CSUM(_p) (((_odp_udphdr_t *)_odp_packet_l4_ptr(_p, NULL))->chksum)
#define PKTIN_CSUM_BITS 0x1C
@@ -451,6 +454,12 @@ static inline int pkt_set_ol_rx(odp_pktin_config_opt_t *pktin_cfg,
if (packet_csum_result == PKT_RX_L4_CKSUM_GOOD) {
pkt_hdr->p.input_flags.l4_chksum_done = 1;
} else if (packet_csum_result != PKT_RX_L4_CKSUM_UNKNOWN) {
+ if (pkt_hdr->p.input_flags.ipv4 &&
+ pkt_hdr->p.input_flags.udp &&
+ !UDP4_CSUM(packet_handle(pkt_hdr))) {
+ pkt_hdr->p.input_flags.l4_chksum_done = 1;
+ return 0;
+ }
if (pktin_cfg->bit.drop_udp_err)
return -1;
@@ -648,11 +657,7 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg,
if (!ipv4_chksum_pkt && !udp_chksum_pkt && !tcp_chksum_pkt)
return;
- if (pkt_p->l4_offset == ODP_PACKET_OFFSET_INVALID)
- return;
-
mbuf->l2_len = pkt_p->l3_offset - pkt_p->l2_offset;
- mbuf->l3_len = pkt_p->l4_offset - pkt_p->l3_offset;
if (l3_proto_v4)
mbuf->ol_flags = PKT_TX_IPV4;
@@ -663,8 +668,14 @@ static inline void pkt_set_ol_tx(odp_pktout_config_opt_t *pktout_cfg,
mbuf->ol_flags |= PKT_TX_IP_CKSUM;
((struct ipv4_hdr *)l3_hdr)->hdr_checksum = 0;
+ mbuf->l3_len = _ODP_IPV4HDR_IHL(*(uint8_t *)l3_hdr) * 4;
}
+ if (pkt_p->l4_offset == ODP_PACKET_OFFSET_INVALID)
+ return;
+
+ mbuf->l3_len = pkt_p->l4_offset - pkt_p->l3_offset;
+
l4_hdr = (void *)(mbuf_data + pkt_p->l4_offset);
if (udp_chksum_pkt) {
@@ -1082,6 +1093,7 @@ static int dpdk_pktio_init(void)
int32_t masklen;
int mem_str_len;
int cmd_len;
+ int numa_nodes;
cpu_set_t original_cpuset;
struct rte_config *cfg;
@@ -1116,21 +1128,29 @@ static int dpdk_pktio_init(void)
return -1;
}
- mem_str_len = snprintf(NULL, 0, "%d", DPDK_MEMORY_MB);
+ mem_str_len = snprintf(NULL, 0, "%d,", DPDK_MEMORY_MB);
+ numa_nodes = numa_num_configured_nodes();
+
+ char mem_str[mem_str_len * numa_nodes];
+
+ for (i = 0; i < numa_nodes; i++)
+ sprintf(&mem_str[i * mem_str_len], "%d,", DPDK_MEMORY_MB);
+ mem_str[mem_str_len * numa_nodes - 1] = '\0';
cmdline = getenv("ODP_PKTIO_DPDK_PARAMS");
if (cmdline == NULL)
cmdline = "";
/* masklen includes the terminating null as well */
- cmd_len = strlen("odpdpdk -c -m ") + masklen + mem_str_len +
- strlen(cmdline) + strlen(" ");
+ cmd_len = strlen("odpdpdk -c --socket-mem ") + masklen +
+ strlen(mem_str) + strlen(cmdline) + strlen(" ");
char full_cmd[cmd_len];
/* first argument is facility log, simply bind it to odpdpdk for now.*/
- cmd_len = snprintf(full_cmd, cmd_len, "odpdpdk -c %s -m %d %s",
- mask_str, DPDK_MEMORY_MB, cmdline);
+ cmd_len = snprintf(full_cmd, cmd_len,
+ "odpdpdk -c %s --socket-mem %s %s", mask_str,
+ mem_str, cmdline);
for (i = 0, dpdk_argc = 1; i < cmd_len; ++i) {
if (isspace(full_cmd[i]))
diff --git a/platform/linux-generic/pktio/ipc.c b/platform/linux-generic/pktio/ipc.c
index 6dcc7a593..08e3e4bc9 100644
--- a/platform/linux-generic/pktio/ipc.c
+++ b/platform/linux-generic/pktio/ipc.c
@@ -11,7 +11,7 @@
#include <odp_packet_io_internal.h>
#include <odp/api/system_info.h>
#include <odp_shm_internal.h>
-#include <_ishm_internal.h>
+#include <odp_ishm_internal.h>
#include <sys/mman.h>
#include <sys/stat.h>
diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c
index 7fc0fd61c..fc80e8019 100644
--- a/platform/linux-generic/pktio/loop.c
+++ b/platform/linux-generic/pktio/loop.c
@@ -15,6 +15,7 @@
#include <odp_debug_internal.h>
#include <odp/api/plat/packet_flag_inlines.h>
#include <odp/api/hints.h>
+#include <odp/api/plat/byteorder_inlines.h>
#include <odp_queue_if.h>
#include <protocols/eth.h>
@@ -32,6 +33,7 @@
static const char pktio_loop_mac[] = {0x02, 0xe9, 0x34, 0x80, 0x73, 0x01};
static int loopback_stats_reset(pktio_entry_t *pktio_entry);
+static int loopback_init_capability(pktio_entry_t *pktio_entry);
static int loopback_open(odp_pktio_t id, pktio_entry_t *pktio_entry,
const char *devname, odp_pool_t pool ODP_UNUSED)
@@ -61,6 +63,7 @@ static int loopback_open(odp_pktio_t id, pktio_entry_t *pktio_entry,
return -1;
loopback_stats_reset(pktio_entry);
+ loopback_init_capability(pktio_entry);
return 0;
}
@@ -172,6 +175,85 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
return num_rx;
}
+#define OL_TX_CHKSUM_PKT(_cfg, _capa, _proto, _ovr_set, _ovr) \
+ (_capa && _proto && (_ovr_set ? _ovr : _cfg))
+
+static inline int check_proto(void *l3_hdr,
+ uint32_t l3_len,
+ odp_bool_t *l3_proto_v4,
+ uint8_t *l4_proto)
+{
+ uint8_t l3_proto_ver = _ODP_IPV4HDR_VER(*(uint8_t *)l3_hdr);
+
+ if (l3_proto_ver == _ODP_IPV4 && l3_len >= _ODP_IPV4HDR_LEN) {
+ _odp_ipv4hdr_t *ip = l3_hdr;
+ uint16_t frag_offset = _odp_be_to_cpu_16(ip->frag_offset);
+
+ *l3_proto_v4 = 1;
+ if (!_ODP_IPV4HDR_IS_FRAGMENT(frag_offset))
+ *l4_proto = ip->proto;
+ else
+ *l4_proto = 255;
+
+ return 0;
+ } else if (l3_proto_ver == _ODP_IPV6 && l3_len >= _ODP_IPV6HDR_LEN) {
+ _odp_ipv6hdr_t *ipv6 = l3_hdr;
+
+ *l3_proto_v4 = 0;
+ *l4_proto = ipv6->next_hdr;
+
+ /* FIXME: check that packet is not a fragment !!!
+ * Might require parsing headers spanning several segments, so
+ * not implemented yet. */
+ return 0;
+ }
+
+ return -1;
+}
+
+static inline void loopback_fix_checksums(odp_packet_t pkt,
+ odp_pktout_config_opt_t *pktout_cfg,
+ odp_pktout_config_opt_t *pktout_capa)
+{
+ odp_bool_t l3_proto_v4 = false;
+ uint8_t l4_proto;
+ void *l3_hdr;
+ uint32_t l3_len;
+ odp_bool_t ipv4_chksum_pkt, udp_chksum_pkt, tcp_chksum_pkt;
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+
+ l3_hdr = odp_packet_l3_ptr(pkt, &l3_len);
+
+ if (l3_hdr == NULL ||
+ check_proto(l3_hdr, l3_len, &l3_proto_v4, &l4_proto))
+ return;
+
+ ipv4_chksum_pkt = OL_TX_CHKSUM_PKT(pktout_cfg->bit.ipv4_chksum,
+ pktout_capa->bit.ipv4_chksum,
+ l3_proto_v4,
+ pkt_hdr->p.flags.l3_chksum_set,
+ pkt_hdr->p.flags.l3_chksum);
+ udp_chksum_pkt = OL_TX_CHKSUM_PKT(pktout_cfg->bit.udp_chksum,
+ pktout_capa->bit.udp_chksum,
+ l4_proto == _ODP_IPPROTO_UDP,
+ pkt_hdr->p.flags.l4_chksum_set,
+ pkt_hdr->p.flags.l4_chksum);
+ tcp_chksum_pkt = OL_TX_CHKSUM_PKT(pktout_cfg->bit.tcp_chksum,
+ pktout_capa->bit.tcp_chksum,
+ l4_proto == _ODP_IPPROTO_TCP,
+ pkt_hdr->p.flags.l4_chksum_set,
+ pkt_hdr->p.flags.l4_chksum);
+
+ if (ipv4_chksum_pkt)
+ _odp_packet_ipv4_chksum_insert(pkt);
+
+ if (tcp_chksum_pkt)
+ _odp_packet_tcp_chksum_insert(pkt);
+
+ if (udp_chksum_pkt)
+ _odp_packet_udp_chksum_insert(pkt);
+}
+
static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
const odp_packet_t pkt_tbl[], int num)
{
@@ -182,6 +264,9 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
int nb_tx = 0;
uint32_t bytes = 0;
uint32_t out_octets_tbl[num];
+ odp_pktout_config_opt_t *pktout_cfg = &pktio_entry->s.config.pktout;
+ odp_pktout_config_opt_t *pktout_capa =
+ &pktio_entry->s.capa.config.pktout;
if (odp_unlikely(num > QUEUE_MULTI_MAX))
num = QUEUE_MULTI_MAX;
@@ -218,6 +303,9 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
packet_subtype_set(pkt_tbl[i], ODP_EVENT_PACKET_BASIC);
}
+ for (i = 0; i < nb_tx; ++i)
+ loopback_fix_checksums(pkt_tbl[i], pktout_cfg, pktout_capa);
+
odp_ticketlock_lock(&pktio_entry->s.txl);
queue = queue_fn->from_ext(pktio_entry->s.pkt_loop.loopq);
@@ -255,9 +343,10 @@ static int loopback_link_status(pktio_entry_t *pktio_entry ODP_UNUSED)
return 1;
}
-static int loopback_capability(pktio_entry_t *pktio_entry ODP_UNUSED,
- odp_pktio_capability_t *capa)
+static int loopback_init_capability(pktio_entry_t *pktio_entry)
{
+ odp_pktio_capability_t *capa = &pktio_entry->s.capa;
+
memset(capa, 0, sizeof(odp_pktio_capability_t));
capa->max_input_queues = 1;
@@ -267,9 +356,26 @@ static int loopback_capability(pktio_entry_t *pktio_entry ODP_UNUSED,
odp_pktio_config_init(&capa->config);
capa->config.pktin.bit.ts_all = 1;
capa->config.pktin.bit.ts_ptp = 1;
+ capa->config.pktout.bit.ipv4_chksum = 1;
+ capa->config.pktout.bit.tcp_chksum = 1;
+ capa->config.pktout.bit.udp_chksum = 1;
capa->config.inbound_ipsec = 1;
capa->config.outbound_ipsec = 1;
+ capa->config.pktout.bit.ipv4_chksum_ena =
+ capa->config.pktout.bit.ipv4_chksum;
+ capa->config.pktout.bit.udp_chksum_ena =
+ capa->config.pktout.bit.udp_chksum;
+ capa->config.pktout.bit.tcp_chksum_ena =
+ capa->config.pktout.bit.tcp_chksum;
+
+ return 0;
+}
+
+static int loopback_capability(pktio_entry_t *pktio_entry ODP_UNUSED,
+ odp_pktio_capability_t *capa)
+{
+ *capa = pktio_entry->s.capa;
return 0;
}
diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c
index 4276d4bc1..835d81ebb 100644
--- a/platform/linux-generic/pktio/netmap.c
+++ b/platform/linux-generic/pktio/netmap.c
@@ -27,6 +27,8 @@
#include <odp_classification_datamodel.h>
#include <odp_classification_inlines.h>
#include <odp_classification_internal.h>
+#include <odp_libconfig_internal.h>
+
#include <inttypes.h>
@@ -46,6 +48,49 @@
static int disable_pktio; /** !0 this pktio disabled, 0 enabled */
static int netmap_stats_reset(pktio_entry_t *pktio_entry);
+static int lookup_opt(const char *opt_name, const char *drv_name, int *val)
+{
+ const char *base = "pktio_netmap";
+ int ret;
+
+ ret = _odp_libconfig_lookup_ext_int(base, drv_name, opt_name, val);
+ if (ret == 0)
+ ODP_ERR("Unable to find netmap configuration option: %s\n",
+ opt_name);
+
+ return ret;
+}
+
+static int init_options(pktio_entry_t *pktio_entry)
+{
+ netmap_opt_t *opt = &pktio_entry->s.pkt_nm.opt;
+
+ if (!lookup_opt("nr_rx_slots", "virt",
+ &opt->nr_rx_slots))
+ return -1;
+ if (opt->nr_rx_slots < 0 ||
+ opt->nr_rx_slots > 4096) {
+ ODP_ERR("Invalid number of RX slots\n");
+ return -1;
+ }
+
+ if (!lookup_opt("nr_tx_slots", "virt",
+ &opt->nr_tx_slots))
+ return -1;
+ if (opt->nr_tx_slots < 0 ||
+ opt->nr_tx_slots > 4096) {
+ ODP_ERR("Invalid number of TX slots\n");
+ return -1;
+ }
+
+ ODP_PRINT("netmap interface: %s\n",
+ pktio_entry->s.pkt_nm.if_name);
+ ODP_PRINT(" num_rx_desc: %d\n", opt->nr_rx_slots);
+ ODP_PRINT(" num_tx_desc: %d\n", opt->nr_tx_slots);
+
+ return 0;
+}
+
static int netmap_do_ioctl(pktio_entry_t *pktio_entry, unsigned long cmd,
int subcmd)
{
@@ -363,6 +408,12 @@ static int netmap_open(odp_pktio_t id ODP_UNUSED, pktio_entry_t *pktio_entry,
netdev);
snprintf(pkt_nm->if_name, sizeof(pkt_nm->if_name), "%s", netdev);
+ /* Initialize runtime options */
+ if (init_options(pktio_entry)) {
+ ODP_ERR("Initializing runtime options failed\n");
+ return -1;
+ }
+
/* Dummy open here to check if netmap module is available and to read
* capability info. */
desc = nm_open(pkt_nm->nm_name, NULL, 0, NULL);
@@ -526,6 +577,12 @@ static int netmap_start(pktio_entry_t *pktio_entry)
base_desc.self = &base_desc;
base_desc.mem = NULL;
+ if (pktio_entry->s.pkt_nm.is_virtual) {
+ base_desc.req.nr_rx_slots =
+ pktio_entry->s.pkt_nm.opt.nr_rx_slots;
+ base_desc.req.nr_tx_slots =
+ pktio_entry->s.pkt_nm.opt.nr_tx_slots;
+ }
base_desc.req.nr_ringid = 0;
if ((base_desc.req.nr_flags & NR_REG_MASK) == NR_REG_ALL_NIC ||
(base_desc.req.nr_flags & NR_REG_MASK) == NR_REG_ONE_NIC) {
@@ -539,6 +596,8 @@ static int netmap_start(pktio_entry_t *pktio_entry)
/* Only the first rx descriptor does mmap */
desc_ring = pkt_nm->rx_desc_ring;
flags = NM_OPEN_IFNAME | NETMAP_NO_TX_POLL;
+ if (pktio_entry->s.pkt_nm.is_virtual)
+ flags |= NM_OPEN_RING_CFG;
desc_ring[0].s.desc[0] = nm_open(pkt_nm->nm_name, NULL, flags,
&base_desc);
if (desc_ring[0].s.desc[0] == NULL) {
@@ -547,6 +606,8 @@ static int netmap_start(pktio_entry_t *pktio_entry)
}
/* Open rest of the rx descriptors (one per netmap ring) */
flags = NM_OPEN_IFNAME | NETMAP_NO_TX_POLL | NM_OPEN_NO_MMAP;
+ if (pktio_entry->s.pkt_nm.is_virtual)
+ flags |= NM_OPEN_RING_CFG;
for (i = 0; i < pktio_entry->s.num_in_queue; i++) {
for (j = desc_ring[i].s.first; j <= desc_ring[i].s.last; j++) {
if (i == 0 && j == 0) { /* First already opened */
@@ -568,6 +629,8 @@ static int netmap_start(pktio_entry_t *pktio_entry)
/* Open tx descriptors */
desc_ring = pkt_nm->tx_desc_ring;
flags = NM_OPEN_IFNAME | NM_OPEN_NO_MMAP;
+ if (pktio_entry->s.pkt_nm.is_virtual)
+ flags |= NM_OPEN_RING_CFG;
if ((base_desc.req.nr_flags & NR_REG_MASK) == NR_REG_ALL_NIC) {
base_desc.req.nr_flags &= ~NR_REG_ALL_NIC;
diff --git a/scripts/build-pktio-dpdk b/scripts/build-pktio-dpdk
index 26afd97c9..b0c0a4d0e 100755
--- a/scripts/build-pktio-dpdk
+++ b/scripts/build-pktio-dpdk
@@ -16,7 +16,7 @@ if [ "$?" != "0" ]; then
exit 1
fi
-git -c advice.detachedHead=false clone -q --depth=1 --single-branch --branch=v17.08 http://dpdk.org/git/dpdk dpdk
+git -c advice.detachedHead=false clone -q --depth=1 --single-branch --branch=17.11 http://dpdk.org/git/dpdk-stable dpdk
pushd dpdk
git log --oneline --decorate
diff --git a/scripts/ci-checkpatches.sh b/scripts/ci-checkpatches.sh
index 798efe052..383045cd7 100755
--- a/scripts/ci-checkpatches.sh
+++ b/scripts/ci-checkpatches.sh
@@ -7,14 +7,14 @@ echo "Run checkpatch for ${PATCHES}"
# validate only the latest commit if it's not merge commit.
if [ "$PATCHES" = "" ]; then
- git format-patch -1 HEAD;
+ git format-patch -1 -M HEAD;
perl ./scripts/checkpatch.pl *.patch;
exit $?
fi
git show --summary HEAD| grep -q '^Merge:';
if [ $? -ne 0 ]; then
- git format-patch -1 HEAD;
+ git format-patch -1 -M HEAD;
perl ./scripts/checkpatch.pl *.patch;
exit $?
fi
diff --git a/test/miscellaneous/odp_api_from_cpp.cpp b/test/miscellaneous/odp_api_from_cpp.cpp
index 4578ae4be..7ed72ff17 100644
--- a/test/miscellaneous/odp_api_from_cpp.cpp
+++ b/test/miscellaneous/odp_api_from_cpp.cpp
@@ -1,11 +1,11 @@
-#include <cstdio>
+#include <iostream>
#include <odp_api.h>
#include <odp/helper/odph_api.h>
int main(int argc ODP_UNUSED, const char *argv[] ODP_UNUSED)
{
- printf("\tODP API version: %s\n", odp_version_api_str());
- printf("\tODP implementation version: %s\n", odp_version_impl_str());
+ std::cout << "\tODP API version: " << odp_version_api_str() << std::endl;
+ std::cout << "\tODP implementation version: " << odp_version_impl_str() << std::endl;
return 0;
}
diff --git a/test/performance/.gitignore b/test/performance/.gitignore
index 72035e002..4c4ab6ed5 100644
--- a/test/performance/.gitignore
+++ b/test/performance/.gitignore
@@ -3,8 +3,10 @@
odp_atomic
odp_bench_packet
odp_crypto
+odp_ipsec
odp_l2fwd
odp_pktio_ordered
odp_pktio_perf
odp_sched_latency
+odp_sched_pktio
odp_scheduling
diff --git a/test/performance/Makefile.am b/test/performance/Makefile.am
index e50c840b3..a1c6c98a7 100644
--- a/test/performance/Makefile.am
+++ b/test/performance/Makefile.am
@@ -4,18 +4,21 @@ TESTS_ENVIRONMENT += TEST_DIR=${builddir}
EXECUTABLES = odp_bench_packet \
odp_crypto \
+ odp_ipsec \
odp_pktio_perf
COMPILE_ONLY = odp_l2fwd \
odp_pktio_ordered \
odp_sched_latency \
+ odp_sched_pktio \
odp_scheduling
TESTSCRIPTS = odp_l2fwd_run.sh \
odp_sched_latency_run.sh \
+ odp_sched_pktio_run.sh \
odp_scheduling_run.sh
-if HAVE_PCAP
+if HAVE_PMD_PCAP
TESTSCRIPTS += odp_pktio_ordered_run.sh
endif
@@ -29,8 +32,10 @@ bin_PROGRAMS = $(EXECUTABLES) $(COMPILE_ONLY)
odp_bench_packet_SOURCES = odp_bench_packet.c
odp_crypto_SOURCES = odp_crypto.c
+odp_ipsec_SOURCES = odp_ipsec.c
odp_pktio_ordered_SOURCES = odp_pktio_ordered.c dummy_crc.h
odp_sched_latency_SOURCES = odp_sched_latency.c
+odp_sched_pktio_SOURCES = odp_sched_pktio.c
odp_scheduling_SOURCES = odp_scheduling.c
odp_pktio_perf_SOURCES = odp_pktio_perf.c
diff --git a/test/performance/odp_ipsec.c b/test/performance/odp_ipsec.c
new file mode 100644
index 000000000..587293dbe
--- /dev/null
+++ b/test/performance/odp_ipsec.c
@@ -0,0 +1,1170 @@
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include "config.h"
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif /* _GNU_SOURCE */
+
+#include <stdlib.h>
+#include <string.h>
+#include <getopt.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+
+#include <odp_api.h>
+#include <odp/helper/odph_api.h>
+
+#define app_err(fmt, ...) \
+ fprintf(stderr, "%s:%d:%s(): Error: " fmt, __FILE__, \
+ __LINE__, __func__, ##__VA_ARGS__)
+
+/** @def POOL_NUM_PKT
+ * Number of packets in the pool
+ */
+#define POOL_NUM_PKT 64
+
+static uint8_t test_salt[16] = "0123456789abcdef";
+
+static uint8_t test_key16[16] = { 0x01, 0x02, 0x03, 0x04, 0x05,
+ 0x06, 0x07, 0x08, 0x09, 0x0a,
+ 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+ 0x10,
+};
+
+static uint8_t test_key20[20] = { 0x01, 0x02, 0x03, 0x04, 0x05,
+ 0x06, 0x07, 0x08, 0x09, 0x0a,
+ 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+ 0x10, 0x11, 0x12, 0x13, 0x14,
+};
+
+static uint8_t test_key24[24] = { 0x01, 0x02, 0x03, 0x04, 0x05,
+ 0x06, 0x07, 0x08, 0x09, 0x0a,
+ 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+ 0x10, 0x11, 0x12, 0x13, 0x14,
+ 0x15, 0x16, 0x17, 0x18
+};
+
+static uint8_t test_key32[32] = { 0x01, 0x02, 0x03, 0x04, 0x05,
+ 0x06, 0x07, 0x08, 0x09, 0x0a,
+ 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+ 0x10, 0x11, 0x12, 0x13, 0x14,
+ 0x15, 0x16, 0x17, 0x18, 0x19,
+ 0x1a, 0x1b, 0x1c, 0x1d, 0x1e,
+ 0x1f, 0x20,
+};
+
+static uint8_t test_key64[64] = { 0x01, 0x02, 0x03, 0x04, 0x05,
+ 0x06, 0x07, 0x08, 0x09, 0x0a,
+ 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+ 0x10, 0x11, 0x12, 0x13, 0x14,
+ 0x15, 0x16, 0x17, 0x18, 0x19,
+ 0x1a, 0x1b, 0x1c, 0x1d, 0x1e,
+ 0x1f, 0x20, 0x21, 0x22, 0x23,
+ 0x24, 0x25, 0x26, 0x27, 0x28,
+ 0x29, 0x2a, 0x4b, 0x2c, 0x2d,
+ 0x2e, 0x2f, 0x30, 0x31, 0x32,
+ 0x33, 0x34, 0x55, 0x36, 0x37,
+ 0x38, 0x39, 0x5a, 0x3b, 0x3c,
+ 0x3d, 0x3e, 0x5f, 0x40,
+};
+
+/**
+ * Structure that holds template for sa create call
+ * for different algorithms supported by test
+ */
+typedef struct {
+ const char *name; /**< Algorithm name */
+ odp_ipsec_crypto_param_t crypto; /**< Prefilled SA crypto param */
+} ipsec_alg_config_t;
+
+/**
+ * Parsed command line crypto arguments. Describes test configuration.
+ */
+typedef struct {
+ /**
+ * If non zero prints content of packets. Enabled by -d or
+ * --debug option.
+ */
+ int debug_packets;
+
+ /**
+ * If non zero Try to run crypto operation in place. Note some
+ * implementation may not support such mode. Enabled by -n or
+ * --inplace option.
+ */
+ int in_place;
+
+ /**
+ * Maximum number of outstanding encryption requests. Note code
+ * poll for results over queue and if nothing is available it can
+ * submit more encryption requests up to maximum number specified by
+ * this option. Specified through -f or --flight option.
+ */
+ int in_flight;
+
+ /**
+ * Number of iteration to repeat crypto operation to get good
+ * average number. Specified through -i or --terations option.
+ * Default is 10000.
+ */
+ int iteration_count;
+
+ /**
+ * Payload size to test. If 0 set of predefined payload sizes
+ * is tested. Specified through -p or --payload option.
+ */
+ unsigned int payload_length;
+
+ /**
+ * Pointer to selected algorithm to test. If NULL all available
+ * alogorthims are tested. Name of algorithm is passed through
+ * -a or --algorithm option.
+ */
+ ipsec_alg_config_t *alg_config;
+
+ /**
+ * Use scheduler to get completion events from crypto operation.
+ * Specified through -s argument.
+ * */
+ int schedule;
+
+ /*
+ * Poll completion queue for crypto completion events.
+ * Specified through -p argument.
+ */
+ int poll;
+
+ /*
+ * Use tunnel instead of transport mode.
+ * Specified through -t argument.
+ */
+ int tunnel;
+
+ /*
+ * Use AH transformation.
+ * Specified through -u argument.
+ */
+ int ah;
+} ipsec_args_t;
+
+/*
+ * Helper structure that holds averages for test of one algorithm
+ * for given payload size.
+ */
+typedef struct {
+ /**
+ * Elapsed time for one crypto operation.
+ */
+ double elapsed;
+
+ /**
+ * CPU time spent pre one crypto operation by whole process
+ * i.e include current and all other threads in process.
+ * It is filled with 'getrusage(RUSAGE_SELF, ...)' call.
+ */
+ double rusage_self;
+
+ /**
+ * CPU time spent per one crypto operation by current thread
+ * only. It is filled with 'getrusage(RUSAGE_THREAD, ...)'
+ * call.
+ */
+ double rusage_thread;
+} ipsec_run_result_t;
+
+/**
+ * Structure holds one snap to misc times of current process.
+ */
+typedef struct {
+ struct timeval tv; /**< Elapsed time */
+ struct rusage ru_self; /**< Rusage value for whole process */
+ struct rusage ru_thread; /**< Rusage value for current thread */
+} time_record_t;
+
+/**
+ * Set of predefined payloads.
+ */
+static unsigned int global_payloads[] = {
+ 64,
+ 256,
+ 1024,
+ 8192,
+ 16384
+};
+
+/** Number of payloads used in the test */
+static unsigned int global_num_payloads;
+
+/**
+ * Set of known algorithms to test
+ */
+static ipsec_alg_config_t algs_config[] = {
+ {
+ .name = "3des-cbc-null",
+ .crypto = {
+ .cipher_alg = ODP_CIPHER_ALG_3DES_CBC,
+ .cipher_key = {
+ .data = test_key24,
+ .length = sizeof(test_key24)
+ },
+ .auth_alg = ODP_AUTH_ALG_NULL
+ },
+ },
+ {
+ .name = "3des-cbc-hmac-md5-96",
+ .crypto = {
+ .cipher_alg = ODP_CIPHER_ALG_3DES_CBC,
+ .cipher_key = {
+ .data = test_key24,
+ .length = sizeof(test_key24)
+ },
+ .auth_alg = ODP_AUTH_ALG_MD5_HMAC,
+ .auth_key = {
+ .data = test_key16,
+ .length = sizeof(test_key16)
+ },
+ },
+ },
+ {
+ .name = "null-hmac-md5-96",
+ .crypto = {
+ .cipher_alg = ODP_CIPHER_ALG_NULL,
+ .auth_alg = ODP_AUTH_ALG_MD5_HMAC,
+ .auth_key = {
+ .data = test_key16,
+ .length = sizeof(test_key16)
+ },
+ },
+ },
+ {
+ .name = "aes-cbc-null",
+ .crypto = {
+ .cipher_alg = ODP_CIPHER_ALG_AES_CBC,
+ .cipher_key = {
+ .data = test_key16,
+ .length = sizeof(test_key16)
+ },
+ .auth_alg = ODP_AUTH_ALG_NULL
+ },
+ },
+ {
+ .name = "aes-cbc-hmac-sha1-96",
+ .crypto = {
+ .cipher_alg = ODP_CIPHER_ALG_AES_CBC,
+ .cipher_key = {
+ .data = test_key16,
+ .length = sizeof(test_key16)
+ },
+ .auth_alg = ODP_AUTH_ALG_SHA1_HMAC,
+ .auth_key = {
+ .data = test_key20,
+ .length = sizeof(test_key20)
+ },
+ },
+ },
+ {
+ .name = "aes-ctr-null",
+ .crypto = {
+ .cipher_alg = ODP_CIPHER_ALG_AES_CTR,
+ .cipher_key = {
+ .data = test_key16,
+ .length = sizeof(test_key16)
+ },
+ .auth_alg = ODP_AUTH_ALG_NULL
+ },
+ },
+ {
+ .name = "aes-ctr-hmac-sha1-96",
+ .crypto = {
+ .cipher_alg = ODP_CIPHER_ALG_AES_CTR,
+ .cipher_key = {
+ .data = test_key16,
+ .length = sizeof(test_key16)
+ },
+ .auth_alg = ODP_AUTH_ALG_SHA1_HMAC,
+ .auth_key = {
+ .data = test_key20,
+ .length = sizeof(test_key20)
+ },
+ },
+ },
+ {
+ .name = "null-hmac-sha1-96",
+ .crypto = {
+ .cipher_alg = ODP_CIPHER_ALG_NULL,
+ .auth_alg = ODP_AUTH_ALG_SHA1_HMAC,
+ .auth_key = {
+ .data = test_key20,
+ .length = sizeof(test_key20)
+ },
+ },
+ },
+ {
+ .name = "null-hmac-sha256-128",
+ .crypto = {
+ .cipher_alg = ODP_CIPHER_ALG_NULL,
+ .auth_alg = ODP_AUTH_ALG_SHA256_HMAC,
+ .auth_key = {
+ .data = test_key32,
+ .length = sizeof(test_key32)
+ },
+ },
+ },
+ {
+ .name = "null-hmac-sha512-256",
+ .crypto = {
+ .cipher_alg = ODP_CIPHER_ALG_NULL,
+ .auth_alg = ODP_AUTH_ALG_SHA512_HMAC,
+ .auth_key = {
+ .data = test_key64,
+ .length = sizeof(test_key64)
+ },
+ },
+ },
+ {
+ .name = "null-aes-gmac",
+ .crypto = {
+ .cipher_alg = ODP_CIPHER_ALG_NULL,
+ .auth_alg = ODP_AUTH_ALG_AES_GMAC,
+ .auth_key = {
+ .data = test_key16,
+ .length = sizeof(test_key16)
+ },
+ .cipher_key_extra = {
+ .data = test_salt,
+ .length = 4,
+ },
+ },
+ },
+ {
+ .name = "aes-gcm",
+ .crypto = {
+ .cipher_alg = ODP_CIPHER_ALG_AES_GCM,
+ .cipher_key = {
+ .data = test_key16,
+ .length = sizeof(test_key16)
+ },
+ .cipher_key_extra = {
+ .data = test_salt,
+ .length = 4,
+ },
+ .auth_alg = ODP_AUTH_ALG_AES_GCM,
+ },
+ },
+ {
+ .name = "aes-ccm",
+ .crypto = {
+ .cipher_alg = ODP_CIPHER_ALG_AES_CCM,
+ .cipher_key = {
+ .data = test_key16,
+ .length = sizeof(test_key16)
+ },
+ .cipher_key_extra = {
+ .data = test_salt,
+ .length = 3,
+ },
+ .auth_alg = ODP_AUTH_ALG_AES_CCM,
+ },
+ },
+ {
+ .name = "chacha20-poly1305",
+ .crypto = {
+ .cipher_alg = ODP_CIPHER_ALG_CHACHA20_POLY1305,
+ .cipher_key = {
+ .data = test_key32,
+ .length = sizeof(test_key32)
+ },
+ .cipher_key_extra = {
+ .data = test_salt,
+ .length = 4,
+ },
+ .auth_alg = ODP_AUTH_ALG_CHACHA20_POLY1305,
+ },
+ },
+};
+
+/**
+ * Find corresponding config for given name. Returns NULL
+ * if config for given name is not found.
+ */
+static ipsec_alg_config_t *
+find_config_by_name(const char *name)
+{
+ unsigned int i;
+ ipsec_alg_config_t *ret = NULL;
+
+ for (i = 0; i < (sizeof(algs_config) / sizeof(ipsec_alg_config_t));
+ i++) {
+ if (strcmp(algs_config[i].name, name) == 0) {
+ ret = algs_config + i;
+ break;
+ }
+ }
+ return ret;
+}
+
+/**
+ * Helper function that prints list of algorithms that this
+ * test understands.
+ */
+static void
+print_config_names(const char *prefix)
+{
+ unsigned int i;
+
+ for (i = 0; i < (sizeof(algs_config) / sizeof(ipsec_alg_config_t));
+ i++) {
+ printf("%s %s\n", prefix, algs_config[i].name);
+ }
+}
+
+/**
+ * Snap current time values and put them into 'rec'.
+ */
+static void
+fill_time_record(time_record_t *rec)
+{
+ gettimeofday(&rec->tv, NULL);
+ getrusage(RUSAGE_SELF, &rec->ru_self);
+ getrusage(RUSAGE_THREAD, &rec->ru_thread);
+}
+
+/**
+ * Calculated CPU time difference for given two rusage structures.
+ * Note it adds user space and system time together.
+ */
+static unsigned long long
+get_rusage_diff(struct rusage *start, struct rusage *end)
+{
+ unsigned long long rusage_diff;
+ unsigned long long rusage_start;
+ unsigned long long rusage_end;
+
+ rusage_start = (start->ru_utime.tv_sec * 1000000) +
+ (start->ru_utime.tv_usec);
+ rusage_start += (start->ru_stime.tv_sec * 1000000) +
+ (start->ru_stime.tv_usec);
+
+ rusage_end = (end->ru_utime.tv_sec * 1000000) +
+ (end->ru_utime.tv_usec);
+ rusage_end += (end->ru_stime.tv_sec * 1000000) +
+ (end->ru_stime.tv_usec);
+
+ rusage_diff = rusage_end - rusage_start;
+
+ return rusage_diff;
+}
+
+/**
+ * Get diff for RUSAGE_SELF (whole process) between two time snap
+ * records.
+ */
+static unsigned long long
+get_rusage_self_diff(time_record_t *start, time_record_t *end)
+{
+ return get_rusage_diff(&start->ru_self, &end->ru_self);
+}
+
+/**
+ * Get diff for RUSAGE_THREAD (current thread only) between two
+ * time snap records.
+ */
+static unsigned long long
+get_rusage_thread_diff(time_record_t *start, time_record_t *end)
+{
+ return get_rusage_diff(&start->ru_thread, &end->ru_thread);
+}
+
+/**
+ * Get diff of elapsed time between two time snap records
+ */
+static unsigned long long
+get_elapsed_usec(time_record_t *start, time_record_t *end)
+{
+ unsigned long long s;
+ unsigned long long e;
+
+ s = (start->tv.tv_sec * 1000000) + (start->tv.tv_usec);
+ e = (end->tv.tv_sec * 1000000) + (end->tv.tv_usec);
+
+ return e - s;
+}
+
+/**
+ * Print header line for our report.
+ */
+static void
+print_result_header(void)
+{
+ printf("\n%30.30s %15s %15s %15s %15s %15s %15s\n",
+ "algorithm", "avg over #", "payload (bytes)", "elapsed (us)",
+ "rusg self (us)", "rusg thrd (us)", "throughput (Kb)");
+}
+
+/**
+ * Print one line of our report.
+ */
+static void
+print_result(ipsec_args_t *cargs,
+ unsigned int payload_length,
+ ipsec_alg_config_t *config,
+ ipsec_run_result_t *result)
+{
+ unsigned int throughput;
+
+ throughput = (1000000.0 / result->elapsed) * payload_length / 1024;
+ printf("%30.30s %15d %15d %15.3f %15.3f %15.3f %15d\n",
+ config->name, cargs->iteration_count, payload_length,
+ result->elapsed, result->rusage_self, result->rusage_thread,
+ throughput);
+}
+
+#define IPV4ADDR(a, b, c, d) odp_cpu_to_be_32((a << 24) | \
+ (b << 16) | \
+ (c << 8) | \
+ (d << 0))
+
+/**
+ * Create ODP IPsec SA for given config.
+ */
+static odp_ipsec_sa_t
+create_sa_from_config(ipsec_alg_config_t *config,
+ ipsec_args_t *cargs)
+{
+ odp_ipsec_sa_param_t param;
+ odp_queue_t out_queue;
+
+ odp_ipsec_sa_param_init(&param);
+ memcpy(&param.crypto, &config->crypto,
+ sizeof(odp_ipsec_crypto_param_t));
+
+ param.proto = ODP_IPSEC_ESP;
+ param.dir = ODP_IPSEC_DIR_OUTBOUND;
+
+ if (cargs->tunnel) {
+ uint32_t src = IPV4ADDR(10, 0, 111, 2);
+ uint32_t dst = IPV4ADDR(10, 0, 222, 2);
+ odp_ipsec_tunnel_param_t tunnel;
+
+ memset(&tunnel, 0, sizeof(tunnel));
+ tunnel.type = ODP_IPSEC_TUNNEL_IPV4;
+ tunnel.ipv4.src_addr = &src;
+ tunnel.ipv4.dst_addr = &dst;
+ tunnel.ipv4.ttl = 64;
+
+ param.mode = ODP_IPSEC_MODE_TUNNEL;
+ param.outbound.tunnel = tunnel;
+ } else {
+ param.mode = ODP_IPSEC_MODE_TRANSPORT;
+ }
+
+ if (cargs->schedule || cargs->poll) {
+ out_queue = odp_queue_lookup("ipsec-out");
+ if (out_queue == ODP_QUEUE_INVALID) {
+ app_err("ipsec-out queue not found\n");
+ return ODP_IPSEC_SA_INVALID;
+ }
+ param.dest_queue = out_queue;
+ } else {
+ param.dest_queue = ODP_QUEUE_INVALID;
+ }
+
+ return odp_ipsec_sa_create(&param);
+}
+
+static uint8_t test_data[] = {
+ /* IP */
+ 0x45, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00,
+ 0x40, 0x01, 0xac, 0x27, 0xc0, 0xa8, 0x6f, 0x02,
+ 0xc0, 0xa8, 0xde, 0x02,
+
+ /* ICMP */
+ 0x08, 0x00, 0xfb, 0x37, 0x12, 0x34, 0x00, 0x00
+};
+
+static odp_packet_t
+make_packet(odp_pool_t pkt_pool, unsigned int payload_length)
+{
+ odp_packet_t pkt;
+
+ if (payload_length < sizeof(test_data))
+ return ODP_PACKET_INVALID;
+
+ pkt = odp_packet_alloc(pkt_pool, payload_length);
+ if (pkt == ODP_PACKET_INVALID) {
+ app_err("failed to allocate buffer\n");
+ return pkt;
+ }
+
+ odp_packet_copy_from_mem(pkt, 0, sizeof(test_data), test_data);
+ odp_packet_l3_offset_set(pkt, 0);
+
+ uint8_t *mem = odp_packet_data(pkt);
+ ((odph_ipv4hdr_t *)mem)->tot_len = odp_cpu_to_be_16(payload_length);
+ memset(mem + sizeof(test_data), 1, payload_length - sizeof(test_data));
+
+ return pkt;
+}
+
+/**
+ * Run measurement iterations for given config and payload size.
+ * Result of run returned in 'result' out parameter.
+ */
+static int
+run_measure_one(ipsec_args_t *cargs,
+ odp_ipsec_sa_t sa,
+ unsigned int payload_length,
+ time_record_t *start,
+ time_record_t *end)
+{
+ odp_ipsec_out_param_t param;
+ odp_pool_t pkt_pool;
+ odp_packet_t pkt = ODP_PACKET_INVALID;
+ int rc = 0;
+
+ pkt_pool = odp_pool_lookup("packet_pool");
+ if (pkt_pool == ODP_POOL_INVALID) {
+ app_err("pkt_pool not found\n");
+ return -1;
+ }
+
+ int packets_sent = 0;
+ int packets_received = 0;
+
+ /* Initialize parameters block */
+ memset(&param, 0, sizeof(param));
+ param.num_sa = 1;
+ param.num_opt = 0;
+ param.sa = &sa;
+
+ fill_time_record(start);
+
+ while ((packets_sent < cargs->iteration_count) ||
+ (packets_received < cargs->iteration_count)) {
+ if ((packets_sent < cargs->iteration_count) &&
+ (packets_sent - packets_received <
+ cargs->in_flight)) {
+ odp_packet_t out_pkt;
+ int num_out = 1;
+
+ pkt = make_packet(pkt_pool, payload_length);
+ if (ODP_PACKET_INVALID == pkt)
+ return -1;
+
+ out_pkt = cargs->in_place ? pkt : ODP_PACKET_INVALID;
+
+ if (cargs->debug_packets)
+ odp_packet_print_data(pkt, 0,
+ odp_packet_len(pkt));
+
+ rc = odp_ipsec_out(&pkt, 1,
+ &out_pkt, &num_out,
+ &param);
+ if (rc <= 0) {
+ app_err("failed odp_ipsec_out: rc = %d\n",
+ rc);
+ odp_packet_free(pkt);
+ break;
+ }
+ if (odp_packet_has_error(out_pkt)) {
+ odp_ipsec_packet_result_t result;
+
+ odp_ipsec_result(&result, out_pkt);
+ app_err("Received error packet: %d\n",
+ result.status.error.all);
+ }
+ packets_sent += rc;
+ packets_received += num_out;
+ if (cargs->debug_packets)
+ odp_packet_print_data(out_pkt, 0,
+ odp_packet_len(out_pkt));
+ odp_packet_free(out_pkt);
+ }
+ }
+
+ fill_time_record(end);
+
+ return rc < 0 ? rc : 0;
+}
+
+static int
+run_measure_one_async(ipsec_args_t *cargs,
+ odp_ipsec_sa_t sa,
+ unsigned int payload_length,
+ time_record_t *start,
+ time_record_t *end)
+{
+ odp_ipsec_out_param_t param;
+ odp_pool_t pkt_pool;
+ odp_queue_t out_queue;
+ odp_packet_t pkt = ODP_PACKET_INVALID;
+ int rc = 0;
+
+ pkt_pool = odp_pool_lookup("packet_pool");
+ if (pkt_pool == ODP_POOL_INVALID) {
+ app_err("pkt_pool not found\n");
+ return -1;
+ }
+
+ out_queue = odp_queue_lookup("ipsec-out");
+ if (out_queue == ODP_QUEUE_INVALID) {
+ app_err("ipsec-out queue not found\n");
+ return -1;
+ }
+
+ int packets_sent = 0;
+ int packets_received = 0;
+
+ /* Initialize parameters block */
+ memset(&param, 0, sizeof(param));
+ param.num_sa = 1;
+ param.num_opt = 0;
+ param.sa = &sa;
+
+ fill_time_record(start);
+
+ while ((packets_sent < cargs->iteration_count) ||
+ (packets_received < cargs->iteration_count)) {
+ odp_event_t ev;
+
+ if ((packets_sent < cargs->iteration_count) &&
+ (packets_sent - packets_received <
+ cargs->in_flight)) {
+ pkt = make_packet(pkt_pool, payload_length);
+ if (ODP_PACKET_INVALID == pkt)
+ return -1;
+
+ if (cargs->debug_packets)
+ odp_packet_print_data(pkt, 0,
+ odp_packet_len(pkt));
+
+ rc = odp_ipsec_out_enq(&pkt, 1,
+ &param);
+ if (rc <= 0) {
+ app_err("failed odp_crypto_packet_op_enq: rc = %d\n",
+ rc);
+ odp_packet_free(pkt);
+ break;
+ }
+ packets_sent += rc;
+ }
+
+ if (cargs->schedule)
+ ev = odp_schedule(NULL,
+ ODP_SCHED_NO_WAIT);
+ else
+ ev = odp_queue_deq(out_queue);
+
+ while (ev != ODP_EVENT_INVALID) {
+ odp_packet_t out_pkt;
+ odp_ipsec_packet_result_t result;
+
+ out_pkt = odp_ipsec_packet_from_event(ev);
+ odp_ipsec_result(&result, out_pkt);
+
+ if (cargs->debug_packets)
+ odp_packet_print_data(out_pkt, 0,
+ odp_packet_len(out_pkt));
+ odp_packet_free(out_pkt);
+ packets_received++;
+ if (cargs->schedule)
+ ev = odp_schedule(NULL,
+ ODP_SCHED_NO_WAIT);
+ else
+ ev = odp_queue_deq(out_queue);
+ }
+ }
+
+ fill_time_record(end);
+
+ return rc < 0 ? rc : 0;
+}
+
+/**
+ * Process one algorithm. Note if paload size is specicified it is
+ * only one run. Or iterate over set of predefined payloads.
+ */
+static int
+run_measure_one_config(ipsec_args_t *cargs,
+ ipsec_alg_config_t *config)
+{
+ odp_ipsec_sa_t sa;
+ int rc = 0;
+ unsigned int num_payloads = global_num_payloads;
+ unsigned int *payloads = global_payloads;
+ unsigned int i;
+
+ sa = create_sa_from_config(config, cargs);
+ if (sa == ODP_IPSEC_SA_INVALID) {
+ app_err("IPsec SA create failed.\n");
+ return -1;
+ }
+
+ print_result_header();
+ if (cargs->payload_length) {
+ num_payloads = 1;
+ payloads = &cargs->payload_length;
+ }
+
+ for (i = 0; i < num_payloads; i++) {
+ double count;
+ ipsec_run_result_t result;
+ time_record_t start, end;
+
+ if (cargs->schedule || cargs->poll)
+ rc = run_measure_one_async(cargs, sa,
+ payloads[i],
+ &start, &end);
+ else
+ rc = run_measure_one(cargs, sa,
+ payloads[i],
+ &start, &end);
+ if (rc)
+ break;
+
+ count = get_elapsed_usec(&start, &end);
+ result.elapsed = count / cargs->iteration_count;
+
+ count = get_rusage_self_diff(&start, &end);
+ result.rusage_self = count / cargs->iteration_count;
+
+ count = get_rusage_thread_diff(&start, &end);
+ result.rusage_thread = count / cargs->iteration_count;
+
+ print_result(cargs, payloads[i],
+ config, &result);
+ }
+
+ odp_ipsec_sa_disable(sa);
+ if (cargs->schedule || cargs->poll) {
+ odp_queue_t out_queue = odp_queue_lookup("ipsec-out");
+ odp_ipsec_status_t status;
+
+ while (1) {
+ odp_event_t event = odp_queue_deq(out_queue);
+
+ if (event != ODP_EVENT_INVALID &&
+ odp_event_type(event) == ODP_EVENT_IPSEC_STATUS &&
+ odp_ipsec_status(&status, event) == ODP_IPSEC_OK &&
+ status.id == ODP_IPSEC_STATUS_SA_DISABLE &&
+ status.sa == sa)
+ break;
+ }
+ }
+ odp_ipsec_sa_destroy(sa);
+
+ return rc;
+}
+
+typedef struct thr_arg {
+ ipsec_args_t ipsec_args;
+ ipsec_alg_config_t *ipsec_alg_config;
+} thr_arg_t;
+
+static int run_thr_func(void *arg)
+{
+ thr_arg_t *thr_args = (thr_arg_t *)arg;
+
+ run_measure_one_config(&thr_args->ipsec_args,
+ thr_args->ipsec_alg_config);
+ return 0;
+}
+
+/**
+ * Prinf usage information
+ */
+static void usage(char *progname)
+{
+ printf("\n"
+ "Usage: %s OPTIONS\n"
+ " E.g. %s -i 100000\n"
+ "\n"
+ "OpenDataPlane crypto speed measure.\n"
+ "Optional OPTIONS\n"
+ " -a, --algorithm <name> Specify algorithm name (default all)\n"
+ " Supported values are:\n",
+ progname, progname);
+
+ print_config_names(" ");
+ printf(" -d, --debug Enable dump of processed packets.\n"
+ " -f, --flight <number> Max number of packet processed in parallel (default 1)\n"
+ " -i, --iterations <number> Number of iterations.\n"
+ " -n, --inplace Encrypt on place.\n"
+ " -l, --payload Payload length.\n"
+ " -s, --schedule Use scheduler for completion events.\n"
+ " -p, --poll Poll completion queue for completion events.\n"
+ " -t, --tunnel Use tunnel-mode IPsec transformation.\n"
+ " -u, --ah Use AH transformation instead of ESP.\n"
+ " -h, --help Display help and exit.\n"
+ "\n");
+}
+
+static void parse_args(int argc, char *argv[], ipsec_args_t *cargs)
+{
+ int opt;
+ int long_index;
+ static const struct option longopts[] = {
+ {"algorithm", optional_argument, NULL, 'a'},
+ {"debug", no_argument, NULL, 'd'},
+ {"flight", optional_argument, NULL, 'f'},
+ {"help", no_argument, NULL, 'h'},
+ {"iterations", optional_argument, NULL, 'i'},
+ {"inplace", no_argument, NULL, 'n'},
+ {"payload", optional_argument, NULL, 'l'},
+ {"sessions", optional_argument, NULL, 'm'},
+ {"poll", no_argument, NULL, 'p'},
+ {"schedule", no_argument, NULL, 's'},
+ {"tunnel", no_argument, NULL, 't'},
+ {"ah", no_argument, NULL, 'u'},
+ {NULL, 0, NULL, 0}
+ };
+
+ static const char *shortopts = "+a:c:df:hi:m:nl:sptu";
+
+ /* let helper collect its own arguments (e.g. --odph_proc) */
+ odph_parse_options(argc, argv, shortopts, longopts);
+
+ cargs->in_place = 0;
+ cargs->in_flight = 1;
+ cargs->debug_packets = 0;
+ cargs->iteration_count = 10000;
+ cargs->payload_length = 0;
+ cargs->alg_config = NULL;
+ cargs->schedule = 0;
+ cargs->ah = 0;
+
+ opterr = 0; /* do not issue errors on helper options */
+
+ while (1) {
+ opt = getopt_long(argc, argv, shortopts, longopts, &long_index);
+
+ if (opt == -1)
+ break; /* No more options */
+
+ switch (opt) {
+ case 'a':
+ cargs->alg_config = find_config_by_name(optarg);
+ if (!cargs->alg_config) {
+ printf("cannot test crypto '%s' configuration\n",
+ optarg);
+ usage(argv[0]);
+ exit(-1);
+ }
+ break;
+ case 'd':
+ cargs->debug_packets = 1;
+ break;
+ case 'i':
+ cargs->iteration_count = atoi(optarg);
+ break;
+ case 'f':
+ cargs->in_flight = atoi(optarg);
+ break;
+ case 'h':
+ usage(argv[0]);
+ exit(EXIT_SUCCESS);
+ break;
+ case 'n':
+ cargs->in_place = 1;
+ break;
+ case 'l':
+ cargs->payload_length = atoi(optarg);
+ break;
+ case 's':
+ cargs->schedule = 1;
+ break;
+ case 'p':
+ cargs->poll = 1;
+ break;
+ case 't':
+ cargs->tunnel = 1;
+ break;
+ case 'u':
+ cargs->ah = 1;
+ break;
+ default:
+ break;
+ }
+ }
+
+ optind = 1; /* reset 'extern optind' from the getopt lib */
+
+ if (cargs->schedule && cargs->poll) {
+ printf("-s (schedule) and -p (poll) options are not compatible\n");
+ usage(argv[0]);
+ exit(-1);
+ }
+}
+
+int main(int argc, char *argv[])
+{
+ ipsec_args_t cargs;
+ odp_pool_t pool;
+ odp_queue_param_t qparam;
+ odp_pool_param_t param;
+ odp_queue_t out_queue = ODP_QUEUE_INVALID;
+ thr_arg_t thr_arg;
+ odp_cpumask_t cpumask;
+ char cpumaskstr[ODP_CPUMASK_STR_SIZE];
+ int num_workers = 1;
+ odph_odpthread_t thr[num_workers];
+ odp_instance_t instance;
+ odp_pool_capability_t capa;
+ odp_ipsec_config_t config;
+ uint32_t max_seg_len;
+ unsigned int i;
+
+ memset(&cargs, 0, sizeof(cargs));
+
+ /* Parse and store the application arguments */
+ parse_args(argc, argv, &cargs);
+
+ /* Init ODP before calling anything else */
+ if (odp_init_global(&instance, NULL, NULL)) {
+ app_err("ODP global init failed.\n");
+ exit(EXIT_FAILURE);
+ }
+
+ /* Init this thread */
+ if (odp_init_local(instance, ODP_THREAD_WORKER)) {
+ app_err("ODP local init failed.\n");
+ exit(EXIT_FAILURE);
+ }
+
+ if (odp_pool_capability(&capa)) {
+ app_err("Pool capability request failed.\n");
+ exit(EXIT_FAILURE);
+ }
+
+ max_seg_len = capa.pkt.max_seg_len;
+
+ for (i = 0; i < sizeof(global_payloads) / sizeof(unsigned int); i++) {
+ if (global_payloads[i] > max_seg_len)
+ break;
+ }
+
+ global_num_payloads = i;
+
+ /* Create packet pool */
+ odp_pool_param_init(&param);
+ param.pkt.seg_len = max_seg_len;
+ param.pkt.len = max_seg_len;
+ param.pkt.num = POOL_NUM_PKT;
+ param.type = ODP_POOL_PACKET;
+ pool = odp_pool_create("packet_pool", &param);
+
+ if (pool == ODP_POOL_INVALID) {
+ app_err("packet pool create failed.\n");
+ exit(EXIT_FAILURE);
+ }
+ odp_pool_print(pool);
+
+ odp_ipsec_config_init(&config);
+ config.max_num_sa = 2;
+ config.inbound.chksums.all_chksum = 0;
+ config.outbound.all_chksum = 0;
+
+ odp_queue_param_init(&qparam);
+ if (cargs.schedule) {
+ qparam.type = ODP_QUEUE_TYPE_SCHED;
+ qparam.sched.prio = ODP_SCHED_PRIO_DEFAULT;
+ qparam.sched.sync = ODP_SCHED_SYNC_PARALLEL;
+ qparam.sched.group = ODP_SCHED_GROUP_ALL;
+ out_queue = odp_queue_create("ipsec-out", &qparam);
+ } else if (cargs.poll) {
+ qparam.type = ODP_QUEUE_TYPE_PLAIN;
+ out_queue = odp_queue_create("ipsec-out", &qparam);
+ }
+ if (cargs.schedule || cargs.poll) {
+ if (out_queue == ODP_QUEUE_INVALID) {
+ app_err("ipsec-out queue create failed.\n");
+ exit(EXIT_FAILURE);
+ }
+ config.inbound_mode = ODP_IPSEC_OP_MODE_ASYNC;
+ config.outbound_mode = ODP_IPSEC_OP_MODE_ASYNC;
+ config.inbound.default_queue = out_queue;
+ } else {
+ config.inbound_mode = ODP_IPSEC_OP_MODE_SYNC;
+ config.outbound_mode = ODP_IPSEC_OP_MODE_SYNC;
+ config.inbound.default_queue = ODP_QUEUE_INVALID;
+ }
+
+ if (cargs.schedule) {
+ printf("Run in async scheduled mode\n");
+
+ thr_arg.ipsec_args = cargs;
+ thr_arg.ipsec_alg_config = cargs.alg_config;
+ num_workers = odp_cpumask_default_worker(&cpumask,
+ num_workers);
+ (void)odp_cpumask_to_str(&cpumask, cpumaskstr,
+ sizeof(cpumaskstr));
+ printf("num worker threads: %i\n",
+ num_workers);
+ printf("first CPU: %i\n",
+ odp_cpumask_first(&cpumask));
+ printf("cpu mask: %s\n",
+ cpumaskstr);
+ } else if (cargs.poll) {
+ printf("Run in async poll mode\n");
+ } else {
+ printf("Run in sync mode\n");
+ }
+
+ memset(thr, 0, sizeof(thr));
+
+ if (cargs.alg_config) {
+ odph_odpthread_params_t thr_param;
+
+ memset(&thr_param, 0, sizeof(thr_param));
+ thr_param.start = run_thr_func;
+ thr_param.arg = &thr_arg;
+ thr_param.thr_type = ODP_THREAD_WORKER;
+ thr_param.instance = instance;
+
+ if (cargs.schedule) {
+ odph_odpthreads_create(&thr[0], &cpumask, &thr_param);
+ odph_odpthreads_join(&thr[0]);
+ } else {
+ run_measure_one_config(&cargs, cargs.alg_config);
+ }
+ } else {
+ unsigned int i;
+
+ for (i = 0;
+ i < (sizeof(algs_config) / sizeof(ipsec_alg_config_t));
+ i++) {
+ if (cargs.ah &&
+ algs_config[i].crypto.cipher_alg !=
+ ODP_CIPHER_ALG_NULL)
+ continue;
+ run_measure_one_config(&cargs, algs_config + i);
+ }
+ }
+
+ if (cargs.schedule || cargs.poll)
+ odp_queue_destroy(out_queue);
+ if (odp_pool_destroy(pool)) {
+ app_err("Error: pool destroy\n");
+ exit(EXIT_FAILURE);
+ }
+
+ if (odp_term_local()) {
+ app_err("Error: term local\n");
+ exit(EXIT_FAILURE);
+ }
+
+ if (odp_term_global(instance)) {
+ app_err("Error: term global\n");
+ exit(EXIT_FAILURE);
+ }
+
+ return 0;
+}
+
diff --git a/test/performance/odp_l2fwd_run.sh b/test/performance/odp_l2fwd_run.sh
index 6166c8b27..af8000607 100755
--- a/test/performance/odp_l2fwd_run.sh
+++ b/test/performance/odp_l2fwd_run.sh
@@ -67,8 +67,12 @@ run_l2fwd()
exit 1
fi
+ export ODP_PLATFORM_PARAMS="-m 512 --file-prefix="gen" \
+--proc-type auto --no-pci \
+--vdev net_pcap0,iface=$IF0"
+
# Run generator with one worker
- (odp_generator${EXEEXT} --interval $FLOOD_MODE -I $IF0 \
+ (odp_generator${EXEEXT} --interval $FLOOD_MODE -I 0 \
--srcip 192.168.0.1 --dstip 192.168.0.2 \
-m u -w 1 2>&1 > /dev/null) \
2>&1 > /dev/null &
@@ -84,8 +88,12 @@ run_l2fwd()
fi
LOG=odp_l2fwd_tmp.log
+ export ODP_PLATFORM_PARAMS="-m 512 --file-prefix="l2fwd" \
+--proc-type auto --no-pci --vdev net_pcap1,iface=$IF1 \
+--vdev net_pcap2,iface=$IF2"
+
# Max 2 workers
- $STDBUF odp_l2fwd${EXEEXT} -i $IF1,$IF2 -m 0 -t 30 -c 2 | tee $LOG
+ $STDBUF odp_l2fwd${EXEEXT} -i 0,1 -m 0 -t 30 -c 2 | tee $LOG
ret=$?
kill ${GEN_PID}
diff --git a/test/performance/odp_pktio_ordered_run.sh b/test/performance/odp_pktio_ordered_run.sh
index d7f238120..a4a7cb347 100755
--- a/test/performance/odp_pktio_ordered_run.sh
+++ b/test/performance/odp_pktio_ordered_run.sh
@@ -11,7 +11,7 @@ TEST_DIR="${TEST_DIR:-$(dirname $0)}"
DURATION=5
LOG=odp_pktio_ordered.log
LOOPS=100000000
-PASS_PPS=5000
+PASS_PPS=350
PCAP_IN=`find . ${TEST_SRC_DIR} $(dirname $0) -name udp64.pcap -print -quit`
PCAP_OUT=/dev/null
@@ -28,11 +28,14 @@ else
STDBUF=
fi
+export ODP_PLATFORM_PARAMS="--no-pci \
+--vdev net_pcap0,rx_pcap=${PCAP_IN},tx_pcap=${PCAP_OUT} \
+--vdev net_pcap1,rx_pcap=${PCAP_IN},tx_pcap=${PCAP_OUT}"
+
$STDBUF ${TEST_DIR}/odp_pktio_ordered${EXEEXT} \
- -i pcap:in=${PCAP_IN}:loops=$LOOPS,pcap:out=${PCAP_OUT} \
+ -i 0,1 \
-t $DURATION | tee $LOG
-
-ret=${PIPESTATUS[0]}
+ret=$?
if [ $ret -ne 0 ]; then
echo "FAIL: no odp_pktio_ordered${EXEEXT}"
diff --git a/test/performance/odp_sched_latency.c b/test/performance/odp_sched_latency.c
index d49a212a0..ca7201193 100644
--- a/test/performance/odp_sched_latency.c
+++ b/test/performance/odp_sched_latency.c
@@ -27,7 +27,6 @@
/* GNU lib C */
#include <getopt.h>
-#define MAX_WORKERS 64 /**< Maximum number of worker threads */
#define MAX_QUEUES 4096 /**< Maximum number of queues */
#define EVENT_POOL_SIZE (1024 * 1024) /**< Event pool size */
#define TEST_ROUNDS (4 * 1024 * 1024) /**< Test rounds for each thread */
@@ -105,7 +104,8 @@ typedef union ODP_ALIGNED_CACHE {
/** Test global variables */
typedef struct {
- core_stat_t core_stat[MAX_WORKERS]; /**< Core specific stats */
+ /** Core specific stats */
+ core_stat_t core_stat[ODP_THREAD_COUNT_MAX];
odp_barrier_t barrier; /**< Barrier for thread synchronization */
odp_pool_t pool; /**< Pool for allocating test events */
test_args_t args; /**< Parsed command line arguments */
@@ -617,8 +617,9 @@ static void parse_args(int argc, char *argv[], test_args_t *args)
}
/* Make sure arguments are valid */
- if (args->cpu_count > MAX_WORKERS)
- args->cpu_count = MAX_WORKERS;
+ /* -1 for main thread */
+ if (args->cpu_count > ODP_THREAD_COUNT_MAX - 1)
+ args->cpu_count = ODP_THREAD_COUNT_MAX - 1;
if (args->prio[LO_PRIO].queues > MAX_QUEUES)
args->prio[LO_PRIO].queues = MAX_QUEUES;
if (args->prio[HI_PRIO].queues > MAX_QUEUES)
diff --git a/test/performance/odp_sched_pktio.c b/test/performance/odp_sched_pktio.c
new file mode 100644
index 000000000..16d14b195
--- /dev/null
+++ b/test/performance/odp_sched_pktio.c
@@ -0,0 +1,800 @@
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <signal.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <odp_api.h>
+#include <odp/helper/odph_api.h>
+
+#define MAX_WORKERS 64
+#define MAX_PKTIOS 32
+#define MAX_PKTIO_NAME 31
+#define MAX_PKTIO_QUEUES MAX_WORKERS
+#define MAX_PKT_LEN 1514
+#define MAX_PKT_NUM (16 * 1024)
+#define MIN_PKT_SEG_LEN 64
+#define BURST_SIZE 32
+#define CHECK_PERIOD 10000
+#define MAX_PKTIO_INDEXES 256
+#define TEST_PASSED_LIMIT 5000
+
+typedef struct {
+ int worker_id;
+ void *test_global_ptr;
+} worker_arg_t;
+
+typedef struct ODP_ALIGNED_CACHE {
+ uint64_t rx_pkt;
+ uint64_t tx_pkt;
+} worker_stat_t;
+
+typedef struct {
+ volatile int stop_workers;
+ odp_barrier_t worker_start;
+
+ struct {
+ int num_worker;
+ int num_pktio;
+ uint8_t collect_stat;
+ } opt;
+
+ int max_workers;
+ odp_cpumask_t cpumask;
+ odp_instance_t instance;
+
+ int worker_cpu[MAX_WORKERS];
+
+ odp_pool_t pool;
+ uint32_t pkt_len;
+ uint32_t pkt_num;
+
+ unsigned int num_input_queues;
+ unsigned int num_output_queues;
+
+ struct {
+ char name[MAX_PKTIO_NAME + 1];
+ odp_pktio_t pktio;
+ int pktio_index;
+ int started;
+ odph_ethaddr_t my_addr;
+ odp_queue_t input_queue[MAX_PKTIO_QUEUES];
+ odp_pktout_queue_t pktout[MAX_PKTIO_QUEUES];
+
+ } pktio[MAX_PKTIOS];
+
+ worker_arg_t worker_arg[MAX_WORKERS];
+
+ /* Maps pktio input index to pktio[] index for output */
+ uint8_t pktio_map[MAX_PKTIO_INDEXES];
+
+ worker_stat_t worker_stat[MAX_WORKERS];
+ uint64_t rx_pkt_sum;
+ uint64_t tx_pkt_sum;
+
+} test_global_t;
+
+static test_global_t *test_global;
+
+static inline void set_dst_eth_addr(odph_ethaddr_t *eth_addr, int index)
+{
+ eth_addr->addr[0] = 0x02;
+ eth_addr->addr[1] = 0;
+ eth_addr->addr[2] = 0;
+ eth_addr->addr[3] = 0;
+ eth_addr->addr[4] = 0;
+ eth_addr->addr[5] = index;
+}
+
+static inline void fill_eth_addr(odp_packet_t pkt[], int num,
+ test_global_t *test_global, int out)
+{
+ odph_ethhdr_t *eth;
+ int i;
+
+ for (i = 0; i < num; ++i) {
+ eth = odp_packet_data(pkt[i]);
+
+ eth->src = test_global->pktio[out].my_addr;
+ set_dst_eth_addr(&eth->dst, out);
+ }
+}
+
+static int worker_thread(void *arg)
+{
+ odp_event_t ev[BURST_SIZE];
+ int num, sent, drop, in, out;
+ odp_pktout_queue_t pktout;
+ worker_arg_t *worker_arg = arg;
+ test_global_t *test_global = worker_arg->test_global_ptr;
+ int worker_id = worker_arg->worker_id;
+ uint32_t polls = 0;
+
+ printf("Worker %i started\n", worker_id);
+
+ /* Wait for other workers to start */
+ odp_barrier_wait(&test_global->worker_start);
+
+ while (1) {
+ odp_packet_t pkt[BURST_SIZE];
+
+ num = odp_schedule_multi(NULL, ODP_SCHED_NO_WAIT,
+ ev, BURST_SIZE);
+
+ polls++;
+
+ if (polls == CHECK_PERIOD) {
+ polls = 0;
+ if (test_global->stop_workers)
+ break;
+ }
+
+ if (num <= 0)
+ continue;
+
+ odp_packet_from_event_multi(pkt, ev, num);
+
+ in = odp_packet_input_index(pkt[0]);
+ out = test_global->pktio_map[in];
+ pktout = test_global->pktio[out].pktout[worker_id];
+
+ fill_eth_addr(pkt, num, test_global, out);
+
+ sent = odp_pktout_send(pktout, pkt, num);
+
+ if (odp_unlikely(sent < 0))
+ sent = 0;
+
+ drop = num - sent;
+
+ if (odp_unlikely(drop))
+ odp_packet_free_multi(&pkt[sent], drop);
+
+ if (odp_unlikely(test_global->opt.collect_stat)) {
+ test_global->worker_stat[worker_id].rx_pkt += num;
+ test_global->worker_stat[worker_id].tx_pkt += sent;
+ }
+ }
+
+ printf("Worker %i stopped\n", worker_id);
+
+ return 0;
+}
+
+static void sig_handler(int signo)
+{
+ (void)signo;
+
+ if (test_global) {
+ test_global->stop_workers = 1;
+ odp_mb_full();
+ }
+}
+
+/* Get rid of path in filename - only for unix-type paths using '/' */
+#define NO_PATH(x) (strrchr((x), '/') ? strrchr((x), '/') + 1 : (x))
+
+static void print_usage(const char *progname)
+{
+ printf("\n"
+ "Scheduler with packet IO test application.\n"
+ "\n"
+ "Usage: %s [options]\n"
+ "\n"
+ "OPTIONS:\n"
+ " -i, --interface <name> Packet IO interfaces (comma-separated, no spaces)\n"
+ " -c, --count <number> Worker thread count. Default: 1\n"
+ " -s, --stat Collect statistics.\n"
+ " -h, --help Display help and exit.\n\n",
+ NO_PATH(progname));
+}
+
+static int parse_options(int argc, char *argv[], test_global_t *test_global)
+{
+ int i, opt, long_index;
+ char *name, *str;
+ int len, str_len;
+ const struct option longopts[] = {
+ {"interface", required_argument, NULL, 'i'},
+ {"count", required_argument, NULL, 'c'},
+ {"stat", no_argument, NULL, 's'},
+ {"help", no_argument, NULL, 'h'},
+ {NULL, 0, NULL, 0}
+ };
+ const char *shortopts = "+i:c:sh";
+ int ret = 0;
+
+ test_global->opt.num_worker = 1;
+
+ /* let helper collect its own arguments (e.g. --odph_proc) */
+ odph_parse_options(argc, argv, shortopts, longopts);
+
+ while (1) {
+ opt = getopt_long(argc, argv, shortopts, longopts, &long_index);
+
+ if (opt == -1)
+ break; /* No more options */
+
+ switch (opt) {
+ case 'i':
+ i = 0;
+ str = optarg;
+ str_len = strlen(str);
+
+ while (str_len > 0) {
+ len = strcspn(str, ",");
+ str_len -= len + 1;
+
+ if (i == MAX_PKTIOS) {
+ printf("Error: Too many interfaces\n");
+ ret = -1;
+ break;
+ }
+
+ if (len > MAX_PKTIO_NAME) {
+ printf("Error: Too long interface name %s\n",
+ str);
+ ret = -1;
+ break;
+ }
+
+ name = test_global->pktio[i].name;
+ memcpy(name, str, len);
+ str += len + 1;
+ i++;
+ }
+
+ test_global->opt.num_pktio = i;
+
+ break;
+ case 'c':
+ test_global->opt.num_worker = atoi(optarg);
+ break;
+ case 's':
+ test_global->opt.collect_stat = 1;
+ break;
+ case 'h':
+ print_usage(argv[0]);
+ ret = -1;
+ break;
+ default:
+ ret = -1;
+ break;
+ }
+ }
+
+ return ret;
+}
+
+static int config_setup(test_global_t *test_global)
+{
+ int i, cpu;
+ odp_pool_capability_t pool_capa;
+ uint32_t pkt_len, pkt_num;
+ odp_cpumask_t *cpumask = &test_global->cpumask;
+
+ test_global->max_workers = odp_cpumask_default_worker(cpumask, 0);
+
+ if (test_global->opt.num_worker > test_global->max_workers ||
+ test_global->opt.num_worker > MAX_WORKERS) {
+ printf("Error: Too many workers %i.\n",
+ test_global->opt.num_worker);
+ return -1;
+ }
+
+ cpu = odp_cpumask_first(cpumask);
+ for (i = 0; i < test_global->opt.num_worker; ++i) {
+ test_global->worker_cpu[i] = cpu;
+ cpu = odp_cpumask_next(cpumask, cpu);
+ }
+
+ if (test_global->opt.num_pktio == 0) {
+ printf("Error: At least one pktio interface needed.\n");
+ return -1;
+ }
+
+ if (MAX_PKTIO_INDEXES <= odp_pktio_max_index()) {
+ printf("Error: Larger pktio_map[] table needed: %u\n",
+ odp_pktio_max_index());
+ return -1;
+ }
+
+ if (odp_pool_capability(&pool_capa)) {
+ printf("Error: Pool capability failed.\n");
+ return -1;
+ }
+
+ pkt_len = MAX_PKT_LEN;
+ pkt_num = MAX_PKT_NUM;
+
+ if (pool_capa.pkt.max_len && pkt_len > pool_capa.pkt.max_len)
+ pkt_len = pool_capa.pkt.max_len;
+
+ if (pool_capa.pkt.max_num && pkt_num > pool_capa.pkt.max_num)
+ pkt_num = pool_capa.pkt.max_num;
+
+ test_global->pkt_len = pkt_len;
+ test_global->pkt_num = pkt_num;
+
+ if (test_global->num_input_queues == 0)
+ test_global->num_input_queues = test_global->opt.num_worker;
+
+ if (test_global->num_output_queues == 0)
+ test_global->num_output_queues = test_global->opt.num_worker;
+
+ return 0;
+}
+
+static void print_config(test_global_t *test_global)
+{
+ char cpumask_str[ODP_CPUMASK_STR_SIZE];
+ int i;
+
+ odp_cpumask_to_str(&test_global->cpumask, cpumask_str,
+ ODP_CPUMASK_STR_SIZE);
+
+ printf("\n"
+ "Test configuration:\n"
+ " max workers: %i\n"
+ " available worker cpus: %s\n"
+ " num workers: %i\n"
+ " worker cpus: ",
+ test_global->max_workers,
+ cpumask_str,
+ test_global->opt.num_worker);
+
+ for (i = 0; i < test_global->opt.num_worker; i++)
+ printf(" %i", test_global->worker_cpu[i]);
+
+ printf("\n"
+ " num interfaces: %i\n"
+ " interface names: ", test_global->opt.num_pktio);
+
+ for (i = 0; i < test_global->opt.num_pktio; i++)
+ printf(" %s", test_global->pktio[i].name);
+
+ printf("\n"
+ " num input queues: %i\n"
+ " num output queues: %i\n",
+ test_global->num_input_queues, test_global->num_output_queues);
+
+ printf(" collect statistics: %u\n", test_global->opt.collect_stat);
+
+ printf("\n");
+}
+
+static void print_stat(test_global_t *test_global, uint64_t nsec)
+{
+ int i;
+ uint64_t rx, tx, drop;
+ uint64_t rx_sum = 0;
+ uint64_t tx_sum = 0;
+ double sec = 0.0;
+
+ printf("\nTest statistics\n");
+ printf(" worker rx_pkt tx_pkt dropped\n");
+
+ for (i = 0; i < test_global->opt.num_worker; i++) {
+ rx = test_global->worker_stat[i].rx_pkt;
+ tx = test_global->worker_stat[i].tx_pkt;
+ rx_sum += rx;
+ tx_sum += tx;
+
+ printf(" %6i %16" PRIu64 " %16" PRIu64 " %16" PRIu64 "\n",
+ i, rx, tx, rx - tx);
+ }
+
+ test_global->rx_pkt_sum = rx_sum;
+ test_global->tx_pkt_sum = tx_sum;
+ drop = rx_sum - tx_sum;
+
+ printf(" --------------------------------------------------\n");
+ printf(" total %16" PRIu64 " %16" PRIu64 " %16" PRIu64 "\n\n",
+ rx_sum, tx_sum, drop);
+
+ sec = nsec / 1000000000.0;
+ printf(" Total test time: %.2f sec\n", sec);
+ printf(" Rx packet rate: %.2f pps\n", rx_sum / sec);
+ printf(" Tx packet rate: %.2f pps\n", tx_sum / sec);
+ printf(" Drop rate: %.2f pps\n\n", drop / sec);
+}
+
+static int open_pktios(test_global_t *test_global)
+{
+ odp_pool_param_t pool_param;
+ odp_pktio_param_t pktio_param;
+ odp_pool_t pool;
+ odp_pktio_t pktio;
+ odp_pktio_capability_t pktio_capa;
+ odp_pktio_config_t pktio_config;
+ odp_pktin_queue_param_t pktin_param;
+ odp_pktout_queue_param_t pktout_param;
+ odp_schedule_sync_t sched_sync;
+ unsigned int num_input, num_output;
+ char *name;
+ int i, num_pktio, ret;
+
+ num_pktio = test_global->opt.num_pktio;
+ num_input = test_global->num_input_queues;
+ num_output = test_global->num_output_queues;
+
+ odp_pool_param_init(&pool_param);
+ pool_param.pkt.seg_len = MIN_PKT_SEG_LEN;
+ pool_param.pkt.len = test_global->pkt_len;
+ pool_param.pkt.num = test_global->pkt_num;
+ pool_param.type = ODP_POOL_PACKET;
+
+ pool = odp_pool_create("packet pool", &pool_param);
+
+ test_global->pool = pool;
+
+ if (pool == ODP_POOL_INVALID) {
+ printf("Error: Pool create.\n");
+ return -1;
+ }
+
+ odp_pktio_param_init(&pktio_param);
+ pktio_param.in_mode = ODP_PKTIN_MODE_SCHED;
+ pktio_param.out_mode = ODP_PKTOUT_MODE_DIRECT;
+
+ sched_sync = ODP_SCHED_SYNC_ATOMIC;
+
+ for (i = 0; i < num_pktio; i++)
+ test_global->pktio[i].pktio = ODP_PKTIO_INVALID;
+
+ /* Open and configure interfaces */
+ for (i = 0; i < num_pktio; i++) {
+ name = test_global->pktio[i].name;
+ pktio = odp_pktio_open(name, pool, &pktio_param);
+
+ if (pktio == ODP_PKTIO_INVALID) {
+ printf("Error (%s): Pktio open failed.\n", name);
+ return -1;
+ }
+
+ test_global->pktio[i].pktio = pktio;
+ test_global->pktio[i].pktio_index = odp_pktio_index(pktio);
+
+ ret = odp_pktio_mac_addr(pktio,
+ test_global->pktio[i].my_addr.addr,
+ ODPH_ETHADDR_LEN);
+ if (ret != ODPH_ETHADDR_LEN) {
+ printf("Error (%s): Bad MAC address len.\n", name);
+ return -1;
+ }
+
+ odp_pktio_print(pktio);
+
+ if (odp_pktio_capability(pktio, &pktio_capa)) {
+ printf("Error (%s): Pktio capa failed.\n", name);
+ return -1;
+ }
+
+ if (num_input > pktio_capa.max_input_queues) {
+ printf("Error (%s): Too many input queues: %u\n",
+ name, num_input);
+ return -1;
+ }
+
+ if (num_output > pktio_capa.max_output_queues) {
+ printf("Error (%s): Too many output queues: %u\n",
+ name, num_output);
+ return -1;
+ }
+
+ odp_pktio_config_init(&pktio_config);
+ pktio_config.parser.layer = ODP_PROTO_LAYER_NONE;
+
+ odp_pktio_config(pktio, &pktio_config);
+
+ odp_pktin_queue_param_init(&pktin_param);
+
+ pktin_param.queue_param.sched.prio = ODP_SCHED_PRIO_DEFAULT;
+ pktin_param.queue_param.sched.sync = sched_sync;
+ pktin_param.queue_param.sched.group = ODP_SCHED_GROUP_ALL;
+
+ if (num_input > 1) {
+ pktin_param.hash_enable = 1;
+ pktin_param.hash_proto.proto.ipv4_udp = 1;
+ }
+
+ pktin_param.num_queues = num_input;
+
+ if (odp_pktin_queue_config(pktio, &pktin_param)) {
+ printf("Error (%s): Pktin config failed.\n", name);
+ return -1;
+ }
+
+ if (odp_pktin_event_queue(pktio,
+ test_global->pktio[i].input_queue,
+ num_input) != (int)num_input) {
+ printf("Error (%s): Input queue query failed.\n", name);
+ return -1;
+ }
+
+ odp_pktout_queue_param_init(&pktout_param);
+ pktout_param.num_queues = num_output;
+ pktout_param.op_mode = ODP_PKTIO_OP_MT_UNSAFE;
+
+ if (odp_pktout_queue_config(pktio, &pktout_param)) {
+ printf("Error (%s): Pktout config failed.\n", name);
+ return -1;
+ }
+
+ if (odp_pktout_queue(pktio,
+ test_global->pktio[i].pktout,
+ num_output) != (int)num_output) {
+ printf("Error (%s): Output queue query failed.\n",
+ name);
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+static void link_pktios(test_global_t *test_global)
+{
+ int i, num_pktio, input, output;
+
+ num_pktio = test_global->opt.num_pktio;
+
+ printf("Forwarding table (pktio indexes)\n");
+
+ /* If single interface loopback, otherwise forward to the next
+ * interface. */
+ for (i = 0; i < num_pktio; i++) {
+ input = test_global->pktio[i].pktio_index;
+ output = (i + 1) % num_pktio;
+ test_global->pktio_map[input] = output;
+ printf(" input %i, output %i\n",
+ input,
+ test_global->pktio[output].pktio_index);
+ }
+
+ printf("\n");
+}
+
+static int start_pktios(test_global_t *test_global)
+{
+ int i;
+
+ for (i = 0; i < test_global->opt.num_pktio; i++) {
+ if (odp_pktio_start(test_global->pktio[i].pktio)) {
+ printf("Error (%s): Pktio start failed.\n",
+ test_global->pktio[i].name);
+
+ return -1;
+ }
+
+ test_global->pktio[i].started = 1;
+ }
+
+ return 0;
+}
+
+static int stop_pktios(test_global_t *test_global)
+{
+ odp_pktio_t pktio;
+ int i, ret = 0;
+
+ for (i = 0; i < test_global->opt.num_pktio; i++) {
+ pktio = test_global->pktio[i].pktio;
+
+ if (pktio == ODP_PKTIO_INVALID ||
+ test_global->pktio[i].started == 0)
+ continue;
+
+ if (odp_pktio_stop(pktio)) {
+ printf("Error (%s): Pktio stop failed.\n",
+ test_global->pktio[i].name);
+ ret = -1;
+ }
+ }
+
+ return ret;
+}
+
+static void empty_queues(void)
+{
+ odp_event_t ev;
+ uint64_t wait_time = odp_schedule_wait_time(ODP_TIME_SEC_IN_NS / 2);
+
+ /* Drop all events from all queues */
+ while (1) {
+ ev = odp_schedule(NULL, wait_time);
+
+ if (ev == ODP_EVENT_INVALID)
+ break;
+
+ odp_event_free(ev);
+ }
+}
+
+static int close_pktios(test_global_t *test_global)
+{
+ odp_pktio_t pktio;
+ odp_pool_t pool;
+ int i, ret = 0;
+
+ for (i = 0; i < test_global->opt.num_pktio; i++) {
+ pktio = test_global->pktio[i].pktio;
+
+ if (pktio == ODP_PKTIO_INVALID)
+ continue;
+
+ if (odp_pktio_close(pktio)) {
+ printf("Error (%s): Pktio close failed.\n",
+ test_global->pktio[i].name);
+ ret = -1;
+ }
+ }
+
+ pool = test_global->pool;
+
+ if (pool == ODP_POOL_INVALID)
+ return ret;
+
+ if (odp_pool_destroy(pool)) {
+ printf("Error: Pool destroy failed.\n");
+ ret = -1;
+ }
+
+ return ret;
+}
+
+static void start_workers(odph_odpthread_t thread[],
+ test_global_t *test_global)
+{
+ int i;
+ odp_cpumask_t cpumask;
+ odph_odpthread_params_t param;
+ int num = test_global->opt.num_worker;
+
+ memset(&param, 0, sizeof(odph_odpthread_params_t));
+ param.start = worker_thread;
+ param.thr_type = ODP_THREAD_WORKER;
+ param.instance = test_global->instance;
+
+ memset(thread, 0, num * sizeof(odph_odpthread_t));
+
+ for (i = 0; i < num; i++) {
+ odp_cpumask_zero(&cpumask);
+ odp_cpumask_set(&cpumask, test_global->worker_cpu[i]);
+ test_global->worker_arg[i].worker_id = i;
+ test_global->worker_arg[i].test_global_ptr = test_global;
+ param.arg = &test_global->worker_arg[i];
+
+ odph_odpthreads_create(&thread[i], &cpumask, &param);
+ }
+}
+
+static void wait_workers(odph_odpthread_t thread[], test_global_t *test_global)
+{
+ int i;
+
+ for (i = 0; i < test_global->opt.num_worker; ++i)
+ odph_odpthreads_join(&thread[i]);
+}
+
+int main(int argc, char *argv[])
+{
+ odp_instance_t instance;
+ odp_init_t init;
+ odp_shm_t shm;
+ odp_time_t t1, t2;
+ odph_odpthread_t thread[MAX_WORKERS];
+ int ret = 0;
+
+ signal(SIGINT, sig_handler);
+
+ /* List features not to be used (may optimize performance) */
+ odp_init_param_init(&init);
+ init.not_used.feat.cls = 1;
+ init.not_used.feat.crypto = 1;
+ init.not_used.feat.ipsec = 1;
+ init.not_used.feat.timer = 1;
+ init.not_used.feat.tm = 1;
+
+ /* Init ODP before calling anything else */
+ if (odp_init_global(&instance, &init, NULL)) {
+ printf("Error: Global init failed.\n");
+ return -1;
+ }
+
+ /* Init this thread */
+ if (odp_init_local(instance, ODP_THREAD_CONTROL)) {
+ printf("Error: Local init failed.\n");
+ return -1;
+ }
+
+ /* Reserve memory for args from shared mem */
+ shm = odp_shm_reserve("test_global", sizeof(test_global_t),
+ ODP_CACHE_LINE_SIZE, 0);
+
+ if (shm == ODP_SHM_INVALID) {
+ printf("Error: shm reserve failed.\n");
+ return -1;
+ }
+
+ test_global = odp_shm_addr(shm);
+ memset(test_global, 0, sizeof(test_global_t));
+
+ test_global->instance = instance;
+ test_global->pool = ODP_POOL_INVALID;
+
+ if (parse_options(argc, argv, test_global))
+ goto quit;
+
+ odp_sys_info_print();
+
+ if (config_setup(test_global))
+ goto quit;
+
+ print_config(test_global);
+
+ if (open_pktios(test_global))
+ goto quit;
+
+ link_pktios(test_global);
+
+ odp_barrier_init(&test_global->worker_start,
+ test_global->opt.num_worker + 1);
+
+ start_workers(thread, test_global);
+
+ /* Synchronize pktio configuration with workers. Worker are now ready
+ * to process packets. */
+ odp_barrier_wait(&test_global->worker_start);
+
+ if (start_pktios(test_global)) {
+ test_global->stop_workers = 1;
+ odp_mb_full();
+ }
+
+ t1 = odp_time_local();
+
+ wait_workers(thread, test_global);
+
+ t2 = odp_time_local();
+
+quit:
+ stop_pktios(test_global);
+ empty_queues();
+ close_pktios(test_global);
+
+ if (test_global->opt.collect_stat) {
+ print_stat(test_global, odp_time_diff_ns(t2, t1));
+
+ /* Encode return value for validation test usage. */
+ if (test_global->rx_pkt_sum > TEST_PASSED_LIMIT)
+ ret += 1;
+
+ if (test_global->tx_pkt_sum > TEST_PASSED_LIMIT)
+ ret += 2;
+ }
+
+ if (odp_shm_free(shm)) {
+ printf("Error: shm free failed.\n");
+ return -1;
+ }
+
+ if (odp_term_local()) {
+ printf("Error: term local failed.\n");
+ return -1;
+ }
+
+ if (odp_term_global(instance)) {
+ printf("Error: term global failed.\n");
+ return -1;
+ }
+
+ return ret;
+}
diff --git a/test/performance/odp_sched_pktio_run.sh b/test/performance/odp_sched_pktio_run.sh
new file mode 100755
index 000000000..566636126
--- /dev/null
+++ b/test/performance/odp_sched_pktio_run.sh
@@ -0,0 +1,109 @@
+#!/bin/sh
+#
+# Copyright (c) 2018, Linaro Limited
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+# directory where test binaries have been built
+TEST_DIR="${TEST_DIR:-$PWD}"
+# directory where test sources are, including scripts
+TEST_SRC_DIR=$(dirname $0)
+
+PATH=$TEST_DIR:$TEST_DIR/../../example/generator:$PATH
+
+# exit codes expected by automake for skipped tests
+TEST_SKIPPED=77
+
+VALIDATION_TESTDIR=platform/$ODP_PLATFORM/test/validation
+PLATFORM_VALIDATION=${TEST_SRC_DIR}/../../$VALIDATION_TESTDIR
+
+FLOOD_MODE=0
+
+# Use installed pktio env or for make check take it from platform directory
+if [ -f "./pktio_env" ]; then
+ . ./pktio_env
+elif [ "$ODP_PLATFORM" = "" ]; then
+ echo "$0: error: ODP_PLATFORM must be defined"
+ # not skipped as this should never happen via "make check"
+ exit 1
+elif [ -f ${PLATFORM_VALIDATION}/api/pktio/pktio_env ]; then
+ . ${PLATFORM_VALIDATION}/api/pktio/pktio_env
+else
+ echo "BUG: unable to find pktio_env!"
+ echo "pktio_env has to be in current directory or "
+ echo "in platform/\$ODP_PLATFORM/test."
+ echo "ODP_PLATFORM=\"$ODP_PLATFORM\""
+ exit 1
+fi
+
+run_sched_pktio()
+{
+ setup_pktio_env clean # install trap to call cleanup_pktio_env
+
+ if [ $? -ne 0 ]; then
+ echo "setup_pktio_env error $?"
+ exit $TEST_SKIPPED
+ fi
+
+ type odp_generator > /dev/null
+ if [ $? -ne 0 ]; then
+ echo "odp_generator not installed. Aborting."
+ cleanup_pktio_env
+ exit 1
+ fi
+
+ # 1 worker
+ export ODP_PLATFORM_PARAMS="-m 512 --file-prefix="sched" \
+--proc-type auto --no-pci --vdev net_pcap1,iface=$IF1 \
+--vdev net_pcap2,iface=$IF2"
+
+ odp_sched_pktio${EXEEXT} -i 0,1 -c 1 -s &
+
+ TEST_PID=$!
+
+ sleep 1
+
+ # Run generator with one worker
+ export ODP_PLATFORM_PARAMS="-m 512 --file-prefix="gen" \
+--proc-type auto --no-pci \
+--vdev net_pcap0,iface=$IF0"
+
+ (odp_generator${EXEEXT} --interval $FLOOD_MODE -I 0 \
+ --srcip 192.168.0.1 --dstip 192.168.0.2 \
+ -m u -w 1 2>&1 > /dev/null) \
+ 2>&1 > /dev/null &
+
+ GEN_PID=$!
+
+ # Run test for 5 sec
+ sleep 5
+
+ kill ${GEN_PID}
+ wait ${GEN_PID}
+
+ # Kill with SIGINT to output statistics
+ kill -2 ${TEST_PID}
+ wait ${TEST_PID}
+
+ ret=$?
+
+ if [ $ret -eq 3 ]; then
+ echo "PASS: received and transmitted over 5000 packets"
+ ret=0
+ else
+ echo "FAIL: less than thousand rx or tx packets $ret"
+ ret=1
+ fi
+
+ cleanup_pktio_env
+
+ exit $ret
+}
+
+case "$1" in
+ setup) setup_pktio_env ;;
+ cleanup) cleanup_pktio_env ;;
+ *) run_sched_pktio ;;
+esac
diff --git a/test/validation/api/crypto/odp_crypto_test_inp.c b/test/validation/api/crypto/odp_crypto_test_inp.c
index bfaf381d0..830702906 100644
--- a/test/validation/api/crypto/odp_crypto_test_inp.c
+++ b/test/validation/api/crypto/odp_crypto_test_inp.c
@@ -47,8 +47,12 @@ static const char *auth_alg_name(odp_auth_alg_t auth)
return "ODP_AUTH_ALG_SHA1_HMAC";
case ODP_AUTH_ALG_SHA256_HMAC:
return "ODP_AUTH_ALG_SHA256_HMAC";
+ case ODP_AUTH_ALG_SHA384_HMAC:
+ return "ODP_AUTH_ALG_SHA384_HMAC";
case ODP_AUTH_ALG_SHA512_HMAC:
return "ODP_AUTH_ALG_SHA512_HMAC";
+ case ODP_AUTH_ALG_AES_XCBC_MAC:
+ return "ODP_AUTH_ALG_AES_XCBC_MAC";
case ODP_AUTH_ALG_AES_GCM:
return "ODP_AUTH_ALG_AES_GCM";
case ODP_AUTH_ALG_AES_GMAC:
@@ -529,9 +533,15 @@ static void check_alg(odp_crypto_op_t op,
if (auth_alg == ODP_AUTH_ALG_SHA256_HMAC &&
!(capa.auths.bit.sha256_hmac))
rc = -1;
+ if (auth_alg == ODP_AUTH_ALG_SHA384_HMAC &&
+ !(capa.auths.bit.sha384_hmac))
+ rc = -1;
if (auth_alg == ODP_AUTH_ALG_SHA512_HMAC &&
!(capa.auths.bit.sha512_hmac))
rc = -1;
+ if (auth_alg == ODP_AUTH_ALG_AES_XCBC_MAC &&
+ !(capa.auths.bit.aes_xcbc_mac))
+ rc = -1;
CU_ASSERT(!rc);
CU_ASSERT((~capa.auths.all_bits & capa.hw_auths.all_bits) == 0);
@@ -713,10 +723,18 @@ static int check_alg_support(odp_cipher_alg_t cipher, odp_auth_alg_t auth)
if (!capability.auths.bit.sha256_hmac)
return ODP_TEST_INACTIVE;
break;
+ case ODP_AUTH_ALG_SHA384_HMAC:
+ if (!capability.auths.bit.sha384_hmac)
+ return ODP_TEST_INACTIVE;
+ break;
case ODP_AUTH_ALG_SHA512_HMAC:
if (!capability.auths.bit.sha512_hmac)
return ODP_TEST_INACTIVE;
break;
+ case ODP_AUTH_ALG_AES_XCBC_MAC:
+ if (!capability.auths.bit.aes_xcbc_mac)
+ return ODP_TEST_INACTIVE;
+ break;
case ODP_AUTH_ALG_AES_GCM:
if (!capability.auths.bit.aes_gcm)
return ODP_TEST_INACTIVE;
@@ -1207,6 +1225,38 @@ static void crypto_test_check_alg_hmac_sha256(void)
false);
}
+static int check_alg_hmac_sha384(void)
+{
+ return check_alg_support(ODP_CIPHER_ALG_NULL, ODP_AUTH_ALG_SHA384_HMAC);
+}
+
+/* This test verifies the correctness of HMAC_SHA384 digest operation.
+ * The output check length is truncated to 24 bytes (192 bits) as
+ * returned by the crypto operation API call.
+ * Note that hash digest is a one-way operation.
+ * In addition the test verifies if the implementation can use the
+ * packet buffer as completion event buffer.
+ * */
+static void crypto_test_gen_alg_hmac_sha384(void)
+{
+ check_alg(ODP_CRYPTO_OP_ENCODE,
+ ODP_CIPHER_ALG_NULL,
+ ODP_AUTH_ALG_SHA384_HMAC,
+ hmac_sha384_reference,
+ ARRAY_SIZE(hmac_sha384_reference),
+ false);
+}
+
+static void crypto_test_check_alg_hmac_sha384(void)
+{
+ check_alg(ODP_CRYPTO_OP_DECODE,
+ ODP_CIPHER_ALG_NULL,
+ ODP_AUTH_ALG_SHA384_HMAC,
+ hmac_sha384_reference,
+ ARRAY_SIZE(hmac_sha384_reference),
+ false);
+}
+
static int check_alg_hmac_sha512(void)
{
return check_alg_support(ODP_CIPHER_ALG_NULL, ODP_AUTH_ALG_SHA512_HMAC);
@@ -1239,6 +1289,39 @@ static void crypto_test_check_alg_hmac_sha512(void)
false);
}
+static int check_alg_aes_xcbc(void)
+{
+ return check_alg_support(ODP_CIPHER_ALG_NULL,
+ ODP_AUTH_ALG_AES_XCBC_MAC);
+}
+
+/* This test verifies the correctness of AES_XCBC_MAC digest operation.
+ * The output check length is truncated to 16 bytes (128 bits) as
+ * returned by the crypto operation API call.
+ * Note that hash digest is a one-way operation.
+ * In addition the test verifies if the implementation can use the
+ * packet buffer as completion event buffer.
+ * */
+static void crypto_test_gen_alg_aes_xcbc(void)
+{
+ check_alg(ODP_CRYPTO_OP_ENCODE,
+ ODP_CIPHER_ALG_NULL,
+ ODP_AUTH_ALG_AES_XCBC_MAC,
+ aes_xcbc_reference,
+ ARRAY_SIZE(aes_xcbc_reference),
+ false);
+}
+
+static void crypto_test_check_alg_aes_xcbc(void)
+{
+ check_alg(ODP_CRYPTO_OP_DECODE,
+ ODP_CIPHER_ALG_NULL,
+ ODP_AUTH_ALG_AES_XCBC_MAC,
+ aes_xcbc_reference,
+ ARRAY_SIZE(aes_xcbc_reference),
+ false);
+}
+
static int check_alg_aes_gmac(void)
{
return check_alg_support(ODP_CIPHER_ALG_NULL, ODP_AUTH_ALG_AES_GMAC);
@@ -1426,10 +1509,18 @@ odp_testinfo_t crypto_suite[] = {
check_alg_hmac_sha256),
ODP_TEST_INFO_CONDITIONAL(crypto_test_check_alg_hmac_sha256,
check_alg_hmac_sha256),
+ ODP_TEST_INFO_CONDITIONAL(crypto_test_gen_alg_hmac_sha384,
+ check_alg_hmac_sha384),
+ ODP_TEST_INFO_CONDITIONAL(crypto_test_check_alg_hmac_sha384,
+ check_alg_hmac_sha384),
ODP_TEST_INFO_CONDITIONAL(crypto_test_gen_alg_hmac_sha512,
check_alg_hmac_sha512),
ODP_TEST_INFO_CONDITIONAL(crypto_test_check_alg_hmac_sha512,
check_alg_hmac_sha512),
+ ODP_TEST_INFO_CONDITIONAL(crypto_test_gen_alg_aes_xcbc,
+ check_alg_aes_xcbc),
+ ODP_TEST_INFO_CONDITIONAL(crypto_test_check_alg_aes_xcbc,
+ check_alg_aes_xcbc),
ODP_TEST_INFO_CONDITIONAL(crypto_test_gen_alg_aes_gmac,
check_alg_aes_gmac),
ODP_TEST_INFO_CONDITIONAL(crypto_test_gen_alg_aes_gmac_ovr_iv,
diff --git a/test/validation/api/crypto/test_vectors.h b/test/validation/api/crypto/test_vectors.h
index 23ed95251..9adb43324 100644
--- a/test/validation/api/crypto/test_vectors.h
+++ b/test/validation/api/crypto/test_vectors.h
@@ -1188,6 +1188,134 @@ static crypto_test_reference_t hmac_sha256_reference[] = {
}
};
+static crypto_test_reference_t hmac_sha384_reference[] = {
+ {
+ .auth_key_length = HMAC_SHA384_KEY_LEN,
+ .auth_key = { 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b,
+ 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b,
+ 0x0b, 0x0b, 0x0b, 0x0b },
+ .length = 8,
+ /* "Hi There" */
+ .plaintext = { 0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65},
+ .ciphertext = { 0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65},
+ .digest_length = HMAC_SHA384_192_CHECK_LEN,
+ .digest = { 0xaf, 0xd0, 0x39, 0x44, 0xd8, 0x48, 0x95, 0x62,
+ 0x6b, 0x08, 0x25, 0xf4, 0xab, 0x46, 0x90, 0x7f,
+ 0x15, 0xf9, 0xda, 0xdb, 0xe4, 0x10, 0x1e, 0xc6 }
+ },
+ {
+ .auth_key_length = HMAC_SHA384_KEY_LEN,
+ /* "Jefe" */
+ .auth_key = { 0x4a, 0x65, 0x66, 0x65 },
+ .length = 28,
+ /* what do ya want for nothing?*/
+ .plaintext = { 0x77, 0x68, 0x61, 0x74, 0x20, 0x64, 0x6f, 0x20,
+ 0x79, 0x61, 0x20, 0x77, 0x61, 0x6e, 0x74, 0x20,
+ 0x66, 0x6f, 0x72, 0x20, 0x6e, 0x6f, 0x74, 0x68,
+ 0x69, 0x6e, 0x67, 0x3f },
+ .ciphertext = { 0x77, 0x68, 0x61, 0x74, 0x20, 0x64, 0x6f, 0x20,
+ 0x79, 0x61, 0x20, 0x77, 0x61, 0x6e, 0x74, 0x20,
+ 0x66, 0x6f, 0x72, 0x20, 0x6e, 0x6f, 0x74, 0x68,
+ 0x69, 0x6e, 0x67, 0x3f },
+ .digest_length = HMAC_SHA384_192_CHECK_LEN,
+ .digest = { 0xaf, 0x45, 0xd2, 0xe3, 0x76, 0x48, 0x40, 0x31,
+ 0x61, 0x7f, 0x78, 0xd2, 0xb5, 0x8a, 0x6b, 0x1b,
+ 0x9c, 0x7e, 0xf4, 0x64, 0xf5, 0xa0, 0x1b, 0x47 }
+ },
+ {
+ .auth_key_length = HMAC_SHA384_KEY_LEN,
+ .auth_key = { 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa,
+ 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa,
+ 0xaa, 0xaa, 0xaa, 0xaa },
+ .length = 50,
+ .plaintext = { 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd },
+ .ciphertext = { 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd },
+ .digest_length = HMAC_SHA384_192_CHECK_LEN,
+ .digest = { 0x88, 0x06, 0x26, 0x08, 0xd3, 0xe6, 0xad, 0x8a,
+ 0x0a, 0xa2, 0xac, 0xe0, 0x14, 0xc8, 0xa8, 0x6f,
+ 0x0a, 0xa6, 0x35, 0xd9, 0x47, 0xac, 0x9f, 0xeb }
+ },
+ {
+ .auth_key_length = HMAC_SHA384_KEY_LEN,
+ .auth_key = { 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b,
+ 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b,
+ 0x0b, 0x0b, 0x0b, 0x0b },
+ .length = 8,
+ /* "Hi There" */
+ .plaintext = { 0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65},
+ .ciphertext = { 0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65},
+ .digest_length = HMAC_SHA384_CHECK_LEN,
+ .digest = { 0xaf, 0xd0, 0x39, 0x44, 0xd8, 0x48, 0x95, 0x62,
+ 0x6b, 0x08, 0x25, 0xf4, 0xab, 0x46, 0x90, 0x7f,
+ 0x15, 0xf9, 0xda, 0xdb, 0xe4, 0x10, 0x1e, 0xc6,
+ 0x82, 0xaa, 0x03, 0x4c, 0x7c, 0xeb, 0xc5, 0x9c,
+ 0xfa, 0xea, 0x9e, 0xa9, 0x07, 0x6e, 0xde, 0x7f,
+ 0x4a, 0xf1, 0x52, 0xe8, 0xb2, 0xfa, 0x9c, 0xb6 }
+ },
+ {
+ .auth_key_length = HMAC_SHA384_KEY_LEN,
+ /* "Jefe" */
+ .auth_key = { 0x4a, 0x65, 0x66, 0x65 },
+ .length = 28,
+ /* what do ya want for nothing?*/
+ .plaintext = { 0x77, 0x68, 0x61, 0x74, 0x20, 0x64, 0x6f, 0x20,
+ 0x79, 0x61, 0x20, 0x77, 0x61, 0x6e, 0x74, 0x20,
+ 0x66, 0x6f, 0x72, 0x20, 0x6e, 0x6f, 0x74, 0x68,
+ 0x69, 0x6e, 0x67, 0x3f },
+ .ciphertext = { 0x77, 0x68, 0x61, 0x74, 0x20, 0x64, 0x6f, 0x20,
+ 0x79, 0x61, 0x20, 0x77, 0x61, 0x6e, 0x74, 0x20,
+ 0x66, 0x6f, 0x72, 0x20, 0x6e, 0x6f, 0x74, 0x68,
+ 0x69, 0x6e, 0x67, 0x3f },
+ .digest_length = HMAC_SHA384_CHECK_LEN,
+ .digest = { 0xaf, 0x45, 0xd2, 0xe3, 0x76, 0x48, 0x40, 0x31,
+ 0x61, 0x7f, 0x78, 0xd2, 0xb5, 0x8a, 0x6b, 0x1b,
+ 0x9c, 0x7e, 0xf4, 0x64, 0xf5, 0xa0, 0x1b, 0x47,
+ 0xe4, 0x2e, 0xc3, 0x73, 0x63, 0x22, 0x44, 0x5e,
+ 0x8e, 0x22, 0x40, 0xca, 0x5e, 0x69, 0xe2, 0xc7,
+ 0x8b, 0x32, 0x39, 0xec, 0xfa, 0xb2, 0x16, 0x49 }
+ },
+ {
+ .auth_key_length = HMAC_SHA384_KEY_LEN,
+ .auth_key = { 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa,
+ 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa,
+ 0xaa, 0xaa, 0xaa, 0xaa },
+ .length = 50,
+ .plaintext = { 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd },
+ .ciphertext = { 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd,
+ 0xdd, 0xdd },
+ .digest_length = HMAC_SHA384_CHECK_LEN,
+ .digest = {0x88, 0x06, 0x26, 0x08, 0xd3, 0xe6, 0xad, 0x8a,
+ 0x0a, 0xa2, 0xac, 0xe0, 0x14, 0xc8, 0xa8, 0x6f,
+ 0x0a, 0xa6, 0x35, 0xd9, 0x47, 0xac, 0x9f, 0xeb,
+ 0xe8, 0x3e, 0xf4, 0xe5, 0x59, 0x66, 0x14, 0x4b,
+ 0x2a, 0x5a, 0xb3, 0x9d, 0xc1, 0x38, 0x14, 0xb9,
+ 0x4e, 0x3a, 0xb6, 0xe1, 0x01, 0xa3, 0x4f, 0x27 }
+ }
+};
+
static crypto_test_reference_t hmac_sha512_reference[] = {
{
.auth_key_length = HMAC_SHA512_KEY_LEN,
@@ -1325,4 +1453,85 @@ static crypto_test_reference_t hmac_sha512_reference[] = {
}
};
+static crypto_test_reference_t aes_xcbc_reference[] = {
+ {
+ .auth_key_length = AES_XCBC_MAC_KEY_LEN,
+ .auth_key = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f },
+ .length = 3,
+ .plaintext = { 0x00, 0x01, 0x02 },
+ .ciphertext = { 0x00, 0x01, 0x02 },
+ .digest_length = AES_XCBC_MAC_96_CHECK_LEN,
+ .digest = { 0x5b, 0x37, 0x65, 0x80, 0xae, 0x2f, 0x19, 0xaf,
+ 0xe7, 0x21, 0x9c, 0xee }
+ },
+ {
+ .auth_key_length = AES_XCBC_MAC_KEY_LEN,
+ .auth_key = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f },
+ .length = 16,
+ .plaintext = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f },
+ .ciphertext = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f },
+ .digest_length = AES_XCBC_MAC_96_CHECK_LEN,
+ .digest = { 0xd2, 0xa2, 0x46, 0xfa, 0x34, 0x9b, 0x68, 0xa7,
+ 0x99, 0x98, 0xa4, 0x39 }
+ },
+ {
+ .auth_key_length = AES_XCBC_MAC_KEY_LEN,
+ .auth_key = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f },
+ .length = 20,
+ .plaintext = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+ 0x10, 0x11, 0x12, 0x13 },
+ .ciphertext = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+ 0x10, 0x11, 0x12, 0x13 },
+ .digest_length = AES_XCBC_MAC_96_CHECK_LEN,
+ .digest = { 0x47, 0xf5, 0x1b, 0x45, 0x64, 0x96, 0x62, 0x15,
+ 0xb8, 0x98, 0x5c, 0x63 }
+ },
+ {
+ .auth_key_length = AES_XCBC_MAC_KEY_LEN,
+ .auth_key = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f },
+ .length = 3,
+ .plaintext = { 0x00, 0x01, 0x02 },
+ .ciphertext = { 0x00, 0x01, 0x02 },
+ .digest_length = AES_XCBC_MAC_CHECK_LEN,
+ .digest = { 0x5b, 0x37, 0x65, 0x80, 0xae, 0x2f, 0x19, 0xaf,
+ 0xe7, 0x21, 0x9c, 0xee, 0xf1, 0x72, 0x75, 0x6f }
+ },
+ {
+ .auth_key_length = AES_XCBC_MAC_KEY_LEN,
+ .auth_key = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f },
+ .length = 16,
+ .plaintext = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f },
+ .ciphertext = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f },
+ .digest_length = AES_XCBC_MAC_CHECK_LEN,
+ .digest = { 0xd2, 0xa2, 0x46, 0xfa, 0x34, 0x9b, 0x68, 0xa7,
+ 0x99, 0x98, 0xa4, 0x39, 0x4f, 0xf7, 0xa2, 0x63 }
+ },
+ {
+ .auth_key_length = AES_XCBC_MAC_KEY_LEN,
+ .auth_key = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f },
+ .length = 20,
+ .plaintext = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+ 0x10, 0x11, 0x12, 0x13 },
+ .ciphertext = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+ 0x10, 0x11, 0x12, 0x13 },
+ .digest_length = AES_XCBC_MAC_CHECK_LEN,
+ .digest = { 0x47, 0xf5, 0x1b, 0x45, 0x64, 0x96, 0x62, 0x15,
+ 0xb8, 0x98, 0x5c, 0x63, 0x05, 0x5e, 0xd3, 0x08 }
+ }
+};
+
#endif
diff --git a/test/validation/api/crypto/test_vectors_len.h b/test/validation/api/crypto/test_vectors_len.h
index 860840cfe..95d202b62 100644
--- a/test/validation/api/crypto/test_vectors_len.h
+++ b/test/validation/api/crypto/test_vectors_len.h
@@ -50,6 +50,11 @@
#define HMAC_SHA1_96_CHECK_LEN 12
#define HMAC_SHA1_CHECK_LEN 20
+/* HMAC-SHA384 */
+#define HMAC_SHA384_KEY_LEN 48
+#define HMAC_SHA384_192_CHECK_LEN 24
+#define HMAC_SHA384_CHECK_LEN 48
+
/* HMAC-SHA512 */
#define HMAC_SHA512_KEY_LEN 64
#define HMAC_SHA512_256_CHECK_LEN 32
@@ -60,4 +65,9 @@
#define CHACHA20_POLY1305_IV_LEN 12
#define CHACHA20_POLY1305_CHECK_LEN 16
+/* AES-XCBC-MAC */
+#define AES_XCBC_MAC_KEY_LEN 16
+#define AES_XCBC_MAC_96_CHECK_LEN 12
+#define AES_XCBC_MAC_CHECK_LEN 16
+
#endif
diff --git a/test/validation/api/ipsec/ipsec.c b/test/validation/api/ipsec/ipsec.c
index 3e2e743d2..31a6f9b53 100644
--- a/test/validation/api/ipsec/ipsec.c
+++ b/test/validation/api/ipsec/ipsec.c
@@ -204,10 +204,18 @@ int ipsec_check(odp_bool_t ah,
if (!capa.auths.bit.sha256_hmac)
return ODP_TEST_INACTIVE;
break;
+ case ODP_AUTH_ALG_SHA384_HMAC:
+ if (!capa.auths.bit.sha384_hmac)
+ return ODP_TEST_INACTIVE;
+ break;
case ODP_AUTH_ALG_SHA512_HMAC:
if (!capa.auths.bit.sha512_hmac)
return ODP_TEST_INACTIVE;
break;
+ case ODP_AUTH_ALG_AES_XCBC_MAC:
+ if (!capa.auths.bit.aes_xcbc_mac)
+ return ODP_TEST_INACTIVE;
+ break;
case ODP_AUTH_ALG_AES_GCM:
if (!capa.auths.bit.aes_gcm)
return ODP_TEST_INACTIVE;
@@ -930,8 +938,10 @@ int ipsec_config(odp_instance_t ODP_UNUSED inst)
odp_ipsec_config_init(&ipsec_config);
ipsec_config.inbound_mode = suite_context.inbound_op_mode;
ipsec_config.outbound_mode = suite_context.outbound_op_mode;
+ ipsec_config.outbound.all_chksum = ~0;
ipsec_config.inbound.default_queue = suite_context.queue;
ipsec_config.inbound.parse_level = ODP_PROTO_LAYER_ALL;
+ ipsec_config.inbound.chksums.all_chksum = ~0;
if (ODP_IPSEC_OK != odp_ipsec_config(&ipsec_config))
return -1;
diff --git a/test/validation/api/ipsec/ipsec_test_out.c b/test/validation/api/ipsec/ipsec_test_out.c
index 5089dfa79..59c631b58 100644
--- a/test/validation/api/ipsec/ipsec_test_out.c
+++ b/test/validation/api/ipsec/ipsec_test_out.c
@@ -1155,6 +1155,36 @@ static void test_out_dummy_esp_null_sha256_tun_ipv6(void)
ipsec_sa_destroy(sa);
}
+static void test_out_ipv4_udp_esp_null_sha256(void)
+{
+ odp_ipsec_sa_param_t param;
+ odp_ipsec_sa_t sa;
+
+ ipsec_sa_param_fill(&param,
+ false, false, 123, NULL,
+ ODP_CIPHER_ALG_NULL, NULL,
+ ODP_AUTH_ALG_SHA256_HMAC, &key_5a_256,
+ NULL);
+
+ sa = odp_ipsec_sa_create(&param);
+
+ CU_ASSERT_NOT_EQUAL_FATAL(ODP_IPSEC_SA_INVALID, sa);
+
+ ipsec_test_part test = {
+ .pkt_in = &pkt_ipv4_udp,
+ .out_pkt = 1,
+ .out = {
+ { .status.warn.all = 0,
+ .status.error.all = 0,
+ .pkt_out = &pkt_ipv4_udp_esp_null_sha256 },
+ },
+ };
+
+ ipsec_check_out_one(&test, sa);
+
+ ipsec_sa_destroy(sa);
+}
+
static void ipsec_test_capability(void)
{
odp_ipsec_capability_t capa;
@@ -1218,5 +1248,7 @@ odp_testinfo_t ipsec_out_suite[] = {
ipsec_check_esp_null_sha256),
ODP_TEST_INFO_CONDITIONAL(test_out_dummy_esp_null_sha256_tun_ipv6,
ipsec_check_esp_null_sha256),
+ ODP_TEST_INFO_CONDITIONAL(test_out_ipv4_udp_esp_null_sha256,
+ ipsec_check_esp_null_sha256),
ODP_TEST_INFO_NULL,
};
diff --git a/test/validation/api/ipsec/test_vectors.h b/test/validation/api/ipsec/test_vectors.h
index 5b357a160..289b8008c 100644
--- a/test/validation/api/ipsec/test_vectors.h
+++ b/test/validation/api/ipsec/test_vectors.h
@@ -269,7 +269,7 @@ static const ODP_UNUSED ipsec_test_packet pkt_ipv4_icmp_0_ah_sha256_1_bad1 = {
/* IP */
0x45, 0x00, 0x00, 0x9a, 0x00, 0x00, 0x00, 0x00,
- 0x40, 0x33, 0xab, 0xd9, 0xc0, 0xa8, 0x6f, 0x02,
+ 0x40, 0x33, 0xab, 0xdb, 0xc0, 0xa8, 0x6f, 0x02,
0xc0, 0xa8, 0xde, 0x02,
/* AH */
@@ -669,7 +669,7 @@ static const ODP_UNUSED ipsec_test_packet pkt_ipv4_icmp_0_esp_aes_cbc_null_1 = {
/* IP */
0x45, 0x00, 0x00, 0x9c, 0x00, 0x00, 0x00, 0x00,
- 0x40, 0x32, 0xab, 0xca, 0xc0, 0xa8, 0x6f, 0x02,
+ 0x40, 0x32, 0xab, 0xda, 0xc0, 0xa8, 0x6f, 0x02,
0xc0, 0xa8, 0xde, 0x02,
/* ESP */
@@ -1499,7 +1499,7 @@ static const ipsec_test_packet pkt_mcgrew_gcm_test_2_esp = {
/* IP - not a part of RFC, added for simplicity */
0x45, 0x00, 0x00, 0x74, 0x69, 0x8f, 0x00, 0x00,
- 0x80, 0x32, 0x4d, 0x76, 0xc0, 0xa8, 0x01, 0x02,
+ 0x80, 0x32, 0x4d, 0x75, 0xc0, 0xa8, 0x01, 0x02,
0xc0, 0xa8, 0x01, 0x01,
/* ESP */
@@ -1557,7 +1557,7 @@ static const ipsec_test_packet pkt_mcgrew_gcm_test_3_esp = {
/* IP - not a part of RFC, added for simplicity */
0x45, 0x00, 0x00, 0x68, 0x69, 0x8f, 0x00, 0x00,
- 0x80, 0x32, 0x4d, 0x82, 0xc0, 0xa8, 0x01, 0x02,
+ 0x80, 0x32, 0x4d, 0x81, 0xc0, 0xa8, 0x01, 0x02,
0xc0, 0xa8, 0x01, 0x01,
/* ESP */
@@ -1615,7 +1615,7 @@ static const ipsec_test_packet pkt_mcgrew_gcm_test_4_esp = {
/* IP - not a part of RFC, added for simplicity */
0x45, 0x00, 0x00, 0x74, 0x69, 0x8f, 0x00, 0x00,
- 0x80, 0x32, 0x4d, 0x76, 0xc0, 0xa8, 0x01, 0x02,
+ 0x80, 0x32, 0x4d, 0x75, 0xc0, 0xa8, 0x01, 0x02,
0xc0, 0xa8, 0x01, 0x01,
/* ESP */
@@ -1730,7 +1730,7 @@ static const ipsec_test_packet pkt_mcgrew_gcm_test_15_esp = {
/* IP - not a part of RFC, added for simplicity */
0x45, 0x00, 0x00, 0x68, 0x69, 0x8f, 0x00, 0x00,
- 0x80, 0x32, 0x4d, 0xb2, 0xc0, 0xa8, 0x01, 0x02,
+ 0x80, 0x32, 0x4d, 0x81, 0xc0, 0xa8, 0x01, 0x02,
0xc0, 0xa8, 0x01, 0x01,
/* ESP */
@@ -1883,4 +1883,63 @@ static const ipsec_test_packet pkt_test_nodata = {
0x0a, 0x0b, 0x0c, 0x0d,
},
};
+
+static const ipsec_test_packet pkt_ipv4_udp = {
+ .len = 76,
+ .l2_offset = 0,
+ .l3_offset = 14,
+ .l4_offset = 34,
+ .data = {
+ /* ETH - not a part of RFC, added for simplicity */
+ 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1,
+ 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x08, 0x00,
+
+ /* IP */
+ 0x45, 0x00, 0x00, 0x3e, 0x69, 0x8f, 0x00, 0x00,
+ 0x80, 0x11, 0x00, 0x00, 0xc0, 0xa8, 0x01, 0x02,
+ 0xc0, 0xa8, 0x01, 0x01,
+
+ /* UDP */
+ 0x0a, 0x98, 0x00, 0x35, 0x00, 0x2a, 0x00, 0x00,
+ 0xb2, 0xd0, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x03, 0x73, 0x69, 0x70,
+ 0x09, 0x63, 0x79, 0x62, 0x65, 0x72, 0x63, 0x69,
+ 0x74, 0x79, 0x02, 0x64, 0x6b, 0x00, 0x00, 0x01,
+ 0x00, 0x01,
+ },
+};
+
+static const ipsec_test_packet pkt_ipv4_udp_esp_null_sha256 = {
+ .len = 102,
+ .l2_offset = 0,
+ .l3_offset = 14,
+ .l4_offset = 34,
+ .data = {
+ /* ETH - not a part of RFC, added for simplicity */
+ 0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1,
+ 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0xf2, 0x08, 0x00,
+
+ /* IP */
+ 0x45, 0x00, 0x00, 0x58, 0x69, 0x8f, 0x00, 0x00,
+ 0x80, 0x32, 0x4d, 0x91, 0xc0, 0xa8, 0x01, 0x02,
+ 0xc0, 0xa8, 0x01, 0x01,
+
+ /* ESP */
+ 0x00, 0x00, 0x00, 0x7b, 0x00, 0x00, 0x00, 0x01,
+
+ /* UDP */
+ 0x0a, 0x98, 0x00, 0x35, 0x00, 0x2a, 0x23, 0x43,
+ 0xb2, 0xd0, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x03, 0x73, 0x69, 0x70,
+ 0x09, 0x63, 0x79, 0x62, 0x65, 0x72, 0x63, 0x69,
+ 0x74, 0x79, 0x02, 0x64, 0x6b, 0x00, 0x00, 0x01,
+ 0x00, 0x01,
+
+ /* ESP TRL */
+ 0x00, 0x11, 0x2d, 0x4a, 0x06, 0x9f, 0x97, 0xcf,
+ 0xa3, 0x05, 0xea, 0x90, 0x7a, 0xf6, 0x6b, 0x0a,
+ 0x3f, 0xc7,
+ },
+};
+
#endif
diff --git a/test/validation/api/pktio/pktio.c b/test/validation/api/pktio/pktio.c
index e82a96839..09367d4ea 100644
--- a/test/validation/api/pktio/pktio.c
+++ b/test/validation/api/pktio/pktio.c
@@ -296,7 +296,7 @@ static uint32_t pktio_init_packet(odp_packet_t pkt)
return pktio_pkt_set_seq(pkt);
}
-static int pktio_fixup_checksums(odp_packet_t pkt)
+static int pktio_zero_checksums(odp_packet_t pkt)
{
odph_ipv4hdr_t *ip;
odph_udphdr_t *udp;
@@ -312,8 +312,20 @@ static int pktio_fixup_checksums(odp_packet_t pkt)
udp = (odph_udphdr_t *)odp_packet_l4_ptr(pkt, &len);
ip->chksum = 0;
- odph_ipv4_csum_update(pkt);
udp->chksum = 0;
+
+ return 0;
+}
+
+static int pktio_fixup_checksums(odp_packet_t pkt)
+{
+ odph_udphdr_t *udp;
+
+ pktio_zero_checksums(pkt);
+
+ udp = (odph_udphdr_t *)odp_packet_l4_ptr(pkt, NULL);
+
+ odph_ipv4_csum_update(pkt);
udp->chksum = odph_ipv4_udp_chksum(pkt);
return 0;
@@ -407,10 +419,14 @@ static int flush_input_queue(odp_pktio_t pktio, odp_pktin_mode_t imode)
return 0;
}
-static int create_packets(odp_packet_t pkt_tbl[], uint32_t pkt_seq[], int num,
- odp_pktio_t pktio_src, odp_pktio_t pktio_dst)
+static int create_packets_cs(odp_packet_t pkt_tbl[],
+ uint32_t pkt_seq[],
+ int num,
+ odp_pktio_t pktio_src,
+ odp_pktio_t pktio_dst,
+ odp_bool_t fix_cs)
{
- int i;
+ int i, ret;
for (i = 0; i < num; i++) {
pkt_tbl[i] = odp_packet_alloc(default_pkt_pool, packet_len);
@@ -425,7 +441,11 @@ static int create_packets(odp_packet_t pkt_tbl[], uint32_t pkt_seq[], int num,
pktio_pkt_set_macs(pkt_tbl[i], pktio_src, pktio_dst);
- if (pktio_fixup_checksums(pkt_tbl[i]) != 0) {
+ if (fix_cs)
+ ret = pktio_fixup_checksums(pkt_tbl[i]);
+ else
+ ret = pktio_zero_checksums(pkt_tbl[i]);
+ if (ret != 0) {
odp_packet_free(pkt_tbl[i]);
break;
}
@@ -434,6 +454,13 @@ static int create_packets(odp_packet_t pkt_tbl[], uint32_t pkt_seq[], int num,
return i;
}
+static int create_packets(odp_packet_t pkt_tbl[], uint32_t pkt_seq[], int num,
+ odp_pktio_t pktio_src, odp_pktio_t pktio_dst)
+{
+ return create_packets_cs(pkt_tbl, pkt_seq, num, pktio_src, pktio_dst,
+ true);
+}
+
static int get_packets(pktio_info_t *pktio_rx, odp_packet_t pkt_tbl[],
int num, txrx_mode_e mode)
{
@@ -1894,6 +1921,416 @@ static void pktio_test_pktin_ts(void)
}
}
+static void pktio_test_chksum(void (*config_fn)(odp_pktio_t, odp_pktio_t),
+ void (*prep_fn)(odp_packet_t pkt),
+ void (*test_fn)(odp_packet_t pkt))
+{
+ odp_pktio_t pktio_tx, pktio_rx;
+ odp_pktio_t pktio[MAX_NUM_IFACES] = {ODP_PKTIO_INVALID};
+ pktio_info_t pktio_rx_info;
+ odp_pktout_queue_t pktout_queue;
+ odp_packet_t pkt_tbl[TX_BATCH_LEN];
+ uint32_t pkt_seq[TX_BATCH_LEN];
+ int ret;
+ int i, num_rx;
+
+ CU_ASSERT_FATAL(num_ifaces >= 1);
+
+ /* Open and configure interfaces */
+ for (i = 0; i < num_ifaces; ++i) {
+ pktio[i] = create_pktio(i, ODP_PKTIN_MODE_DIRECT,
+ ODP_PKTOUT_MODE_DIRECT);
+ CU_ASSERT_FATAL(pktio[i] != ODP_PKTIO_INVALID);
+ }
+
+ pktio_tx = pktio[0];
+ pktio_rx = (num_ifaces > 1) ? pktio[1] : pktio_tx;
+ pktio_rx_info.id = pktio_rx;
+ pktio_rx_info.inq = ODP_QUEUE_INVALID;
+ pktio_rx_info.in_mode = ODP_PKTIN_MODE_DIRECT;
+
+ config_fn(pktio_tx, pktio_rx);
+
+ for (i = 0; i < num_ifaces; ++i) {
+ CU_ASSERT_FATAL(odp_pktio_start(pktio[i]) == 0);
+ _pktio_wait_linkup(pktio[i]);
+ }
+
+ ret = create_packets_cs(pkt_tbl, pkt_seq, TX_BATCH_LEN, pktio_tx,
+ pktio_rx, false);
+ CU_ASSERT_FATAL(ret == TX_BATCH_LEN);
+
+ ret = odp_pktout_queue(pktio_tx, &pktout_queue, 1);
+ CU_ASSERT_FATAL(ret > 0);
+
+ for (i = 0; i < TX_BATCH_LEN; i++)
+ if (prep_fn)
+ prep_fn(pkt_tbl[i]);
+
+ send_packets(pktout_queue, pkt_tbl, TX_BATCH_LEN);
+ num_rx = wait_for_packets(&pktio_rx_info, pkt_tbl, pkt_seq,
+ TX_BATCH_LEN, TXRX_MODE_MULTI,
+ ODP_TIME_SEC_IN_NS);
+ CU_ASSERT(num_rx == TX_BATCH_LEN);
+ for (i = 0; i < num_rx; i++) {
+ test_fn(pkt_tbl[i]);
+ odp_packet_free(pkt_tbl[i]);
+ }
+
+ for (i = 0; i < num_ifaces; i++) {
+ CU_ASSERT_FATAL(odp_pktio_stop(pktio[i]) == 0);
+ CU_ASSERT_FATAL(odp_pktio_close(pktio[i]) == 0);
+ }
+}
+
+static int pktio_check_chksum_in_ipv4(void)
+{
+ odp_pktio_t pktio;
+ odp_pktio_capability_t capa;
+ odp_pktio_param_t pktio_param;
+ int idx = (num_ifaces == 1) ? 0 : 1;
+ int ret;
+
+ odp_pktio_param_init(&pktio_param);
+ pktio_param.in_mode = ODP_PKTIN_MODE_DIRECT;
+
+ pktio = odp_pktio_open(iface_name[idx], pool[idx], &pktio_param);
+ if (pktio == ODP_PKTIO_INVALID)
+ return ODP_TEST_INACTIVE;
+
+ ret = odp_pktio_capability(pktio, &capa);
+ (void)odp_pktio_close(pktio);
+
+ if (ret < 0 ||
+ !capa.config.pktin.bit.ipv4_chksum)
+ return ODP_TEST_INACTIVE;
+
+ return ODP_TEST_ACTIVE;
+}
+
+static void pktio_test_chksum_in_ipv4_config(odp_pktio_t pktio_tx ODP_UNUSED,
+ odp_pktio_t pktio_rx)
+{
+ odp_pktio_capability_t capa;
+ odp_pktio_config_t config;
+
+ CU_ASSERT_FATAL(odp_pktio_capability(pktio_rx, &capa) == 0);
+ CU_ASSERT_FATAL(capa.config.pktin.bit.ipv4_chksum);
+
+ odp_pktio_config_init(&config);
+ config.pktin.bit.ipv4_chksum = 1;
+ CU_ASSERT_FATAL(odp_pktio_config(pktio_rx, &config) == 0);
+}
+
+static void pktio_test_chksum_in_ipv4_prep(odp_packet_t pkt)
+{
+ odph_ipv4_csum_update(pkt);
+}
+
+static void pktio_test_chksum_in_ipv4_test(odp_packet_t pkt)
+{
+ CU_ASSERT(odp_packet_l3_chksum_status(pkt) == ODP_PACKET_CHKSUM_OK);
+}
+
+static void pktio_test_chksum_in_ipv4(void)
+{
+ pktio_test_chksum(pktio_test_chksum_in_ipv4_config,
+ pktio_test_chksum_in_ipv4_prep,
+ pktio_test_chksum_in_ipv4_test);
+}
+
+static int pktio_check_chksum_in_udp(void)
+{
+ odp_pktio_t pktio;
+ odp_pktio_capability_t capa;
+ odp_pktio_param_t pktio_param;
+ int idx = (num_ifaces == 1) ? 0 : 1;
+ int ret;
+
+ odp_pktio_param_init(&pktio_param);
+ pktio_param.in_mode = ODP_PKTIN_MODE_DIRECT;
+
+ pktio = odp_pktio_open(iface_name[idx], pool[idx], &pktio_param);
+ if (pktio == ODP_PKTIO_INVALID)
+ return ODP_TEST_INACTIVE;
+
+ ret = odp_pktio_capability(pktio, &capa);
+ (void)odp_pktio_close(pktio);
+
+ if (ret < 0 ||
+ !capa.config.pktin.bit.udp_chksum)
+ return ODP_TEST_INACTIVE;
+
+ return ODP_TEST_ACTIVE;
+}
+
+static void pktio_test_chksum_in_udp_config(odp_pktio_t pktio_tx ODP_UNUSED,
+ odp_pktio_t pktio_rx)
+{
+ odp_pktio_capability_t capa;
+ odp_pktio_config_t config;
+
+ CU_ASSERT_FATAL(odp_pktio_capability(pktio_rx, &capa) == 0);
+ CU_ASSERT_FATAL(capa.config.pktin.bit.udp_chksum);
+
+ odp_pktio_config_init(&config);
+ config.pktin.bit.udp_chksum = 1;
+ CU_ASSERT_FATAL(odp_pktio_config(pktio_rx, &config) == 0);
+}
+
+static void pktio_test_chksum_in_udp_prep(odp_packet_t pkt)
+{
+ odp_packet_has_ipv4_set(pkt, 1);
+ odp_packet_has_udp_set(pkt, 1);
+ odph_ipv4_csum_update(pkt);
+ odph_udp_chksum_set(pkt);
+}
+
+static void pktio_test_chksum_in_udp_test(odp_packet_t pkt)
+{
+ CU_ASSERT(odp_packet_l4_chksum_status(pkt) == ODP_PACKET_CHKSUM_OK);
+}
+
+static void pktio_test_chksum_in_udp(void)
+{
+ pktio_test_chksum(pktio_test_chksum_in_udp_config,
+ pktio_test_chksum_in_udp_prep,
+ pktio_test_chksum_in_udp_test);
+}
+
+static int pktio_check_chksum_out_ipv4(void)
+{
+ odp_pktio_t pktio;
+ odp_pktio_capability_t capa;
+ odp_pktio_param_t pktio_param;
+ int ret;
+
+ odp_pktio_param_init(&pktio_param);
+ pktio_param.in_mode = ODP_PKTIN_MODE_DIRECT;
+
+ pktio = odp_pktio_open(iface_name[0], pool[0], &pktio_param);
+ if (pktio == ODP_PKTIO_INVALID)
+ return ODP_TEST_INACTIVE;
+
+ ret = odp_pktio_capability(pktio, &capa);
+ (void)odp_pktio_close(pktio);
+
+ if (ret < 0 ||
+ !capa.config.pktout.bit.ipv4_chksum_ena ||
+ !capa.config.pktout.bit.ipv4_chksum)
+ return ODP_TEST_INACTIVE;
+
+ return ODP_TEST_ACTIVE;
+}
+
+static void pktio_test_chksum_out_ipv4_config(odp_pktio_t pktio_tx,
+ odp_pktio_t pktio_rx ODP_UNUSED)
+{
+ odp_pktio_capability_t capa;
+ odp_pktio_config_t config;
+
+ CU_ASSERT_FATAL(odp_pktio_capability(pktio_tx, &capa) == 0);
+ CU_ASSERT_FATAL(capa.config.pktout.bit.ipv4_chksum_ena);
+ CU_ASSERT_FATAL(capa.config.pktout.bit.ipv4_chksum);
+
+ odp_pktio_config_init(&config);
+ config.pktout.bit.ipv4_chksum_ena = 1;
+ CU_ASSERT_FATAL(odp_pktio_config(pktio_tx, &config) == 0);
+}
+
+static void pktio_test_chksum_out_ipv4_test(odp_packet_t pkt)
+{
+ odph_ipv4hdr_t *ip = odp_packet_l3_ptr(pkt, NULL);
+
+ CU_ASSERT(ip != NULL);
+ if (ip != NULL)
+ CU_ASSERT(ip->chksum != 0);
+}
+
+static void pktio_test_chksum_out_ipv4_no_ovr_prep(odp_packet_t pkt)
+{
+ odp_packet_l3_chksum_insert(pkt, false);
+}
+
+static void pktio_test_chksum_out_ipv4_no_ovr_test(odp_packet_t pkt)
+{
+ odph_ipv4hdr_t *ip = odp_packet_l3_ptr(pkt, NULL);
+
+ CU_ASSERT(ip != NULL);
+ if (ip != NULL)
+ CU_ASSERT(ip->chksum == 0);
+}
+
+static void pktio_test_chksum_out_ipv4_no_ovr(void)
+{
+ pktio_test_chksum(pktio_test_chksum_out_ipv4_config,
+ pktio_test_chksum_out_ipv4_no_ovr_prep,
+ pktio_test_chksum_out_ipv4_no_ovr_test);
+}
+
+static void pktio_test_chksum_out_ipv4_ovr_prep(odp_packet_t pkt)
+{
+ odp_packet_l3_chksum_insert(pkt, true);
+}
+
+static void pktio_test_chksum_out_ipv4_ovr_test(odp_packet_t pkt)
+{
+ odph_ipv4hdr_t *ip = odp_packet_l3_ptr(pkt, NULL);
+
+ CU_ASSERT(ip != NULL);
+ if (ip != NULL)
+ CU_ASSERT(ip->chksum != 0);
+}
+
+static void pktio_test_chksum_out_ipv4_ovr(void)
+{
+ pktio_test_chksum(pktio_test_chksum_out_ipv4_config,
+ pktio_test_chksum_out_ipv4_ovr_prep,
+ pktio_test_chksum_out_ipv4_ovr_test);
+}
+
+static void pktio_test_chksum_out_ipv4_pktio_config(odp_pktio_t pktio_tx,
+ odp_pktio_t pktio_rx
+ ODP_UNUSED)
+{
+ odp_pktio_capability_t capa;
+ odp_pktio_config_t config;
+
+ CU_ASSERT_FATAL(odp_pktio_capability(pktio_tx, &capa) == 0);
+ CU_ASSERT_FATAL(capa.config.pktout.bit.ipv4_chksum_ena);
+ CU_ASSERT_FATAL(capa.config.pktout.bit.ipv4_chksum);
+
+ odp_pktio_config_init(&config);
+ config.pktout.bit.ipv4_chksum_ena = 1;
+ config.pktout.bit.ipv4_chksum = 1;
+ CU_ASSERT_FATAL(odp_pktio_config(pktio_tx, &config) == 0);
+}
+
+static void pktio_test_chksum_out_ipv4_pktio(void)
+{
+ pktio_test_chksum(pktio_test_chksum_out_ipv4_pktio_config,
+ NULL,
+ pktio_test_chksum_out_ipv4_test);
+}
+
+static int pktio_check_chksum_out_udp(void)
+{
+ odp_pktio_t pktio;
+ odp_pktio_capability_t capa;
+ odp_pktio_param_t pktio_param;
+ int ret;
+
+ odp_pktio_param_init(&pktio_param);
+ pktio_param.in_mode = ODP_PKTIN_MODE_DIRECT;
+
+ pktio = odp_pktio_open(iface_name[0], pool[0], &pktio_param);
+ if (pktio == ODP_PKTIO_INVALID)
+ return ODP_TEST_INACTIVE;
+
+ ret = odp_pktio_capability(pktio, &capa);
+ (void)odp_pktio_close(pktio);
+
+ if (ret < 0 ||
+ !capa.config.pktout.bit.udp_chksum_ena ||
+ !capa.config.pktout.bit.udp_chksum)
+ return ODP_TEST_INACTIVE;
+
+ return ODP_TEST_ACTIVE;
+}
+
+static void pktio_test_chksum_out_udp_config(odp_pktio_t pktio_tx,
+ odp_pktio_t pktio_rx ODP_UNUSED)
+{
+ odp_pktio_capability_t capa;
+ odp_pktio_config_t config;
+
+ CU_ASSERT_FATAL(odp_pktio_capability(pktio_tx, &capa) == 0);
+ CU_ASSERT_FATAL(capa.config.pktout.bit.udp_chksum_ena);
+ CU_ASSERT_FATAL(capa.config.pktout.bit.udp_chksum);
+
+ odp_pktio_config_init(&config);
+ config.pktout.bit.udp_chksum_ena = 1;
+ CU_ASSERT_FATAL(odp_pktio_config(pktio_tx, &config) == 0);
+}
+
+static void pktio_test_chksum_out_udp_test(odp_packet_t pkt)
+{
+ odph_udphdr_t *udp = odp_packet_l4_ptr(pkt, NULL);
+
+ CU_ASSERT(udp != NULL);
+ if (udp != NULL) {
+ CU_ASSERT(udp->chksum != 0);
+ CU_ASSERT(!odph_udp_chksum_verify(pkt));
+ }
+}
+
+static void pktio_test_chksum_out_udp_no_ovr_prep(odp_packet_t pkt)
+{
+ odph_ipv4_csum_update(pkt);
+ odp_packet_l4_chksum_insert(pkt, false);
+}
+
+static void pktio_test_chksum_out_udp_no_ovr_test(odp_packet_t pkt)
+{
+ odph_udphdr_t *udp = odp_packet_l4_ptr(pkt, NULL);
+
+ CU_ASSERT(udp != NULL);
+ if (udp != NULL)
+ CU_ASSERT(udp->chksum == 0);
+}
+
+static void pktio_test_chksum_out_udp_no_ovr(void)
+{
+ pktio_test_chksum(pktio_test_chksum_out_udp_config,
+ pktio_test_chksum_out_udp_no_ovr_prep,
+ pktio_test_chksum_out_udp_no_ovr_test);
+}
+
+static void pktio_test_chksum_out_udp_ovr_prep(odp_packet_t pkt)
+{
+ odp_packet_l4_chksum_insert(pkt, true);
+}
+
+static void pktio_test_chksum_out_udp_ovr_test(odp_packet_t pkt)
+{
+ odph_udphdr_t *udp = odp_packet_l4_ptr(pkt, NULL);
+
+ CU_ASSERT(udp != NULL);
+ if (udp != NULL)
+ CU_ASSERT(udp->chksum != 0);
+}
+
+static void pktio_test_chksum_out_udp_ovr(void)
+{
+ pktio_test_chksum(pktio_test_chksum_out_udp_config,
+ pktio_test_chksum_out_udp_ovr_prep,
+ pktio_test_chksum_out_udp_ovr_test);
+}
+
+static void pktio_test_chksum_out_udp_pktio_config(odp_pktio_t pktio_tx,
+ odp_pktio_t pktio_rx
+ ODP_UNUSED)
+{
+ odp_pktio_capability_t capa;
+ odp_pktio_config_t config;
+
+ CU_ASSERT_FATAL(odp_pktio_capability(pktio_tx, &capa) == 0);
+ CU_ASSERT_FATAL(capa.config.pktout.bit.udp_chksum_ena);
+ CU_ASSERT_FATAL(capa.config.pktout.bit.udp_chksum);
+
+ odp_pktio_config_init(&config);
+ config.pktout.bit.udp_chksum_ena = 1;
+ config.pktout.bit.udp_chksum = 1;
+ CU_ASSERT_FATAL(odp_pktio_config(pktio_tx, &config) == 0);
+}
+
+static void pktio_test_chksum_out_udp_pktio(void)
+{
+ pktio_test_chksum(pktio_test_chksum_out_udp_pktio_config,
+ NULL,
+ pktio_test_chksum_out_udp_test);
+}
+
static int create_pool(const char *iface, int num)
{
char pool_name[ODP_POOL_NAME_LEN];
@@ -2034,6 +2471,22 @@ odp_testinfo_t pktio_suite_unsegmented[] = {
pktio_check_statistics_counters),
ODP_TEST_INFO_CONDITIONAL(pktio_test_pktin_ts,
pktio_check_pktin_ts),
+ ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_in_ipv4,
+ pktio_check_chksum_in_ipv4),
+ ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_in_udp,
+ pktio_check_chksum_in_udp),
+ ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_out_ipv4_no_ovr,
+ pktio_check_chksum_out_ipv4),
+ ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_out_ipv4_pktio,
+ pktio_check_chksum_out_ipv4),
+ ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_out_ipv4_ovr,
+ pktio_check_chksum_out_ipv4),
+ ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_out_udp_no_ovr,
+ pktio_check_chksum_out_udp),
+ ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_out_udp_pktio,
+ pktio_check_chksum_out_udp),
+ ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_out_udp_ovr,
+ pktio_check_chksum_out_udp),
ODP_TEST_INFO_NULL
};