diff options
151 files changed, 5927 insertions, 2631 deletions
diff --git a/.checkpatch.conf b/.checkpatch.conf index 1e7d6638a..990a54d2d 100644 --- a/.checkpatch.conf +++ b/.checkpatch.conf @@ -6,5 +6,9 @@ --ignore=DEPRECATED_VARIABLE --ignore=COMPARISON_TO_NULL --ignore=BIT_MACRO +--ignore=PREFER_PRINTF +--ignore=PREFER_SCANF +--ignore=VOLATILE +--ignore=AVOID_EXTERNS --codespell --codespellfile=/usr/share/codespell/dictionary.txt diff --git a/.codecov.yml b/.codecov.yml index 36270d618..327f65496 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -8,10 +8,16 @@ coverage: range: "50...75" status: project: - target: 70% - threshold: 5% - patch: yes - changes: yes + default: + enabled: yes + target: 70% + threshold: 5% + patch: + default: + enabled: yes + target: 70% + threshold: 5% + changes: no parsers: gcov: diff --git a/.gitignore b/.gitignore index fbc0eab96..cce24282a 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ libtool ltmain.sh m4/*.m4 missing +perf.data* pkgconfig/libodp*.pc tags test-driver diff --git a/.travis.yml b/.travis.yml index 83ec65f89..50b3b9c27 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,112 +33,128 @@ env: # you need generated new one at https://codecov.io specific for your repo. - CODECOV_TOKEN=8e1c0fd8-62ff-411e-a79f-5839f6662c11 -matrix: - include: - - compiler: gcc - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc - env: MY_CF="-O0 -coverage" MY_LDF="--coverage" DOCOV=1 - - compiler: clang - addons: +addons: apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-precise-3.8 - packages: - - clang-3.8 + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-precise-3.8 + packages: + - gcc + - clang-3.8 + - automake autoconf autoconf-archive libtool libssl-dev graphviz mscgen doxygen + - libpcap-dev +# coverity_scan: +# project: +# name: "$TRAVIS_REPO_SLUG" +# notification_email: xxxx +# build_command_prepend: "./bootstrap && ./configure --enable-test-cpp --enable-test-vald --enable-test-helper --enable-test-perf --enable-user-guides --enable-test-perf-proc --enable-test-example" +# build_command: "make" +# branch_pattern: coverity_scan -before_install: +compiler: + - gcc + - clang-3.8 + +env: + - CONF="" + - CONF="--disable-abi-compat" + - CONF="--enable-schedule-sp" + - CONF="--enable-schedule-iquery" + +install: - echo 1000 | sudo tee /proc/sys/vm/nr_hugepages - sudo mkdir -p /mnt/huge - sudo mount -t hugetlbfs nodev /mnt/huge - sudo apt-get -qq update - - sudo apt-get install automake autoconf libtool libssl-dev graphviz mscgen doxygen - - sudo apt-get install libpcap-dev linux-headers-`uname -r` + - sudo apt-get install linux-headers-`uname -r` - sudo pip install coverage - gem install asciidoctor - PATH=${PATH//:\.\/node_modules\/\.bin/} - - if [ "$CC" = "clang" ]; then export CXX="clang++-3.8" CC="clang-3.8" LD="clang-3.8"; fi # Install cunit for the validation tests because distro version is too old and fails C99 compile - sudo apt-get remove libcunit1-dev libcunit1 - export CUNIT_VERSION=2.1-3 - curl -sSOL https://github.com/Linaro/libcunit/releases/download/${CUNIT_VERSION}/CUnit-${CUNIT_VERSION}.tar.bz2 - tar -jxf *.bz2 - - cd CUnit* + - pushd CUnit* - ./bootstrap - ./configure --enable-debug --enable-automated --enable-basic --enable-console --enable-examples --enable-test - make - sudo make install - - cd .. + - popd - export LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" # DPDK pktio - TARGET=${TARGET:-"x86_64-native-linuxapp-gcc"} - - git -c advice.detachedHead=false clone -q --depth=1 --single-branch --branch=v16.07 http://dpdk.org/git/dpdk dpdk + - git -c advice.detachedHead=false clone -q --depth=1 --single-branch --branch=v17.02 http://dpdk.org/git/dpdk dpdk - pushd dpdk - git log --oneline --decorate - make config T=${TARGET} O=${TARGET} - pushd ${TARGET} - sed -ri 's,(CONFIG_RTE_LIBRTE_PMD_PCAP=).*,\1y,' .config - popd - - make install T=${TARGET} EXTRA_CFLAGS="-fPIC" > /dev/null + - make install T=${TARGET} EXTRA_CFLAGS="-fPIC" - popd # Netmap pktio - - sudo apt-get source linux-image-$(uname -r) > /dev/null - - KSRC="$PWD/`find . -name kernel-parameters.txt | cut -d / -f 2 `" - git -c advice.detachedHead=false clone -q --depth=1 --single-branch --branch=v11.2 https://github.com/luigirizzo/netmap.git - pushd netmap/LINUX - - ./configure --kernel-sources=$KSRC - - make > /dev/null + - ./configure + - make - sudo insmod ./netmap.ko - popd script: - - echo $TRAVIS_COMMIT_RANGE - - ODP_PACHES=`echo $TRAVIS_COMMIT_RANGE | sed 's/\.//'` -# Generate patches provided with $TRAVIS_COMMIT_RANGE. -# In case of force push and range is broken validate only the latest commit if it's not merge commit. - - git format-patch $ODP_PACHES; - if [ $? -ne 0 ]; then - git show --summary HEAD| grep -q '^Merge:'; - if [ $? -ne 0 ]; then - git format-patch HEAD^; - perl ./scripts/checkpatch.pl *.patch; - fi; - else - perl ./scripts/checkpatch.pl *.patch; - fi - - ./bootstrap - - ./configure -# doxygen does not trap on warnings, check for them here. - - make doxygen-doc |tee doxygen.log - - fgrep -rvq warning ./doxygen.log - - make distcheck - - - ./bootstrap - - ./configure --enable-test-cpp --enable-test-vald --enable-test-helper --enable-test-perf --enable-user-guides --enable-test-perf-proc --enable-test-example --with-dpdk-path=`pwd`/dpdk/${TARGET} --with-netmap-path=`pwd`/netmap CFLAGS="$MY_CF" CXXFLAGS="$MY_CF" LDFLAGS="$MY_LDF" - - make -j 4 + - ./configure --prefix=$HOME/odp-install --enable-test-cpp --enable-test-vald --enable-test-helper --enable-test-perf --enable-user-guides --enable-test-perf-proc --enable-test-example --with-dpdk-path=`pwd`/dpdk/${TARGET} --with-netmap-path=`pwd`/netmap $CONF + - make -j $(nproc) - sudo LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" make check - - sudo rm -rf dpdk - - sudo rm -rf netmap - - sudo rm -rf $KSRC + - make install -after_success: - - if [ -n "$DOCOV" ]; then find . -type f -iname '*.[ch]' -not -path ".git/*" -execdir gcov {} \; ; bash <(curl -s https://codecov.io/bash) -X coveragepy; fi + - echo "Checking linking and run from install..." + - pushd $HOME + - echo "Dynamic link.." + - ${CC} ${OLDPWD}/example/hello/odp_hello.c -o odp_hello_inst -I${HOME}/odp-install/include -L${HOME}/odp-install/lib -lodp-linux -L${OLDPWD}/dpdk/x86_64-native-linuxapp-gcc/lib -lrt -ldpdk -lpthread -lcrypto -lpcap -ldl + - LD_LIBRARY_PATH="${HOME}/odp-install/lib:$LD_LIBRARY_PATH" ./odp_hello_inst + - echo "Static link.." + - ${CC} ${OLDPWD}/example/hello/odp_hello.c -o odp_hello_inst -I${HOME}/odp-install/include -L${HOME}/odp-install/lib -lodp-linux -L${OLDPWD}/dpdk/x86_64-native-linuxapp-gcc/lib -lrt -ldpdk -lpthread -lcrypto -lpcap -ldl -static + - ./odp_hello_inst -addons: - coverity_scan: - project: - name: "$TRAVIS_REPO_SLUG" - notification_email: xxxx - build_command_prepend: "./bootstrap && ./configure --enable-test-cpp --enable-test-vald --enable-test-helper --enable-test-perf --enable-user-guides --enable-test-perf-proc --enable-test-example" - build_command: "make" - branch_pattern: coverity_scan +jobs: + include: + - stage: test + env: TEST=coverage + compiler: gcc + script: + - ./bootstrap + - ./configure --prefix=$HOME/odp-install --enable-test-cpp --enable-test-vald --enable-test-helper --enable-test-perf --enable-user-guides --enable-test-perf-proc --enable-test-example --with-dpdk-path=`pwd`/dpdk/${TARGET} --with-netmap-path=`pwd`/netmap CFLAGS="-O0 -coverage" CXXFLAGS="-O0 -coverage" LDFLAGS="--coverage" + - sudo LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" PATH=${PATH//:\.\/node_modules\/\.bin/} make check + - find . -type f -iname '*.[ch]' -not -path ".git/*" -execdir gcov {} \; ; bash <(curl -s https://codecov.io/bash) -X coveragepy + - stage: test + env: TEST=distcheck + compiler: gcc + script: + - ./bootstrap + - ./configure + - sudo LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" make distcheck + - stage: test + env: TEST=doxygen + compiler: gcc + script: + # doxygen does not trap on warnings, check for them here. + - ./bootstrap + - ./configure + - make doxygen-doc |tee doxygen.log + - fgrep -rvq warning ./doxygen.log + - stage: test + env: TEST=checkpatch + compiler: gcc + script: + - echo ${TRAVIS_COMMIT_RANGE}; + - ODP_PATCHES=`echo ${TRAVIS_COMMIT_RANGE} | sed 's/\.//'`; + - if [ -z "${ODP_PATCHES}" ]; then env; exit 1; fi; + - ./scripts/ci-checkpatches.sh ${ODP_PATCHES}; + +after_failure: + - find . -name 'test-suite.log' -execdir grep -il "FAILED" {} \; -exec echo {} \; -exec cat {} \; @@ -1,3 +1,270 @@ +== OpenDataPlane (1.15.0.0) +=== New Features +ODP v1.15.0.0 continues the preview of Tiger Moth, introducing new APIs and +extensions, as well as numerous bug fixes and functional improvements. + +==== Deprecation Framework +To permit smoother evolution of the ODP API specification, a deprecation +framework is introduced to permit controlled deprecation. + +When an ODP API or defined struct parameter is deprecated, ODP validation +tests will be updated to no longer use that API and instead use the +replacement API. By default, attempts to compile with the older API/feature +will fail and applications wishing to move to the new ODP release should be +updated to use the replacement API. To permit evaluation of new ODP +releases in advance of such updating, however, ODP supports the `configure` +option `--enable-deprecated`, which makes the obsolete APIs visible again. + +This feature will be used on a case-by-case basis and documented in the +release notes for each release that introduces replacement API(s). In general +the deprecated forms will not be maintained for more than a single release +cycle. After that they will no longer be present in the API specification and +the replacement forms must be used to compile with that level of the API +specification. + +==== APIs +A number of new and refined APIs are introduced in crypto, packet parsing, +and queue configuration: + +===== Crypto Enhancements +The ODP crypto APIs receive several enhancements in this release: + +====== New Authentication Algorithms +Additional enumerations added for HMAC-SHA-1 and HMAC-SHA-512 authentication. + +====== Deprecated Cipher/Authentication Algorithms +The following cipher/authentication algorithms have been deprecated in favor +of newer replacement algorithms: + +* `ODP_CIPHER_ALG_AES128_CBC` is replaced by `ODP_CIPHER_ALG_AES_CBC` +* `ODP_CIPHER_ALG_AES128_GCM` is replaced by `ODP_CIPHER_ALG_AES_GCM` +* `ODP_AUTH_ALG_MD5_96` is replaced by `ODP_AUTH_ALG_MD5_HMAC` +* `ODP_AUTH_ALG_SHA256_128` is replaced by `ODP_AUTH_ALG_SHA256_HMAC` +* `ODP_AUTH_ALG_AES128_GCM1 is replaced by `ODP_AUTH_ALG_AES_GCM` + +====== Deprecated Name for Crypto Parameter struct +`odp_crypto_op_params_t` is deprecated in favor of `odp_crypto_op_param_t` +for consistency with other ODP `param` structs. + +====== Digest Length Session Parameter +`odp_crypto_session_param_t` adds the field `auth_digest_len` to permit +specification of the authentication digest length to be used for this +session. The `odp_crypto_auth_capability()` API returns the list of +supported digest lengths. + +====== Additional Authentication Data (AAD) +The `odp_crypto_op_param_t` struct adds an optional pointer and length for +AAD information. This allows applications to specify AAD information from +the list of supported lengths provided by `odp_crypto_auth_capability()`. + +===== Packet Range Data +The former `odp_crypto_data_range_t` type is deprecated and renamed to +`odp_packet_data_range_t` since it can be used to specify ranges for other +than crypto purposes. + +===== Parser Configuration +Applications may now specify the maximum packet layer of interest. For +example, a router may not care about anything beyond ISO Layer 3 in packets. +This can be used by ODP implementations to control the depth of packet +parsing needed by the application and may allow greater efficiency, +especially in software implementations. + +This is controlled by a new `odp_pktio_parser_layer_t` enum that is +part of the new `odp_pktio_parser_config_t` struct that is added to the +`odp_pktio_config_t` struct used by the `odp_pktio_config()` API. The +supported parser layers are also returned in this struct as part of the +`odp_pktio_capability_t` struct returned by the `odp_pktio_capability()` API. + +===== Queue Size Parameter +The `odp_queue_capability_t` struct returned by the `odp_queue_capability()` +API is enhanced as follows: +* The `max_queues` field is now defined to return the maximum number of event +queues of any type. +* New sub-structs (`plain` and `sched`) are added that detail the `max_num` of +queues of that type supported as well as the new field `max_size` that +specifies the maximum number of elements that each queue of this type can +store. A value of zero for `max_num` indicates no fixed limit. + +In addition, the `odp_queue_param_t` passed to `odp_queue_create()` now adds +a `size` field to allow applications to specify the minimum number of events +that this queue should be able to store. A value of zero specified requests that +the implementation default limits be used. + +The ODP examples have been updated to show this configuration, for example, +the `l2fwd_simple` example does Layer 2 forwarding and hence doesn't need +packets to be parsed beyond Layer 2. So prior to opening an interface via +`odp_pktio_open()` the configuration processing now includes code of the form: +[source,c] +----- +odp_pktio_config_init(&config); +config.parser.layer = ODP_PKTIO_PARSER_LAYER_L2; +odp_pktio_config(pktio, &config); +----- +This serves as a hint to the ODP implementation that parsing beyond Layer 2 +is not required on this interface. + +===== Removal of `odp_time_to_u64()` API +The `odp_time_to_u64()` API is removed without deprecation in this release. +This is redundant since the `odp_time_to_ns()` API already returns a `uint64_t` +value for `odp_time_t` and can be used for the same purpose. + +===== New `odp_sys_info_print()` API +For debugging / diagnostic use, the `odp_sys_info_print()` API is added. +This prints implementation defined information about the system and ODP +environment and may contain information about CPUs, memory, and other +hardware configuration. + +=== Helpers +==== Linux Helpers +ODP helper functions fall into two categories: system-independent, and those +that rely on Linux. For backwards compatibility, these have been +reorganized into a new `helper/linux` directory that is installed only when +the `--enable-helper-linux` `configure` option is specified. + +=== `odp-linux` Performance +==== Scheduler Improvements +The default scheduler is streamlined by using precomputed weight tables and +separated priority queues for different scheduling groups. This improves +both latency and scalability. + +Also included in this change, the default number of scheduling groups is +lowered from 256 to 32, which experience has shown is sufficient for most +applications. + +The scheduler fairness checks are also enhanced to avoid low priority event +starvation if `CONFIG_BURST_SIZE` is set to 1. + +=== `odp_linux` Crypto Improvements +* Errors in implicit packet copy operations performed during crypto operation +are now handled properly. +* If `odp_crypto_session_create()` fails, proper cleanup is now performed to +avoid memory leaks. +* The auth digest len is now used when calculating HMACs, and full-length +digests are now added to supported SHA capabilities. +* Numerous crypto functions have been rewritten to use the OpenSSL `EVP_` +functions for improved efficiency and accuracy. + +=== `odp-linux` Packet Improvements +* The packet parser now recognizes ICMPv6 packets in addition to ICMPv4. + +=== `odp-linux` PktIO Improvements +* The `ethtool` and `socket` drivers add additional initializations to avoid +valgrind warnings. +* The `dpdk` driver is upgraded to DPDK 17.02 and is now also built with the +`--whole-archive` option, which means that PMDs no longer need to be linked +individually but are automatically included in builds that make use of DPDK +pktio support. + +=== `odp-linux` Time Improvements +* The ODP Time APIs are now more efficiently handled by replacing the previous +Linux timespec with simple nanoseconds (ns) and using native hardware time +counters in x86 and ARM environments, when available. + +=== `odp-linux` Traffic Manager Improvements +* Weighting is now handled properly when weight=1 is specified. Previously +this caused an overflow that distorted packet priorities. + +=== Miscellaneous Fixes and Improvements +==== Native Clang build on ARMv8 +ARMv8 compilation now works properly when using the clang compiler. + +==== Test Improvements +* The `odp_scheduling` performance test now handles dequeueing from +potentially concurrent queues properly. +* The traffic manager and time tests are now invoked via scripts to better +account for load-sensitivity when running regressions in CI environments. +* The l2fwd test now supports an additional parameter to specify the number +of scheduling groups to use in the test. +* The `odp_sched_latency` performance test now handles queue draining in +a robust manner. +* The crypto validation tests now properly check and validate message digests, +and include negative tests to verify that incorrect digests are properly +flagged. Note that this may expose omissions in other ODP implementations that +had previously passed this validation test. +* The crypto validation test now adds an explicit test for the NULL cipher, +as well as HMAC-SHA-1 and HMAC-SHA-512. +* The pktio_ipc test now properly cleans up only its own temp files. + +==== Examples Improvements +* The `odp_generator` example now properly honors stop criteria based on +number of packets sent. +* The `odp_generator` example now properly flushes output messages +* The `odp_generator` example now supports port configuration. + +=== Bug Fixes +==== https://bugs.linaro.org/show_bug.cgi?id=2416[Bug 2416] +example/generator/odp_generator.c contains todo items + +==== https://bugs.linaro.org/show_bug.cgi?id=2779[Bug 2779] +Error handling issues (CHECKED_RETURN) + +==== https://bugs.linaro.org/show_bug.cgi?id=2826[Bug 2826] +Unchecked return in pool.c + +==== https://bugs.linaro.org/show_bug.cgi?id=2831[Bug 2831] +Unchecked return in mmap_unmap_sock() + +==== https://bugs.linaro.org/show_bug.cgi?id=2852[Bug 2852] +ODP_STATIC_ASSERT() fails when used by C++ + +==== https://bugs.linaro.org/show_bug.cgi?id=2872[Bug 2872] +odp_pktio_ordered_run.sh: line 34: integer expression expected + +==== https://bugs.linaro.org/show_bug.cgi?id=2881[Bug 2881] +load sensitive tests fail on CI + +==== https://bugs.linaro.org/show_bug.cgi?id=2895[Bug 2895] +The `odp_crypto_operation()` routine does not work with multi-segment packets. + +==== https://bugs.linaro.org/show_bug.cgi?id=2908[Bug 2908] +Packet validation test fails if implementation does not limit packet length + +==== https://bugs.linaro.org/show_bug.cgi?id=2910[Bug 2910] +odph_iplookup_table_put_value() uses overlapping pointer addresses + +==== https://bugs.linaro.org/show_bug.cgi?id=2933[Bug 2933] +Miss to call unlock if there are some errors happen in loopback_send() function + +==== https://bugs.linaro.org/show_bug.cgi?id=2940[Bug 2940] +`odp_packet_seg_t` fails ABI compatibility between odp-linux and odp-dpdk + +==== https://bugs.linaro.org/show_bug.cgi?id=2942[Bug 2942] +Compilation fails using clang 4.0.0 + +==== https://bugs.linaro.org/show_bug.cgi?id=2952[Bug 2952] +doxygen errors and travis does not catch them + +==== https://bugs.linaro.org/show_bug.cgi?id=2969[Bug 2969] +TM validation test does find traffic_mngr_main + +==== https://bugs.linaro.org/show_bug.cgi?id=2974[Bug 2974] +`odp_rwlock_read_trylock()` fails if lock is already held for read access + +==== https://bugs.linaro.org/show_bug.cgi?id=3002[Bug 3002] +Packet order lost when enqueuing to pktout queue + +==== https://bugs.linaro.org/show_bug.cgi?id=3003[Bug 3003] +AES-GCM returns 'valid' tag when checking invalid tag + +==== https://bugs.linaro.org/show_bug.cgi?id=3013[Bug 3013] +Various failures if CONFIG_PACKET_MAX_SEGS is set to 1 + +==== https://bugs.linaro.org/show_bug.cgi?id=3017[Bug 3017] +Travis: time main test out of boundaries + +==== https://bugs.linaro.org/show_bug.cgi?id=3027[Bug 3027] +Compilation failures using GCC 7 series + +==== https://bugs.linaro.org/show_bug.cgi?id=3039[Bug 3039] +Socket pktio recv fails on large number of packet + +=== Known Issues +==== https://bugs.linaro.org/show_bug.cgi?id=3024[Bug 3024] +odp_traffic_mngr example is broken + +==== https://bugs.linaro.org/show_bug.cgi?id=3026[Bug 3026] +pktio_ips_run test can fail due to segfault + == OpenDataPlane (1.14.0.0) === New Features ==== APIs diff --git a/DEPENDENCIES b/DEPENDENCIES index 574859ce5..7bcbd5ebc 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -8,13 +8,14 @@ Prerequisites for building the OpenDataPlane (ODP) API automake autoconf + autoconf-archive libtool On Debian/Ubuntu systems: - $ sudo apt-get install automake autoconf libtool + $ sudo apt-get install automake autoconf autoconf-archive libtool On CentOS/RedHat/Fedora systems: - $ sudo yum install automake autoconf libtool + $ sudo yum install automake autoconf autoconf-archive libtool 3. Required libraries @@ -165,7 +166,7 @@ Prerequisites for building the OpenDataPlane (ODP) API 3.4.1 Building DPDK and ODP with DPDK pktio support - DPDK packet I/O has been tested to work with DPDK v16.07. + DPDK packet I/O has been tested to work with DPDK v17.02. Follow steps in ./scripts/build-pktio-dpdk @@ -215,8 +216,22 @@ Prerequisites for building the OpenDataPlane (ODP) API tar -jxf *.bz2 cd CUnit* ./bootstrap + + # Install CUnit into the default location (/usr/local). This is needed for + # 'make distcheck'. + ./configure + make + sudo make install + + # ... OR ... Install CUnit into user defined location. The same path is + # used in step 4.4 --with-cunit-path=/home/<my_cunit_path> + ./configure --prefix=/home/<my_cunit_path> + make make install - #In Step 4.4 use --with-cunit-path=/home/${USER}/CUnitHome + + # Also (in Ubuntu at least) run ldconfig to update shared lib cache or + # reboot, before trying to run e.g. 'make distcheck'. + sudo ldconfig 4.3 Cross compile of Cunit @@ -227,8 +242,11 @@ Prerequisites for building the OpenDataPlane (ODP) API 4.4 Using Cunit with ODP $ Add the configuration option to the regular configuration options - ./configure --enable-cunit #if cunit is in the PATH - ./configure --with-cunit-path=DIR #only if you need a path to Cunit libs and headers + # Use the default path ... + ./configure --enable-cunit + + # ... OR the user defined path. + ./configure --with-cunit-path=/home/<my_cunit_path> 5.0 Documentation Images & Doxygen diff --git a/Makefile.am b/Makefile.am index 2a9a658df..daa497886 100644 --- a/Makefile.am +++ b/Makefile.am @@ -8,10 +8,10 @@ AM_DISTCHECK_CONFIGURE_FLAGS = --enable-test-cpp \ #@with_platform@ works alone in subdir but not as part of a path??? SUBDIRS = @platform_with_platform@ \ helper \ - test \ helper/test \ doc \ - example + example . \ + test @DX_RULES@ diff --git a/configure.ac b/configure.ac index 6fa7f970b..001ebfc28 100644 --- a/configure.ac +++ b/configure.ac @@ -3,7 +3,7 @@ AC_PREREQ([2.5]) # Set correct API version ########################################################################## m4_define([odpapi_generation_version], [1]) -m4_define([odpapi_major_version], [14]) +m4_define([odpapi_major_version], [15]) m4_define([odpapi_minor_version], [0]) m4_define([odpapi_point_version], [0]) m4_define([odpapi_version], @@ -16,7 +16,8 @@ ODP_VERSION_API_MAJOR=odpapi_major_version AC_SUBST(ODP_VERSION_API_MAJOR) ODP_VERSION_API_MINOR=odpapi_minor_version AC_SUBST(ODP_VERSION_API_MINOR) -AC_CONFIG_FILES([include/odp/api/spec/version.h]) +AC_CONFIG_FILES([include/odp/api/spec/version.h + include/odp/api/spec/deprecated.h]) AM_INIT_AUTOMAKE([1.9 tar-pax subdir-objects]) AC_CONFIG_SRCDIR([helper/config.h.in]) @@ -38,10 +39,10 @@ AM_SILENT_RULES([yes]) # 3. if interfaces were removed, then use C+1:0:0 ########################################################################## -ODP_LIBSO_VERSION=114:0:1 +ODP_LIBSO_VERSION=115:0:2 AC_SUBST(ODP_LIBSO_VERSION) -ODPHELPER_LIBSO_VERSION=112:0:0 +ODPHELPER_LIBSO_VERSION=112:1:0 AC_SUBST(ODPHELPER_LIBSO_VERSION) # Checks for programs. @@ -213,7 +214,6 @@ AC_SUBST([testdir]) # Set conditionals as computed within platform specific files ########################################################################## AM_CONDITIONAL([netmap_support], [test x$netmap_support = xyes ]) -AM_CONDITIONAL([PKTIO_IPC], [test x$pktio_ipc_support = xyes]) AM_CONDITIONAL([PKTIO_DPDK], [test x$pktio_dpdk_support = xyes ]) AM_CONDITIONAL([HAVE_PCAP], [test $have_pcap = yes]) AM_CONDITIONAL([DPDK_DEFAULT_DIR], [test "x${DPDK_DEFAULT_DIR}" = "x1"]) @@ -229,6 +229,7 @@ AM_CONDITIONAL([HAVE_DOXYGEN], [test "x${DOXYGEN}" = "xdoxygen"]) AM_CONDITIONAL([user_guide], [test "x${user_guides}" = "xyes" ]) AM_CONDITIONAL([HAVE_MSCGEN], [test "x${MSCGEN}" = "xmscgen"]) AM_CONDITIONAL([helper_linux], [test x$helper_linux = xyes ]) +AM_CONDITIONAL([ARCH_IS_X86], [test "x${ARCH_DIR}" = "xx86"]) ########################################################################## # Setup doxygen documentation @@ -288,7 +289,7 @@ ODP_CFLAGS="$ODP_CFLAGS -DODP_DEBUG=$ODP_DEBUG" ODP_ABI_COMPAT=1 abi_compat=yes AC_ARG_ENABLE([abi-compat], - [ --disable-abi-compat disables ABI compatible mode, enables inline code in header files], + [ --disable-abi-compat disables ABI compatible mode, enables inline code in header files], [if test "x$enableval" = "xno"; then ODP_ABI_COMPAT=0 abi_compat=no @@ -298,6 +299,19 @@ AC_ARG_ENABLE([abi-compat], AC_SUBST(ODP_ABI_COMPAT) ########################################################################## +# Enable/disable deprecated API definitions +########################################################################## +ODP_DEPRECATED_API=0 +deprecated=no +AC_ARG_ENABLE([deprecated], + [ --enable-deprecated enable deprecated API definitions], + [if test "x$enableval" = "xyes"; then + ODP_DEPRECATED_API=1 + deprecated=yes + fi]) +AC_SUBST(ODP_DEPRECATED_API) + +########################################################################## # Default warning setup ########################################################################## ODP_CFLAGS="$ODP_CFLAGS -W -Wall -Werror -Wstrict-prototypes -Wmissing-prototypes" @@ -305,24 +319,41 @@ ODP_CFLAGS="$ODP_CFLAGS -Wmissing-declarations -Wold-style-definition -Wpointer- ODP_CFLAGS="$ODP_CFLAGS -Wcast-align -Wnested-externs -Wcast-qual -Wformat-nonliteral" ODP_CFLAGS="$ODP_CFLAGS -Wformat-security -Wundef -Wwrite-strings" ODP_CFLAGS="$ODP_CFLAGS -std=gnu99" + +dnl Use -Werror in the checks below since Clang emits a warning instead of +dnl an error when it encounters an unknown warning option. +AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough=0], + [ODP_CFLAGS="$ODP_CFLAGS -Wimplicit-fallthrough=0"], + [], [-Werror]) +AX_CHECK_COMPILE_FLAG([-Wformat-truncation=0], + [ODP_CFLAGS="$ODP_CFLAGS -Wformat-truncation=0"], + [], [-Werror]) +AX_CHECK_COMPILE_FLAG([-Wformat-overflow=0], + [ODP_CFLAGS="$ODP_CFLAGS -Wformat-overflow=0"], + [], [-Werror]) + # Extra flags for example to suppress certain warning types ODP_CFLAGS="$ODP_CFLAGS $ODP_CFLAGS_EXTRA" -######################################################################### -# Check if compiler supports cmpxchng16 -########################################################################## -if test "${CC}" != "gcc" -o ${CC_VERSION_MAJOR} -ge 5; then - my_save_cflags="$CFLAGS" - - CFLAGS=-mcx16 - AC_MSG_CHECKING([whether CC supports -mcx16]) - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([])], - [AC_MSG_RESULT([yes])] - [ODP_CFLAGS="$ODP_CFLAGS $CFLAGS"], - [AC_MSG_RESULT([no])] - ) - CFLAGS="$my_save_cflags" -fi +########################################################################## +# Check if compiler supports cmpxchng16 on x86-based architectures +########################################################################## +case "${host}" in + i?86? | x86*) + if test "${CC}" != "gcc" -o ${CC_VERSION_MAJOR} -ge 5; then + my_save_cflags="$CFLAGS" + + CFLAGS=-mcx16 + AC_MSG_CHECKING([whether CC supports -mcx16]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([])], + [AC_MSG_RESULT([yes])] + [ODP_CFLAGS="$ODP_CFLAGS $CFLAGS"], + [AC_MSG_RESULT([no])] + ) + CFLAGS="$my_save_cflags" + fi + ;; +esac ########################################################################## # Default include setup @@ -332,7 +363,7 @@ AM_CXXFLAGS="-std=c++11" AC_CONFIG_FILES([Makefile pkgconfig/libodp-dpdk.pc - pkgconfig/libodphelper-linux-generic.pc + pkgconfig/libodphelper.pc ]) AC_SEARCH_LIBS([timer_create],[rt posix4]) @@ -383,6 +414,7 @@ AC_MSG_RESULT([ static libraries: ${enable_static} shared libraries: ${enable_shared} ABI compatible: ${abi_compat} + Deprecated APIs: ${deprecated} cunit: ${cunit_support} test_vald: ${test_vald} test_perf: ${test_perf} diff --git a/doc/application-api-guide/api_guide_lines.dox b/doc/application-api-guide/api_guide_lines.dox index 394e95829..a6488c322 100644 --- a/doc/application-api-guide/api_guide_lines.dox +++ b/doc/application-api-guide/api_guide_lines.dox @@ -75,7 +75,7 @@ The former is a compile-time assertion and hence adds no additional path length. The latter is controlled by the ODP_NO_DEBUG compile-time switch and so is suitable for use in development/debug builds that can be compiled out for production use. Other mechanisms available to the implementer are: - ODP_ABORT() is provided for situations where further execution of the code must not occur and a level of tracing information should be left in the log. - - ODP_DEPRECATED() is used to signify that a call is planned for obsolescence. + - ODP_DEPRECATE() is used to signify that a call is planned for obsolescence. - ODP_LOG() is used to direct implementation messages to the application. @@ -197,8 +197,8 @@ This is one of the reasons why some features MAY be defined as OPTIONAL. While allowed, the proliferation of OPTIONAL features SHOULD be avoided to enable broad application portability across many implementations. At the same time, a "least common denominator" approach MUST NOT be taken as that defeats the purpose of providing higher-level abstractions in APIs. -@subsection odp_deprecated ODP DEPRECATED -A deprecated API will remain marked as such in the public API using #ODP_DEPRECATED for two release cycles for the #ODP_VERSION_API_MAJOR number. +@subsection odp_deprecate ODP DEPRECATE +A deprecated API will remain marked as such in the public API using #ODP_DEPRECATE() for two release cycles for the #ODP_VERSION_API_MAJOR number. For example an API marked as deprecated in 1.1.0 will still be present in 1.2.0 and removed in 1.3.0. A deprecated API will contain the doxygen tag \@deprecated with a description of the reason for the change. diff --git a/doc/platform-api-guide/Doxyfile b/doc/platform-api-guide/Doxyfile index 93f6edfe3..1f2d49a4b 100644 --- a/doc/platform-api-guide/Doxyfile +++ b/doc/platform-api-guide/Doxyfile @@ -5,9 +5,9 @@ PROJECT_NUMBER = $(VERSION) PROJECT_LOGO = $(SRCDIR)/doc/images/ODP-Logo-HQ.svg INPUT = $(SRCDIR)/doc/application-api-guide \ $(SRCDIR)/doc/platform-api-guide \ - $(SRCDIR)/include \ + $(SRCDIR)/include/odp/api \ $(SRCDIR)/platform/$(WITH_PLATFORM)/doc \ - $(SRCDIR)/platform/$(WITH_PLATFORM)/include/odp \ + $(SRCDIR)/platform/$(WITH_PLATFORM)/include/odp/api \ $(SRCDIR)/platform/$(WITH_PLATFORM)/arch/$(WITH_ARCH) EXAMPLE_PATH = $(SRCDIR)/example $(SRCDIR)/platform $(SRCDIR) PREDEFINED = __GNUC__ \ @@ -17,4 +17,5 @@ PREDEFINED = __GNUC__ \ __LITTLE_ENDIAN_BITFIELD \ __x86_64__ \ ODP_PACKED \ + ODP_DEPRECATE(x)=x \ "ODP_HANDLE_T(type)=odp_handle_t type" diff --git a/doc/process-guide/release-guide.adoc b/doc/process-guide/release-guide.adoc index 8ea147af8..595af91a4 100644 --- a/doc/process-guide/release-guide.adoc +++ b/doc/process-guide/release-guide.adoc @@ -251,7 +251,7 @@ Deleting or changing the published API follows the normal <<anchor-1,process>>, * A deprecated indication is applied to the old API using the @deprecated doxygen syntax. * For a function change the old API it is additionally marked using the -ODP_DEPRECATED preprocessor macro. +ODP_DEPRECATE() preprocessor macro. * The CHANGELOG will have an entry in the API change section. * The Release Manager will resolve the duration for which the deprecated API. will be supported, and determine which future release it will be applied to. + @@ -275,7 +275,7 @@ The new API must have comparable coverage to the old API. * * @param name ... */ -odp_foo_t odp_foo_create(const char *name) ODP_DEPRECATED; +odp_foo_t ODP_DEPRECATE(odp_foo_create)(const char *name); /** * Create a bar @@ -298,7 +298,7 @@ compiler warning. * * @param name ... */ -odp_foo_t odp_foo_create(const char *name) ODP_DEPRECATED; +odp_foo_t ODP_DEPRECATE(odp_foo_create)(const char *name); ---- === Changing a struct member diff --git a/example/generator/odp_generator.c b/example/generator/odp_generator.c index 8062d8733..9336cec16 100644 --- a/example/generator/odp_generator.c +++ b/example/generator/odp_generator.c @@ -26,6 +26,8 @@ #define POOL_NUM_PKT 2048 /* Number of packets in packet pool */ #define POOL_PKT_LEN 1856 /* Max packet length */ #define DEFAULT_PKT_INTERVAL 1000 /* Interval between each packet */ +#define MAX_UDP_TX_BURST 32 +#define MAX_RX_BURST 32 #define APPL_MODE_UDP 0 /**< UDP mode */ #define APPL_MODE_PING 1 /**< ping mode */ @@ -37,6 +39,17 @@ /** Get rid of path in filename - only for unix-type paths using '/' */ #define NO_PATH(file_name) (strrchr((file_name), '/') ? \ strrchr((file_name), '/') + 1 : (file_name)) + +/** + * Interfaces + */ + +typedef struct { + odp_pktio_t pktio; + odp_pktout_queue_t pktout[MAX_WORKERS]; + unsigned pktout_count; +} interface_t; + /** * Parsed command line application arguments */ @@ -51,12 +64,16 @@ typedef struct { odph_ethaddr_t dstmac; /**< dest mac addr */ unsigned int srcip; /**< src ip addr */ unsigned int dstip; /**< dest ip addr */ + uint16_t srcport; /**< src udp port */ + uint16_t dstport; /**< dest udp port */ int mode; /**< work mode */ int number; /**< packets number to be sent */ int payload; /**< data len */ int timeout; /**< wait time */ int interval; /**< wait interval ms between sending each packet */ + int udp_tx_burst; /**< number of udp packets to send with one + API call */ } appl_args_t; /** @@ -74,7 +91,7 @@ static struct { /** * Thread specific arguments */ typedef struct { - char *pktio_dev; /**< Interface name to use */ + odp_pktout_queue_t pktout; /**< Packet output queue to use*/ odp_pool_t pool; /**< Pool for packet IO */ odp_timer_pool_t tp; /**< Timer pool handle */ odp_queue_t tq; /**< Queue for timeouts */ @@ -104,7 +121,6 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args); static void print_info(char *progname, appl_args_t *appl_args); static void usage(char *progname); static int scan_ip(char *buf, unsigned int *paddr); -static void tv_sub(struct timeval *recvtime, struct timeval *sendtime); static void print_global_stats(int num_workers); /** @@ -170,21 +186,20 @@ static int scan_ip(char *buf, unsigned int *paddr) } /** - * set up an udp packet + * set up an udp packet reference * * @param pool Buffer pool to create packet in * * @return Handle of created packet * @retval ODP_PACKET_INVALID Packet could not be created */ -static odp_packet_t pack_udp_pkt(odp_pool_t pool) +static odp_packet_t setup_udp_pkt_ref(odp_pool_t pool) { odp_packet_t pkt; char *buf; odph_ethhdr_t *eth; odph_ipv4hdr_t *ip; odph_udphdr_t *udp; - unsigned short seq; pkt = odp_packet_alloc(pool, args->appl.payload + ODPH_UDPHDR_LEN + ODPH_IPV4HDR_LEN + ODPH_ETHHDR_LEN); @@ -200,8 +215,10 @@ static odp_packet_t pack_udp_pkt(odp_pool_t pool) memcpy((char *)eth->src.addr, args->appl.srcmac.addr, ODPH_ETHADDR_LEN); memcpy((char *)eth->dst.addr, args->appl.dstmac.addr, ODPH_ETHADDR_LEN); eth->type = odp_cpu_to_be_16(ODPH_ETHTYPE_IPV4); + /* ip */ odp_packet_l3_offset_set(pkt, ODPH_ETHHDR_LEN); + odp_packet_has_ipv4_set(pkt, 1); ip = (odph_ipv4hdr_t *)(buf + ODPH_ETHHDR_LEN); ip->dst_addr = odp_cpu_to_be_32(args->appl.dstip); ip->src_addr = odp_cpu_to_be_32(args->appl.srcip); @@ -209,15 +226,16 @@ static odp_packet_t pack_udp_pkt(odp_pool_t pool) ip->tot_len = odp_cpu_to_be_16(args->appl.payload + ODPH_UDPHDR_LEN + ODPH_IPV4HDR_LEN); ip->proto = ODPH_IPPROTO_UDP; - seq = odp_atomic_fetch_add_u64(&counters.seq, 1) % 0xFFFF; - ip->id = odp_cpu_to_be_16(seq); + ip->id = 0; + ip->ttl = 64; ip->chksum = 0; - odph_ipv4_csum_update(pkt); + /* udp */ odp_packet_l4_offset_set(pkt, ODPH_ETHHDR_LEN + ODPH_IPV4HDR_LEN); + odp_packet_has_udp_set(pkt, 1); udp = (odph_udphdr_t *)(buf + ODPH_ETHHDR_LEN + ODPH_IPV4HDR_LEN); - udp->src_port = 0; - udp->dst_port = 0; + udp->src_port = odp_cpu_to_be_16(args->appl.srcport); + udp->dst_port = odp_cpu_to_be_16(args->appl.dstport); udp->length = odp_cpu_to_be_16(args->appl.payload + ODPH_UDPHDR_LEN); udp->chksum = 0; udp->chksum = odph_ipv4_udp_chksum(pkt); @@ -226,27 +244,60 @@ static odp_packet_t pack_udp_pkt(odp_pool_t pool) } /** - * Set up an icmp packet + * set up an udp packet * * @param pool Buffer pool to create packet in + * @param pkt_ref Reference UDP packet * * @return Handle of created packet * @retval ODP_PACKET_INVALID Packet could not be created */ -static odp_packet_t pack_icmp_pkt(odp_pool_t pool) +static odp_packet_t pack_udp_pkt(odp_pool_t pool, odp_packet_t pkt_ref) +{ + odp_packet_t pkt; + char *buf; + odph_ipv4hdr_t *ip; + unsigned short seq; + + pkt = odp_packet_alloc(pool, args->appl.payload + ODPH_UDPHDR_LEN + + ODPH_IPV4HDR_LEN + ODPH_ETHHDR_LEN); + + if (pkt == ODP_PACKET_INVALID) + return pkt; + + buf = (char *)odp_packet_data(pkt); + odp_memcpy(buf, odp_packet_data(pkt_ref), + args->appl.payload + ODPH_UDPHDR_LEN + + ODPH_IPV4HDR_LEN + ODPH_ETHHDR_LEN); + + /*Update IP ID and checksum*/ + ip = (odph_ipv4hdr_t *)(buf + ODPH_ETHHDR_LEN); + seq = odp_atomic_fetch_add_u64(&counters.seq, 1) % 0xFFFF; + ip->id = odp_cpu_to_be_16(seq); + ip->chksum = odph_chksum(ip, ODPH_IPV4HDR_LEN); + + return pkt; +} + +/** + * Set up an icmp packet reference + * + * @param pool Buffer pool to create packet in + * + * @return Handle of created packet + * @retval ODP_PACKET_INVALID Packet could not be created + */ +static odp_packet_t setup_icmp_pkt_ref(odp_pool_t pool) { odp_packet_t pkt; char *buf; odph_ethhdr_t *eth; odph_ipv4hdr_t *ip; odph_icmphdr_t *icmp; - struct timeval tval; - uint8_t *tval_d; - unsigned short seq; args->appl.payload = 56; pkt = odp_packet_alloc(pool, args->appl.payload + ODPH_ICMPHDR_LEN + - ODPH_IPV4HDR_LEN + ODPH_ETHHDR_LEN); + ODPH_IPV4HDR_LEN + ODPH_ETHHDR_LEN); if (pkt == ODP_PACKET_INVALID) return pkt; @@ -265,25 +316,69 @@ static odp_packet_t pack_icmp_pkt(odp_pool_t pool) ip->dst_addr = odp_cpu_to_be_32(args->appl.dstip); ip->src_addr = odp_cpu_to_be_32(args->appl.srcip); ip->ver_ihl = ODPH_IPV4 << 4 | ODPH_IPV4HDR_IHL_MIN; + ip->ttl = 64; ip->tot_len = odp_cpu_to_be_16(args->appl.payload + ODPH_ICMPHDR_LEN + ODPH_IPV4HDR_LEN); - ip->proto = ODPH_IPPROTO_ICMP; - seq = odp_atomic_fetch_add_u64(&counters.seq, 1) % 0xffff; - ip->id = odp_cpu_to_be_16(seq); + ip->proto = ODPH_IPPROTO_ICMPv4; + ip->id = 0; ip->chksum = 0; - odph_ipv4_csum_update(pkt); + /* icmp */ icmp = (odph_icmphdr_t *)(buf + ODPH_ETHHDR_LEN + ODPH_IPV4HDR_LEN); icmp->type = ICMP_ECHO; icmp->code = 0; icmp->un.echo.id = 0; + icmp->un.echo.sequence = 0; + icmp->chksum = 0; + + return pkt; +} + +/** + * Set up an icmp packet + * + * @param pool Buffer pool to create packet in + * @param pkt_ref Reference ICMP packet + * + * @return Handle of created packet + * @retval ODP_PACKET_INVALID Packet could not be created + */ +static odp_packet_t pack_icmp_pkt(odp_pool_t pool, odp_packet_t pkt_ref) +{ + odp_packet_t pkt; + char *buf; + odph_ipv4hdr_t *ip; + odph_icmphdr_t *icmp; + uint64_t tval; + uint8_t *tval_d; + unsigned short seq; + + pkt = odp_packet_alloc(pool, args->appl.payload + ODPH_ICMPHDR_LEN + + ODPH_IPV4HDR_LEN + ODPH_ETHHDR_LEN); + + if (pkt == ODP_PACKET_INVALID) + return pkt; + + buf = (char *)odp_packet_data(pkt); + odp_memcpy(buf, odp_packet_data(pkt_ref), + args->appl.payload + ODPH_ICMPHDR_LEN + + ODPH_IPV4HDR_LEN + ODPH_ETHHDR_LEN); + + /* ip */ + ip = (odph_ipv4hdr_t *)(buf + ODPH_ETHHDR_LEN); + seq = odp_atomic_fetch_add_u64(&counters.seq, 1) % 0xffff; + ip->id = odp_cpu_to_be_16(seq); + ip->chksum = odph_chksum(ip, ODPH_IPV4HDR_LEN); + + /* icmp */ + icmp = (odph_icmphdr_t *)(buf + ODPH_ETHHDR_LEN + ODPH_IPV4HDR_LEN); icmp->un.echo.sequence = ip->id; + tval_d = (uint8_t *)(buf + ODPH_ETHHDR_LEN + ODPH_IPV4HDR_LEN + ODPH_ICMPHDR_LEN); - /* TODO This should be changed to use an - * ODP timer API once one exists. */ - gettimeofday(&tval, NULL); - memcpy(tval_d, &tval, sizeof(struct timeval)); + tval = odp_time_to_ns(odp_time_local()); + memcpy(tval_d, &tval, sizeof(uint64_t)); + icmp->chksum = 0; icmp->chksum = odph_chksum(icmp, args->appl.payload + ODPH_ICMPHDR_LEN); @@ -299,48 +394,80 @@ static odp_packet_t pack_icmp_pkt(odp_pool_t pool) * @return The handle of the created pktio object. * @warning This routine aborts if the create is unsuccessful. */ -static odp_pktio_t create_pktio(const char *dev, odp_pool_t pool) +static int create_pktio(const char *dev, odp_pool_t pool, + unsigned num_rx_queues, + unsigned num_tx_queues, + interface_t *itf) { - odp_pktio_t pktio; + odp_pktio_capability_t capa; int ret; odp_pktio_param_t pktio_param; odp_pktin_queue_param_t pktin_param; + odp_pktout_queue_param_t pktout_param; + odp_pktio_op_mode_t pktout_mode; odp_pktio_param_init(&pktio_param); pktio_param.in_mode = ODP_PKTIN_MODE_SCHED; /* Open a packet IO instance */ - pktio = odp_pktio_open(dev, pool, &pktio_param); + itf->pktio = odp_pktio_open(dev, pool, &pktio_param); - if (pktio == ODP_PKTIO_INVALID) { + if (itf->pktio == ODP_PKTIO_INVALID) { EXAMPLE_ERR("Error: pktio create failed for %s\n", dev); - exit(EXIT_FAILURE); + return -1; + } + + if (odp_pktio_capability(itf->pktio, &capa)) { + EXAMPLE_ERR("Error: Failed to get interface capabilities %s\n", + dev); + return -1; } + if (num_rx_queues > capa.max_input_queues) + num_rx_queues = capa.max_input_queues; odp_pktin_queue_param_init(&pktin_param); + pktin_param.num_queues = num_rx_queues; pktin_param.queue_param.sched.sync = ODP_SCHED_SYNC_ATOMIC; - if (odp_pktin_queue_config(pktio, &pktin_param)) { + if (odp_pktin_queue_config(itf->pktio, &pktin_param)) { EXAMPLE_ERR("Error: pktin queue config failed for %s\n", dev); - exit(EXIT_FAILURE); + return -1; } - if (odp_pktout_queue_config(pktio, NULL)) { + pktout_mode = ODP_PKTIO_OP_MT_UNSAFE; + if (num_tx_queues > capa.max_output_queues) { + num_tx_queues = capa.max_output_queues; + pktout_mode = ODP_PKTIO_OP_MT; + } + + odp_pktout_queue_param_init(&pktout_param); + pktout_param.num_queues = num_tx_queues; + pktout_param.op_mode = pktout_mode; + + if (odp_pktout_queue_config(itf->pktio, &pktout_param)) { EXAMPLE_ERR("Error: pktout queue config failed for %s\n", dev); - exit(EXIT_FAILURE); + return -1; } - ret = odp_pktio_start(pktio); + ret = odp_pktio_start(itf->pktio); if (ret) EXAMPLE_ABORT("Error: unable to start %s\n", dev); + itf->pktout_count = num_tx_queues; + if (odp_pktout_queue(itf->pktio, itf->pktout, itf->pktout_count) != + (int)itf->pktout_count) { + EXAMPLE_ERR("Error: failed to get output queues for %s\n", dev); + return -1; + } + printf(" created pktio:%02" PRIu64 ", dev:%s, queue mode (ATOMIC queues)\n" " default pktio%02" PRIu64 "\n", - odp_pktio_to_u64(pktio), dev, - odp_pktio_to_u64(pktio)); + odp_pktio_to_u64(itf->pktio), dev, + odp_pktio_to_u64(itf->pktio)); + fflush(NULL); - return pktio; + return 0; } /** @@ -352,24 +479,33 @@ static odp_pktio_t create_pktio(const char *dev, odp_pool_t pool) static int gen_send_thread(void *arg) { int thr; - int ret; - odp_pktio_t pktio; + int ret, i; thread_args_t *thr_args; odp_pktout_queue_t pktout; - odp_packet_t pkt; + odp_packet_t pkt_array[MAX_UDP_TX_BURST]; + int pkt_array_size; + int burst_start, burst_size; + odp_packet_t pkt_ref = ODP_PACKET_INVALID; thr = odp_thread_id(); thr_args = arg; - pktio = odp_pktio_lookup(thr_args->pktio_dev); - if (pktio == ODP_PKTIO_INVALID) { - EXAMPLE_ERR(" [%02i] Error: lookup of pktio %s failed\n", - thr, thr_args->pktio_dev); + pktout = thr_args->pktout; + + if (args->appl.mode == APPL_MODE_UDP) { + pkt_ref = setup_udp_pkt_ref(thr_args->pool); + pkt_array_size = args->appl.udp_tx_burst; + } else if (args->appl.mode == APPL_MODE_PING) { + pkt_ref = setup_icmp_pkt_ref(thr_args->pool); + pkt_array_size = 1; + } else { + EXAMPLE_ERR(" [%02i] Error: invalid processing mode %d\n", + thr, args->appl.mode); return -1; } - - if (odp_pktout_queue(pktio, &pktout, 1) != 1) { - EXAMPLE_ERR(" [%02i] Error: no output queue\n", thr); + if (pkt_ref == ODP_PACKET_INVALID) { + EXAMPLE_ERR(" [%2i] Error: reference packet creation failed\n", + thr); return -1; } @@ -379,36 +515,50 @@ static int gen_send_thread(void *arg) for (;;) { if (args->appl.number != -1 && - odp_atomic_fetch_add_u64(&counters.cnt, 1) >= - (unsigned int)args->appl.number) + odp_atomic_fetch_add_u64(&counters.cnt, pkt_array_size) >= + (unsigned int)args->appl.number) break; - pkt = ODP_PACKET_INVALID; - - if (args->appl.mode == APPL_MODE_UDP) - pkt = pack_udp_pkt(thr_args->pool); - else if (args->appl.mode == APPL_MODE_PING) - pkt = pack_icmp_pkt(thr_args->pool); - - if (pkt == ODP_PACKET_INVALID) { - /* Thread gives up as soon as it sees the pool empty. - * Depending on pool size and transmit latency, it may - * be normal that pool gets empty sometimes. */ - EXAMPLE_ERR(" [%2i] alloc_single failed\n", thr); + if (args->appl.mode == APPL_MODE_UDP) { + for (i = 0; i < pkt_array_size; i++) { + pkt_array[i] = pack_udp_pkt(thr_args->pool, + pkt_ref); + if (!odp_packet_is_valid(pkt_array[i])) + break; + } + if (i != pkt_array_size) { + EXAMPLE_ERR(" [%2i] alloc_multi failed\n", + thr); + odp_packet_free_multi(pkt_array, i); + break; + } + } else if (args->appl.mode == APPL_MODE_PING) { + pkt_array[0] = pack_icmp_pkt(thr_args->pool, pkt_ref); + if (!odp_packet_is_valid(pkt_array[0])) { + EXAMPLE_ERR(" [%2i] alloc_single failed\n", + thr); + break; + } + } else { break; } - for (;;) { - ret = odp_pktout_send(pktout, &pkt, 1); - if (ret == 1) { + for (burst_start = 0, burst_size = pkt_array_size;;) { + ret = odp_pktout_send(pktout, &pkt_array[burst_start], + burst_size); + if (ret == burst_size) { break; - } else if (ret == 0) { - odp_atomic_add_u64(&counters.tx_drops, 1); + } else if (ret >= 0 && ret < burst_size) { + odp_atomic_add_u64(&counters.tx_drops, + burst_size - ret); + burst_start += ret; + burst_size -= ret; odp_time_wait_ns(ODP_TIME_MSEC_IN_NS); continue; } EXAMPLE_ERR(" [%02i] packet send failed\n", thr); - odp_packet_free(pkt); + odp_packet_free_multi(&pkt_array[burst_start], + burst_size); break; } @@ -440,11 +590,46 @@ static int gen_send_thread(void *arg) args->appl.timeout--; } } + odp_packet_free(pkt_ref); return 0; } /** + * Process icmp packets + * + * @param icmp icmp header address + * @param msg output buffer + */ + +static void process_icmp_pkt(odph_icmphdr_t *icmp, char *msg) +{ + uint64_t trecv; + uint64_t tsend; + uint64_t rtt_ms, rtt_us; + + msg[0] = 0; + + if (icmp->type == ICMP_ECHOREPLY) { + odp_atomic_inc_u64(&counters.icmp); + + memcpy(&tsend, (uint8_t *)icmp + ODPH_ICMPHDR_LEN, + sizeof(uint64_t)); + trecv = odp_time_to_ns(odp_time_local()); + rtt_ms = (trecv - tsend) / ODP_TIME_MSEC_IN_NS; + rtt_us = (trecv - tsend) / ODP_TIME_USEC_IN_NS - + 1000 * rtt_ms; + sprintf(msg, + "ICMP Echo Reply seq %d time %" + PRIu64 ".%.03" PRIu64" ms", + odp_be_to_cpu_16(icmp->un.echo.sequence), + rtt_ms, rtt_us); + } else if (icmp->type == ICMP_ECHO) { + sprintf(msg, "Icmp Echo Request"); + } +} + +/** * Print odp packets * * @param thr worker id @@ -457,16 +642,12 @@ static void print_pkts(int thr, odp_packet_t pkt_tbl[], unsigned len) char *buf; odph_ipv4hdr_t *ip; odph_icmphdr_t *icmp; - struct timeval tvrecv; - struct timeval tvsend; - double rtt; unsigned i; size_t offset; char msg[1024]; - int rlen; + for (i = 0; i < len; ++i) { pkt = pkt_tbl[i]; - rlen = 0; /* only ip pkts */ if (!odp_packet_has_ipv4(pkt)) @@ -483,28 +664,10 @@ static void print_pkts(int thr, odp_packet_t pkt_tbl[], unsigned len) } /* icmp */ - if (ip->proto == ODPH_IPPROTO_ICMP) { + if (ip->proto == ODPH_IPPROTO_ICMPv4) { icmp = (odph_icmphdr_t *)(buf + offset); - /* echo reply */ - if (icmp->type == ICMP_ECHOREPLY) { - odp_atomic_inc_u64(&counters.icmp); - memcpy(&tvsend, buf + offset + ODPH_ICMPHDR_LEN, - sizeof(struct timeval)); - /* TODO This should be changed to use an - * ODP timer API once one exists. */ - gettimeofday(&tvrecv, NULL); - tv_sub(&tvrecv, &tvsend); - rtt = tvrecv.tv_sec*1000 + tvrecv.tv_usec/1000; - rlen += sprintf(msg + rlen, - "ICMP Echo Reply seq %d time %.1f ", - odp_be_to_cpu_16(icmp->un.echo.sequence) - , rtt); - } else if (icmp->type == ICMP_ECHO) { - rlen += sprintf(msg + rlen, - "Icmp Echo Request"); - } - msg[rlen] = '\0'; + process_icmp_pkt(icmp, msg); printf(" [%02i] %s\n", thr, msg); } } @@ -518,20 +681,12 @@ static void print_pkts(int thr, odp_packet_t pkt_tbl[], unsigned len) static int gen_recv_thread(void *arg) { int thr; - odp_pktio_t pktio; - thread_args_t *thr_args; - odp_packet_t pkt; - odp_event_t ev; + odp_packet_t pkts[MAX_RX_BURST], pkt; + odp_event_t events[MAX_RX_BURST]; + int pkt_cnt, ev_cnt, i; thr = odp_thread_id(); - thr_args = arg; - - pktio = odp_pktio_lookup(thr_args->pktio_dev); - if (pktio == ODP_PKTIO_INVALID) { - EXAMPLE_ERR(" [%02i] Error: lookup of pktio %s failed\n", - thr, thr_args->pktio_dev); - return -1; - } + (void)arg; printf(" [%02i] created mode: RECEIVE\n", thr); odp_barrier_wait(&barrier); @@ -544,18 +699,24 @@ static int gen_recv_thread(void *arg) } /* Use schedule to get buf from any input queue */ - ev = odp_schedule(NULL, ODP_SCHED_WAIT); - - pkt = odp_packet_from_event(ev); - /* Drop packets with errors */ - if (odp_unlikely(odp_packet_has_error(pkt))) { - odp_packet_free(pkt); + ev_cnt = odp_schedule_multi(NULL, ODP_SCHED_WAIT, + events, MAX_RX_BURST); + if (ev_cnt == 0) continue; + for (i = 0, pkt_cnt = 0; i < ev_cnt; i++) { + pkt = odp_packet_from_event(events[i]); + + /* Drop packets with errors */ + if (odp_unlikely(odp_packet_has_error(pkt))) { + odp_packet_free(pkt); + continue; + } + pkts[pkt_cnt++] = pkt; } - print_pkts(thr, &pkt, 1); + print_pkts(thr, pkts, pkt_cnt); - odp_packet_free(pkt); + odp_packet_free_multi(pkts, pkt_cnt); } return 0; @@ -568,7 +729,10 @@ static int gen_recv_thread(void *arg) static void print_global_stats(int num_workers) { odp_time_t cur, wait, next; - uint64_t pkts, pkts_prev = 0, pps, maximum_pps = 0; + uint64_t pkts_snd = 0, pkts_snd_prev = 0; + uint64_t pps_snd = 0, maximum_pps_snd = 0; + uint64_t pkts_rcv = 0, pkts_rcv_prev = 0; + uint64_t pps_rcv = 0, maximum_pps_rcv = 0; int verbose_interval = 20; odp_thrmask_t thrd_mask; @@ -589,30 +753,41 @@ static void print_global_stats(int num_workers) continue; next = odp_time_sum(cur, wait); - - if (args->appl.mode == APPL_MODE_RCV) { - pkts = odp_atomic_load_u64(&counters.udp); - printf(" total receive(UDP: %" PRIu64 ")\n", pkts); + switch (args->appl.mode) { + case APPL_MODE_RCV: + pkts_rcv = odp_atomic_load_u64(&counters.ip); + break; + case APPL_MODE_PING: + pkts_snd = odp_atomic_load_u64(&counters.seq); + pkts_rcv = odp_atomic_load_u64(&counters.icmp); + break; + case APPL_MODE_UDP: + pkts_snd = odp_atomic_load_u64(&counters.seq); + break; + default: continue; } - if (args->appl.mode == APPL_MODE_PING) { - pkts = odp_atomic_load_u64(&counters.icmp); - printf(" total receive(ICMP: %" PRIu64 ")\n", pkts); - } - - pkts = odp_atomic_load_u64(&counters.seq); - printf(" total sent: %" PRIu64 ", drops: %" PRIu64 "\n", pkts, - odp_atomic_load_u64(&counters.tx_drops)); - - if (args->appl.mode == APPL_MODE_UDP) { - pps = (pkts - pkts_prev) / verbose_interval; - if (pps > maximum_pps) - maximum_pps = pps; - printf(" %" PRIu64 " pps, %" PRIu64 " max pps\n", - pps, maximum_pps); - pkts_prev = pkts; - } + pps_snd = (pkts_snd - pkts_snd_prev) / verbose_interval; + pkts_snd_prev = pkts_snd; + if (pps_snd > maximum_pps_snd) + maximum_pps_snd = pps_snd; + + pps_rcv = (pkts_rcv - pkts_rcv_prev) / verbose_interval; + pkts_rcv_prev = pkts_rcv; + if (pps_rcv > maximum_pps_rcv) + maximum_pps_rcv = pps_rcv; + + printf("sent: %" PRIu64 ", drops: %" PRIu64 ", " + "send rate: %" PRIu64 " pps, " + "max send rate: %" PRIu64 " pps, " + "rcv: %" PRIu64 ", " + "recv rate: %" PRIu64 " pps, " + "max recv rate: %" PRIu64 " pps\n", + pkts_snd, odp_atomic_load_u64(&counters.tx_drops), + pps_snd, maximum_pps_snd, + pkts_rcv, pps_rcv, maximum_pps_rcv); + fflush(NULL); } } @@ -624,6 +799,7 @@ int main(int argc, char *argv[]) odph_odpthread_t thread_tbl[MAX_WORKERS]; odp_pool_t pool; int num_workers; + unsigned num_rx_queues, num_tx_queues; int i; odp_shm_t shm; odp_cpumask_t cpumask; @@ -634,7 +810,7 @@ int main(int argc, char *argv[]) odp_pool_t tmop; odp_queue_t tq; odp_event_t ev; - odp_pktio_t *pktio; + interface_t *ifs; odp_instance_t instance; odph_odpthread_params_t thr_params; @@ -694,6 +870,7 @@ int main(int argc, char *argv[]) printf("num worker threads: %i\n", num_workers); printf("first CPU: %i\n", odp_cpumask_first(&cpumask)); printf("cpu mask: %s\n", cpumaskstr); + fflush(NULL); /* ping mode need two workers */ if (args->appl.mode == APPL_MODE_PING) { @@ -745,10 +922,30 @@ int main(int argc, char *argv[]) exit(EXIT_FAILURE); } - pktio = malloc(sizeof(odp_pktio_t) * args->appl.if_count); + ifs = malloc(sizeof(interface_t) * args->appl.if_count); + + if (args->appl.mode == APPL_MODE_PING || + args->appl.mode == APPL_MODE_UDP) + num_rx_queues = 1; + else + num_rx_queues = num_workers; + + if (args->appl.mode == APPL_MODE_PING || + args->appl.mode == APPL_MODE_RCV) + num_tx_queues = 1; + else { + num_tx_queues = num_workers / args->appl.if_count; + if (num_workers % args->appl.if_count) + num_tx_queues++; + } for (i = 0; i < args->appl.if_count; ++i) - pktio[i] = create_pktio(args->appl.if_names[i], pool); + if (create_pktio(args->appl.if_names[i], pool, num_rx_queues, + num_tx_queues, &ifs[i])) { + EXAMPLE_ERR("Error: create interface %s failed.\n", + args->appl.if_names[i]); + exit(EXIT_FAILURE); + } /* Create and init worker threads */ memset(thread_tbl, 0, sizeof(thread_tbl)); @@ -774,7 +971,7 @@ int main(int argc, char *argv[]) EXAMPLE_ERR("queue_create failed\n"); abort(); } - args->thread[1].pktio_dev = args->appl.if_names[0]; + (void)args->thread[1].pktout; /* Not used*/ args->thread[1].pool = pool; args->thread[1].tp = tp; args->thread[1].tq = tq; @@ -803,7 +1000,7 @@ int main(int argc, char *argv[]) EXAMPLE_ERR("queue_create failed\n"); abort(); } - args->thread[0].pktio_dev = args->appl.if_names[0]; + args->thread[0].pktout = ifs[0].pktout[0]; args->thread[0].pool = pool; args->thread[0].tp = tp; args->thread[0].tq = tq; @@ -829,14 +1026,22 @@ int main(int argc, char *argv[]) } else { int cpu = odp_cpumask_first(&cpumask); + for (i = 0; i < num_workers; ++i) { odp_cpumask_t thd_mask; int (*thr_run_func)(void *); - int if_idx; + int if_idx, pktout_idx; - if_idx = i % args->appl.if_count; + if (args->appl.mode == APPL_MODE_RCV) + (void)args->thread[i].pktout; /*not used*/ + else { + if_idx = i % args->appl.if_count; + pktout_idx = (i / args->appl.if_count) % + ifs[if_idx].pktout_count; - args->thread[i].pktio_dev = args->appl.if_names[if_idx]; + args->thread[i].pktout = + ifs[if_idx].pktout[pktout_idx]; + } tq = odp_queue_create("", NULL); if (tq == ODP_QUEUE_INVALID) { EXAMPLE_ERR("queue_create failed\n"); @@ -890,7 +1095,7 @@ int main(int argc, char *argv[]) odph_odpthreads_join(&thread_tbl[i]); for (i = 0; i < args->appl.if_count; ++i) - odp_pktio_stop(pktio[i]); + odp_pktio_stop(ifs[i].pktio); for (i = 0; i < num_workers; ++i) { odp_timer_cancel(args->thread[i].tim, &ev); @@ -909,8 +1114,8 @@ int main(int argc, char *argv[]) } for (i = 0; i < args->appl.if_count; ++i) - odp_pktio_close(pktio[i]); - free(pktio); + odp_pktio_close(ifs[i].pktio); + free(ifs); free(args->appl.if_names); free(args->appl.if_str); if (0 != odp_pool_destroy(pool)) @@ -951,16 +1156,19 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) {"dstmac", required_argument, NULL, 'b'}, {"srcip", required_argument, NULL, 's'}, {"dstip", required_argument, NULL, 'd'}, + {"srcport", required_argument, NULL, 'e'}, + {"dstport", required_argument, NULL, 'f'}, {"packetsize", required_argument, NULL, 'p'}, {"mode", required_argument, NULL, 'm'}, {"count", required_argument, NULL, 'n'}, {"timeout", required_argument, NULL, 't'}, {"interval", required_argument, NULL, 'i'}, {"help", no_argument, NULL, 'h'}, + {"udp_tx_burst", required_argument, NULL, 'x'}, {NULL, 0, NULL, 0} }; - static const char *shortopts = "+I:a:b:s:d:p:i:m:n:t:w:c:h"; + static const char *shortopts = "+I:a:b:s:d:p:i:m:n:t:w:c:x:he:f:"; /* let helper collect its own arguments (e.g. --odph_proc) */ odph_parse_options(argc, argv, shortopts, longopts); @@ -970,6 +1178,9 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) appl_args->payload = 56; appl_args->timeout = -1; appl_args->interval = DEFAULT_PKT_INTERVAL; + appl_args->udp_tx_burst = 16; + appl_args->srcport = 0; + appl_args->dstport = 0; opterr = 0; /* do not issue errors on helper options */ @@ -1076,6 +1287,12 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) } break; + case 'e': + appl_args->srcport = (unsigned short)atoi(optarg); + break; + case 'f': + appl_args->dstport = (unsigned short)atoi(optarg); + break; case 'p': appl_args->payload = atoi(optarg); break; @@ -1095,6 +1312,14 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) exit(EXIT_FAILURE); } break; + case 'x': + appl_args->udp_tx_burst = atoi(optarg); + if (appl_args->udp_tx_burst > MAX_UDP_TX_BURST) { + EXAMPLE_ERR("wrong UDP Tx burst size (max %d)\n", + MAX_UDP_TX_BURST); + exit(EXIT_FAILURE); + } + break; case 'h': usage(argv[0]); @@ -1181,6 +1406,8 @@ static void usage(char *progname) "\n" "Optional OPTIONS\n" " -h, --help Display help and exit.\n" + " -e, --srcport src udp port\n" + " -f, --dstport dst udp port\n" " -p, --packetsize payload length of the packets\n" " -t, --timeout only for ping mode, wait ICMP reply timeout seconds\n" " -i, --interval wait interval ms between sending each packet\n" @@ -1189,24 +1416,7 @@ static void usage(char *progname) " default is to assign all\n" " -n, --count the number of packets to be send\n" " -c, --cpumask to set on cores\n" + " -x, --udp_tx_burst size of UDP TX burst\n" "\n", NO_PATH(progname), NO_PATH(progname) ); } -/** - * calc time period - * - *@param recvtime start time - *@param sendtime end time -*/ -static void tv_sub(struct timeval *recvtime, struct timeval *sendtime) -{ - long sec = recvtime->tv_sec - sendtime->tv_sec; - long usec = recvtime->tv_usec - sendtime->tv_usec; - if (usec >= 0) { - recvtime->tv_sec = sec; - recvtime->tv_usec = usec; - } else { - recvtime->tv_sec = sec - 1; - recvtime->tv_usec = -usec; - } -} diff --git a/example/ipsec/odp_ipsec_cache.c b/example/ipsec/odp_ipsec_cache.c index b2a91c242..dba0ea0ab 100644 --- a/example/ipsec/odp_ipsec_cache.c +++ b/example/ipsec/odp_ipsec_cache.c @@ -100,6 +100,7 @@ int create_ipsec_cache_entry(sa_db_entry_t *cipher_sa, params.auth_alg = auth_sa->alg.u.auth; params.auth_key.data = auth_sa->key.data; params.auth_key.length = auth_sa->key.length; + params.auth_digest_len = auth_sa->icv_len; mode = auth_sa->mode; } else { params.auth_alg = ODP_AUTH_ALG_NULL; diff --git a/example/ipsec/odp_ipsec_misc.h b/example/ipsec/odp_ipsec_misc.h index e0320eb4e..45cb022eb 100644 --- a/example/ipsec/odp_ipsec_misc.h +++ b/example/ipsec/odp_ipsec_misc.h @@ -98,10 +98,10 @@ int parse_key_string(char *keystring, key->length = key_bits_in / 8; } else { - if ((alg->u.auth == ODP_AUTH_ALG_MD5_96) && + if ((alg->u.auth == ODP_AUTH_ALG_MD5_HMAC) && (KEY_BITS_MD5_96 == key_bits_in)) key->length = key_bits_in / 8; - else if ((alg->u.auth == ODP_AUTH_ALG_SHA256_128) && + else if ((alg->u.auth == ODP_AUTH_ALG_SHA256_HMAC) && (KEY_BITS_SHA256_128 == key_bits_in)) key->length = key_bits_in / 8; } diff --git a/example/ipsec/odp_ipsec_sa_db.c b/example/ipsec/odp_ipsec_sa_db.c index 28215b5b7..10bbcb8f2 100644 --- a/example/ipsec/odp_ipsec_sa_db.c +++ b/example/ipsec/odp_ipsec_sa_db.c @@ -111,11 +111,11 @@ int create_sa_db_entry(char *input, odp_bool_t cipher) } else { if (0 == strcmp(token, "md5")) { entry->alg.u.auth = - ODP_AUTH_ALG_MD5_96; + ODP_AUTH_ALG_MD5_HMAC; entry->icv_len = 12; } else if (!strcmp(token, "sha256")) { entry->alg.u.auth = - ODP_AUTH_ALG_SHA256_128; + ODP_AUTH_ALG_SHA256_HMAC; entry->icv_len = 16; } else { entry->alg.u.auth = ODP_AUTH_ALG_NULL; diff --git a/example/ipsec/odp_ipsec_stream.c b/example/ipsec/odp_ipsec_stream.c index 428ec04da..e37fbee29 100644 --- a/example/ipsec/odp_ipsec_stream.c +++ b/example/ipsec/odp_ipsec_stream.c @@ -219,7 +219,7 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream, ip->src_addr = odp_cpu_to_be_32(entry->tun_src_ip); ip->dst_addr = odp_cpu_to_be_32(entry->tun_dst_ip); } else { - ip->proto = ODPH_IPPROTO_ICMP; + ip->proto = ODPH_IPPROTO_ICMPv4; ip->src_addr = odp_cpu_to_be_32(stream->src_ip); ip->dst_addr = odp_cpu_to_be_32(stream->dst_ip); } @@ -227,8 +227,8 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream, /* AH (if specified) */ if (entry && (entry == stream->input.entry) && (ODP_AUTH_ALG_NULL != entry->ah.alg)) { - if (entry->ah.alg != ODP_AUTH_ALG_MD5_96 && - entry->ah.alg != ODP_AUTH_ALG_SHA256_128) + if (entry->ah.alg != ODP_AUTH_ALG_MD5_HMAC && + entry->ah.alg != ODP_AUTH_ALG_SHA256_HMAC) abort(); ah = (odph_ahhdr_t *)data; @@ -262,7 +262,7 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream, inner_ip = (odph_ipv4hdr_t *)data; memset((char *)inner_ip, 0, sizeof(*inner_ip)); inner_ip->ver_ihl = 0x45; - inner_ip->proto = ODPH_IPPROTO_ICMP; + inner_ip->proto = ODPH_IPPROTO_ICMPv4; inner_ip->id = odp_cpu_to_be_16(stream->id); inner_ip->ttl = 64; inner_ip->tos = 0; @@ -424,7 +424,7 @@ odp_bool_t verify_ipv4_packet(stream_db_entry_t *stream, return FALSE; if (odp_be_to_cpu_32(ah->spi) != entry->ah.spi) return FALSE; - if (ODP_AUTH_ALG_MD5_96 != entry->ah.alg) + if (ODP_AUTH_ALG_MD5_HMAC != entry->ah.alg) abort(); } else { if (entry && (ODP_AUTH_ALG_NULL != entry->ah.alg)) @@ -519,7 +519,7 @@ clear_packet: icmp = (odph_icmphdr_t *)(inner_ip + 1); data = (uint8_t *)icmp; } else { - if (ODPH_IPPROTO_ICMP != ip->proto) + if (ODPH_IPPROTO_ICMPv4 != ip->proto) return FALSE; icmp = (odph_icmphdr_t *)data; } diff --git a/example/l2fwd_simple/odp_l2fwd_simple.c b/example/l2fwd_simple/odp_l2fwd_simple.c index b97808f4b..e63814555 100644 --- a/example/l2fwd_simple/odp_l2fwd_simple.c +++ b/example/l2fwd_simple/odp_l2fwd_simple.c @@ -41,6 +41,7 @@ static odp_pktio_t create_pktio(const char *name, odp_pool_t pool, odp_pktin_queue_param_t in_queue_param; odp_pktout_queue_param_t out_queue_param; odp_pktio_t pktio; + odp_pktio_config_t config; odp_pktio_param_init(&pktio_param); @@ -50,6 +51,10 @@ static odp_pktio_t create_pktio(const char *name, odp_pool_t pool, exit(1); } + odp_pktio_config_init(&config); + config.parser.layer = ODP_PKTIO_PARSER_LAYER_L2; + odp_pktio_config(pktio, &config); + odp_pktin_queue_param_init(&in_queue_param); odp_pktout_queue_param_init(&out_queue_param); diff --git a/example/l3fwd/odp_l3fwd.c b/example/l3fwd/odp_l3fwd.c index ba0697201..f579d36d3 100644 --- a/example/l3fwd/odp_l3fwd.c +++ b/example/l3fwd/odp_l3fwd.c @@ -101,6 +101,7 @@ static int create_pktio(const char *name, odp_pool_t pool, odp_pktio_param_t pktio_param; odp_pktio_t pktio; odp_pktio_capability_t capa; + odp_pktio_config_t config; int rc; odp_pktio_param_init(&pktio_param); @@ -120,6 +121,12 @@ static int create_pktio(const char *name, odp_pool_t pool, return -1; } + odp_pktio_config_init(&config); + config.parser.layer = global.cmd_args.error_check ? + ODP_PKTIO_PARSER_LAYER_ALL : + ODP_PKTIO_PARSER_LAYER_L4; + odp_pktio_config(pktio, &config); + fwd_pktio->nb_rxq = (int)capa.max_input_queues; fwd_pktio->nb_txq = (int)capa.max_output_queues; diff --git a/example/l3fwd/odp_l3fwd_db.c b/example/l3fwd/odp_l3fwd_db.c index 082b2c276..0670aa455 100644 --- a/example/l3fwd/odp_l3fwd_db.c +++ b/example/l3fwd/odp_l3fwd_db.c @@ -394,7 +394,10 @@ int create_fwd_db_entry(char *input, char **oif, uint8_t **dst_mac) *oif = entry->oif; break; case 2: - odph_eth_addr_parse(&entry->dst_mac, token); + if (odph_eth_addr_parse(&entry->dst_mac, token) < 0) { + free(local); + return -1; + } *dst_mac = entry->dst_mac.addr; break; diff --git a/example/switch/odp_switch.c b/example/switch/odp_switch.c index 561c2934b..5bec6a043 100644 --- a/example/switch/odp_switch.c +++ b/example/switch/odp_switch.c @@ -217,6 +217,7 @@ static int create_pktio(const char *dev, int idx, int num_rx, int num_tx, odp_pktio_t pktio; odp_pktio_param_t pktio_param; odp_pktio_capability_t capa; + odp_pktio_config_t config; odp_pktin_queue_param_t pktin_param; odp_pktout_queue_param_t pktout_param; odp_pktio_op_mode_t mode_rx; @@ -238,6 +239,10 @@ static int create_pktio(const char *dev, int idx, int num_rx, int num_tx, return -1; } + odp_pktio_config_init(&config); + config.parser.layer = ODP_PKTIO_PARSER_LAYER_L2; + odp_pktio_config(pktio, &config); + odp_pktin_queue_param_init(&pktin_param); odp_pktout_queue_param_init(&pktout_param); diff --git a/helper/Makefile.am b/helper/Makefile.am index 5757c7c81..2c5452dcc 100644 --- a/helper/Makefile.am +++ b/helper/Makefile.am @@ -1,7 +1,7 @@ include $(top_srcdir)/platform/@with_platform@/Makefile.inc pkgconfigdir = $(libdir)/pkgconfig -pkgconfig_DATA = $(top_builddir)/pkgconfig/libodphelper-linux-generic.pc +pkgconfig_DATA = $(top_builddir)/pkgconfig/libodphelper.pc LIB = $(top_builddir)/lib AM_CFLAGS += -I$(srcdir)/include @@ -32,6 +32,9 @@ helperinclude_HEADERS = \ $(srcdir)/include/odp/helper/udp.h if helper_linux +helperinclude_HEADERS += \ + $(srcdir)/include/odp/helper/linux.h + helperlinuxincludedir = $(includedir)/odp/helper/linux helperlinuxinclude_HEADERS = \ $(srcdir)/include/odp/helper/linux/pthread.h \ diff --git a/helper/chksum.c b/helper/chksum.c index f740618d3..ae70d97e6 100644 --- a/helper/chksum.c +++ b/helper/chksum.c @@ -128,7 +128,7 @@ static inline int odph_process_l4_hdr(odp_packet_t odp_pkt, * should come from the udp header, unlike for TCP where is * derived. */ l4_len = odp_be_to_cpu_16(udp_hdr_ptr->length); - pkt_chksum_ptr = &udp_hdr_ptr->chksum; + pkt_chksum_ptr = (uint16_t *)(void *)&udp_hdr_ptr->chksum; pkt_chksum_offset = l4_offset + offsetof(odph_udphdr_t, chksum); } else if (odp_packet_has_tcp(odp_pkt)) { tcp_hdr_ptr = (odph_tcphdr_t *)l4_ptr; @@ -139,7 +139,7 @@ static inline int odph_process_l4_hdr(odp_packet_t odp_pkt, ODPH_TCPHDR_LEN, tcp_hdr_ptr); } - pkt_chksum_ptr = &tcp_hdr_ptr->cksm; + pkt_chksum_ptr = (uint16_t *)(void *)&tcp_hdr_ptr->cksm; pkt_chksum_offset = l4_offset + offsetof(odph_tcphdr_t, cksm); is_tcp = true; } else { @@ -203,7 +203,7 @@ static inline int odph_process_l3_hdr(odp_packet_t odp_pkt, ipv4_hdr_ptr = &ipv4_hdr; } - addrs_ptr = (uint16_t *)&ipv4_hdr_ptr->src_addr; + addrs_ptr = (uint16_t *)(void *)&ipv4_hdr_ptr->src_addr; addrs_len = 2 * ODPH_IPV4ADDR_LEN; protocol = ipv4_hdr_ptr->proto; l3_len = odp_be_to_cpu_16(ipv4_hdr_ptr->tot_len); diff --git a/helper/cuckootable.c b/helper/cuckootable.c index 80ff49893..0d46300e0 100644 --- a/helper/cuckootable.c +++ b/helper/cuckootable.c @@ -256,6 +256,7 @@ odph_cuckoo_table_create( /* initialize free_slots queue */ odp_queue_param_init(&qparam); qparam.type = ODP_QUEUE_TYPE_PLAIN; + qparam.size = capacity; snprintf(queue_name, sizeof(queue_name), "fs_%s", name); queue = odp_queue_create(queue_name, &qparam); diff --git a/helper/hashtable.c b/helper/hashtable.c index 9d4114445..f26b18b27 100644 --- a/helper/hashtable.c +++ b/helper/hashtable.c @@ -76,8 +76,7 @@ odph_table_t odph_hash_table_create(const char *name, uint32_t capacity, ODPH_DBG("create para input error!\n"); return NULL; } - tbl = (odph_hash_table_imp *)odp_shm_addr(odp_shm_lookup(name)); - if (tbl != NULL) { + if (odp_shm_lookup(name) != ODP_SHM_INVALID) { ODPH_DBG("name already exist\n"); return NULL; } diff --git a/helper/include/odp/helper/icmp.h b/helper/include/odp/helper/icmp.h index e25646ad0..bef967842 100644 --- a/helper/include/odp/helper/icmp.h +++ b/helper/include/odp/helper/icmp.h @@ -32,15 +32,18 @@ typedef struct ODP_PACKED { uint8_t type; /**< message type */ uint8_t code; /**< type sub-code */ odp_u16sum_t chksum; /**< checksum of icmp header */ + /** Variant mappings of ICMP fields */ union { + /** Fields used for ICMP echo msgs */ struct { - odp_u16be_t id; - odp_u16be_t sequence; + odp_u16be_t id; /**< id */ + odp_u16be_t sequence; /**< sequence */ } echo; /**< echo datagram */ odp_u32be_t gateway; /**< gateway address */ + /** Fields used for ICMP frag msgs */ struct { - odp_u16be_t __unused; - odp_u16be_t mtu; + odp_u16be_t __unused; /**< @internal */ + odp_u16be_t mtu; /**< mtu */ } frag; /**< path mtu discovery */ } un; /**< icmp sub header */ } odph_icmphdr_t; diff --git a/helper/include/odp/helper/ip.h b/helper/include/odp/helper/ip.h index ba6e675f4..91776fad6 100644 --- a/helper/include/odp/helper/ip.h +++ b/helper/include/odp/helper/ip.h @@ -205,13 +205,14 @@ typedef struct ODP_PACKED { * IP protocol values (IPv4:'proto' or IPv6:'next_hdr') * @{*/ #define ODPH_IPPROTO_HOPOPTS 0x00 /**< IPv6 hop-by-hop options */ -#define ODPH_IPPROTO_ICMP 0x01 /**< Internet Control Message Protocol (1) */ +#define ODPH_IPPROTO_ICMPv4 0x01 /**< Internet Control Message Protocol (1) */ #define ODPH_IPPROTO_TCP 0x06 /**< Transmission Control Protocol (6) */ #define ODPH_IPPROTO_UDP 0x11 /**< User Datagram Protocol (17) */ #define ODPH_IPPROTO_ROUTE 0x2B /**< IPv6 Routing header (43) */ #define ODPH_IPPROTO_FRAG 0x2C /**< IPv6 Fragment (44) */ #define ODPH_IPPROTO_AH 0x33 /**< Authentication Header (51) */ #define ODPH_IPPROTO_ESP 0x32 /**< Encapsulating Security Payload (50) */ +#define ODPH_IPPROTO_ICMPv6 0x3A /**< Internet Control Message Protocol (58) */ #define ODPH_IPPROTO_INVALID 0xFF /**< Reserved invalid by IANA */ /**@}*/ diff --git a/helper/include/odp/helper/linux.h b/helper/include/odp/helper/linux.h new file mode 100644 index 000000000..396203a27 --- /dev/null +++ b/helper/include/odp/helper/linux.h @@ -0,0 +1,27 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * All ODP Linux helpers + * + */ + +#ifndef ODP_HELPER_LINUX_H_ +#define ODP_HELPER_LINUX_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/helper/linux/process.h> +#include <odp/helper/linux/pthread.h> + +#ifdef __cplusplus +} +#endif +#endif diff --git a/helper/include/odp/helper/strong_types.h b/helper/include/odp/helper/strong_types.h index 13e35a438..501d0f28f 100644 --- a/helper/include/odp/helper/strong_types.h +++ b/helper/include/odp/helper/strong_types.h @@ -20,10 +20,11 @@ /** Use strong typing for ODP types */ #ifdef __cplusplus +/** @internal C++ helper macro for strong typing @param type @return */ #define ODPH_HANDLE_T(type) struct _##type { uint8_t unused_dummy_var; } *type #else #define odph_handle_t struct { uint8_t unused_dummy_var; } * -/** C/C++ helper macro for strong typing */ +/** @internal C helper macro for strong typing @param type @return */ #define ODPH_HANDLE_T(type) odph_handle_t type #endif diff --git a/helper/include/odp/helper/table.h b/helper/include/odp/helper/table.h index b3440ef53..96c9c5fe7 100644 --- a/helper/include/odp/helper/table.h +++ b/helper/include/odp/helper/table.h @@ -94,7 +94,7 @@ extern "C" { #define ODPH_TABLE_NAME_LEN 32 #include <odp/helper/strong_types.h> -/** ODP table handle */ +/** @internal ODPH table handle @return */ typedef ODPH_HANDLE_T(odph_table_t); /** diff --git a/helper/include/odp/helper/tcp.h b/helper/include/odp/helper/tcp.h index fd234e583..e91b52e24 100644 --- a/helper/include/odp/helper/tcp.h +++ b/helper/include/odp/helper/tcp.h @@ -32,8 +32,9 @@ typedef struct ODP_PACKED { odp_u16be_t dst_port; /**< Destination port */ odp_u32be_t seq_no; /**< Sequence number */ odp_u32be_t ack_no; /**< Acknowledgment number */ + /** Variant maps for TCP header fields */ union { - odp_u16be_t doffset_flags; + odp_u16be_t doffset_flags; /**< TCP Flags aggregate */ #if ODP_BIG_ENDIAN_BITFIELD struct { odp_u16be_t rsvd1:8; @@ -42,14 +43,14 @@ typedef struct ODP_PACKED { struct { odp_u16be_t hl:4; /**< Hdr len, in words */ odp_u16be_t rsvd3:4; /**< Reserved */ - odp_u16be_t cwr:1; - odp_u16be_t ece:1; - odp_u16be_t urg:1; - odp_u16be_t ack:1; - odp_u16be_t psh:1; - odp_u16be_t rst:1; - odp_u16be_t syn:1; - odp_u16be_t fin:1; + odp_u16be_t cwr:1; /**< cwr bit */ + odp_u16be_t ece:1; /**< ece bit */ + odp_u16be_t urg:1; /**< urg bit */ + odp_u16be_t ack:1; /**< ack bit */ + odp_u16be_t psh:1; /**< psh bit */ + odp_u16be_t rst:1; /**< rst bit */ + odp_u16be_t syn:1; /**< syn bit */ + odp_u16be_t fin:1; /**< fin bit */ }; #elif ODP_LITTLE_ENDIAN_BITFIELD struct { @@ -59,14 +60,14 @@ typedef struct ODP_PACKED { struct { odp_u16be_t rsvd3:4; /**< Reserved */ odp_u16be_t hl:4; /**< Hdr len, in words */ - odp_u16be_t fin:1; - odp_u16be_t syn:1; - odp_u16be_t rst:1; - odp_u16be_t psh:1; - odp_u16be_t ack:1; - odp_u16be_t urg:1; - odp_u16be_t ece:1; - odp_u16be_t cwr:1; + odp_u16be_t fin:1; /**< fin bit */ + odp_u16be_t syn:1; /**< syn bit */ + odp_u16be_t rst:1; /**< rst bit */ + odp_u16be_t psh:1; /**< psh bit */ + odp_u16be_t ack:1; /**< ack bit */ + odp_u16be_t urg:1; /**< urg bit */ + odp_u16be_t ece:1; /**< ece bit */ + odp_u16be_t cwr:1; /**< cwr bit */ }; #else diff --git a/helper/include/odp/helper/threads.h b/helper/include/odp/helper/threads.h index 5682bab99..526f0d489 100644 --- a/helper/include/odp/helper/threads.h +++ b/helper/include/odp/helper/threads.h @@ -80,12 +80,15 @@ typedef struct { odph_odpthread_start_args_t start_args; /**< start arguments */ int cpu; /**< CPU ID */ int last; /**< true if last table entry */ + /** Variant field mappings for thread/process modes */ union { - struct { /* for thread implementation */ + /** For thread implementation */ + struct { pthread_t thread_id; /**< Pthread ID */ pthread_attr_t attr; /**< Pthread attributes */ } thread; - struct { /* for process implementation */ + /** For process implementation */ + struct { pid_t pid; /**< Process ID */ int status; /**< Process state chge status*/ } proc; diff --git a/helper/iplookuptable.c b/helper/iplookuptable.c index aae219945..37d31e368 100644 --- a/helper/iplookuptable.c +++ b/helper/iplookuptable.c @@ -42,6 +42,10 @@ */ typedef struct { union { + odp_buffer_t nexthop; + void *ptr; + }; + union { uint8_t u8; struct { #if ODP_BYTE_ORDER == ODP_BIG_ENDIAN @@ -53,10 +57,6 @@ typedef struct { #endif }; }; - union { - odp_buffer_t nexthop; - void *ptr; - }; } prefix_entry_t; #define ENTRY_SIZE (sizeof(prefix_entry_t) + sizeof(odp_buffer_t)) diff --git a/helper/lineartable.c b/helper/lineartable.c index 32c4956ee..dd4a59958 100644 --- a/helper/lineartable.c +++ b/helper/lineartable.c @@ -55,8 +55,7 @@ odph_table_t odph_linear_table_create(const char *name, uint32_t capacity, return NULL; } /* check name confict in shm*/ - tbl = (odph_linear_table_imp *)odp_shm_addr(odp_shm_lookup(name)); - if (tbl != NULL) { + if (odp_shm_lookup(name) != ODP_SHM_INVALID) { ODPH_DBG("name already exist\n"); return NULL; } diff --git a/include/odp/api/spec/.gitignore b/include/odp/api/spec/.gitignore index 67020331b..df9c87d90 100644 --- a/include/odp/api/spec/.gitignore +++ b/include/odp/api/spec/.gitignore @@ -1 +1,2 @@ +deprecated.h version.h diff --git a/include/odp/api/spec/classification.h b/include/odp/api/spec/classification.h index 0e1addd6e..39831b240 100644 --- a/include/odp/api/spec/classification.h +++ b/include/odp/api/spec/classification.h @@ -366,7 +366,9 @@ typedef struct odp_pmr_param_t { /** True if the value is range and false if match */ odp_bool_t range_term; + /** Variant mappings for types of matches */ union { + /** Parameters for single-valued matches */ struct { /** Value to be matched */ const void *value; @@ -374,6 +376,8 @@ typedef struct odp_pmr_param_t { /** Masked set of bits to be matched */ const void *mask; } match; + + /** Parameter for range value matches */ struct { /** Start and End values are included in the range */ /** start value of range */ diff --git a/include/odp/api/spec/crypto.h b/include/odp/api/spec/crypto.h index 9855bf989..470cba050 100644 --- a/include/odp/api/spec/crypto.h +++ b/include/odp/api/spec/crypto.h @@ -15,10 +15,14 @@ #define ODP_API_CRYPTO_H_ #include <odp/visibility_begin.h> +#include <odp/api/deprecated.h> + #ifdef __cplusplus extern "C" { #endif +#include <odp/api/packet.h> + /** @defgroup odp_crypto ODP CRYPTO * Macros, enums, types and operations to utilise crypto. * @{ @@ -82,10 +86,10 @@ typedef enum { ODP_CIPHER_ALG_AES_GCM, /** @deprecated Use ODP_CIPHER_ALG_AES_CBC instead */ - ODP_CIPHER_ALG_AES128_CBC, + ODP_DEPRECATE(ODP_CIPHER_ALG_AES128_CBC), /** @deprecated Use ODP_CIPHER_ALG_AES_GCM instead */ - ODP_CIPHER_ALG_AES128_GCM + ODP_DEPRECATE(ODP_CIPHER_ALG_AES128_GCM) } odp_cipher_alg_t; @@ -102,12 +106,24 @@ typedef enum { */ ODP_AUTH_ALG_MD5_HMAC, + /** HMAC-SHA-1 + * + * SHA-1 algorithm in HMAC mode + */ + ODP_AUTH_ALG_SHA1_HMAC, + /** HMAC-SHA-256 * * SHA-256 algorithm in HMAC mode */ ODP_AUTH_ALG_SHA256_HMAC, + /** HMAC-SHA-512 + * + * SHA-512 algorithm in HMAC mode + */ + ODP_AUTH_ALG_SHA512_HMAC, + /** AES in Galois/Counter Mode * * @note Must be paired with cipher ODP_CIPHER_ALG_AES_GCM @@ -115,13 +131,14 @@ typedef enum { ODP_AUTH_ALG_AES_GCM, /** @deprecated Use ODP_AUTH_ALG_MD5_HMAC instead */ - ODP_AUTH_ALG_MD5_96, + ODP_DEPRECATE(ODP_AUTH_ALG_MD5_96), /** @deprecated Use ODP_AUTH_ALG_SHA256_HMAC instead */ - ODP_AUTH_ALG_SHA256_128, + ODP_DEPRECATE(ODP_AUTH_ALG_SHA256_128), /** @deprecated Use ODP_AUTH_ALG_AES_GCM instead */ - ODP_AUTH_ALG_AES128_GCM + ODP_DEPRECATE(ODP_AUTH_ALG_AES128_GCM) + } odp_auth_alg_t; /** @@ -146,10 +163,11 @@ typedef union odp_crypto_cipher_algos_t { uint32_t aes_gcm : 1; /** @deprecated Use aes_cbc instead */ - uint32_t aes128_cbc : 1; + uint32_t ODP_DEPRECATE(aes128_cbc) : 1; /** @deprecated Use aes_gcm instead */ - uint32_t aes128_gcm : 1; + uint32_t ODP_DEPRECATE(aes128_gcm) : 1; + } bit; /** All bits of the bit field structure @@ -171,20 +189,27 @@ typedef union odp_crypto_auth_algos_t { /** ODP_AUTH_ALG_MD5_HMAC */ uint32_t md5_hmac : 1; + /** ODP_AUTH_ALG_SHA1_HMAC */ + uint32_t sha1_hmac : 1; + /** ODP_AUTH_ALG_SHA256_HMAC */ uint32_t sha256_hmac : 1; + /** ODP_AUTH_ALG_SHA512_HMAC */ + uint32_t sha512_hmac : 1; + /** ODP_AUTH_ALG_AES_GCM */ uint32_t aes_gcm : 1; /** @deprecated Use md5_hmac instead */ - uint32_t md5_96 : 1; + uint32_t ODP_DEPRECATE(md5_96) : 1; /** @deprecated Use sha256_hmac instead */ - uint32_t sha256_128 : 1; + uint32_t ODP_DEPRECATE(sha256_128) : 1; /** @deprecated Use aes_gcm instead */ - uint32_t aes128_gcm : 1; + uint32_t ODP_DEPRECATE(aes128_gcm) : 1; + } bit; /** All bits of the bit field structure @@ -220,15 +245,10 @@ typedef struct odp_crypto_iv { /** * Crypto API data range specifier + * + * @deprecated Use odp_packet_data_range_t instead */ -typedef struct odp_crypto_data_range { - /** Offset from beginning of packet */ - uint32_t offset; - - /** Length of data to operate on */ - uint32_t length; - -} odp_crypto_data_range_t; +typedef odp_packet_data_range_t ODP_DEPRECATE(odp_crypto_data_range_t); /** * Crypto API session creation parameters @@ -276,10 +296,16 @@ typedef struct odp_crypto_session_param_t { /** Authentication key * - * Use odp_crypto_auth_capa() for supported digest and key lengths. + * Use odp_crypto_auth_capability() for supported key lengths. */ odp_crypto_key_t auth_key; + /** Authentication digest length in bytes + * + * Use odp_crypto_auth_capability() for supported digest lengths. + */ + uint32_t auth_digest_len; + /** Async mode completion event queue * * When odp_crypto_operation() is asynchronous, the completion queue is @@ -299,7 +325,7 @@ typedef struct odp_crypto_session_param_t { } odp_crypto_session_param_t; /** @deprecated Use odp_crypto_session_param_t instead */ -typedef odp_crypto_session_param_t odp_crypto_session_params_t; +typedef odp_crypto_session_param_t ODP_DEPRECATE(odp_crypto_session_params_t); /** * Crypto API per packet operation parameters @@ -346,16 +372,26 @@ typedef struct odp_crypto_op_param_t { */ uint32_t hash_result_offset; + /** Additional Authenticated Data (AAD) */ + struct { + /** Pointer to ADD */ + uint8_t *ptr; + + /** AAD length in bytes. Use odp_crypto_auth_capability() for + * supported AAD lengths. */ + uint32_t length; + } aad; + /** Data range to apply cipher */ - odp_crypto_data_range_t cipher_range; + odp_packet_data_range_t cipher_range; /** Data range to authenticate */ - odp_crypto_data_range_t auth_range; + odp_packet_data_range_t auth_range; } odp_crypto_op_param_t; /** @deprecated Use odp_crypto_op_param_t instead */ -typedef odp_crypto_op_param_t odp_crypto_op_params_t; +typedef odp_crypto_op_param_t ODP_DEPRECATE(odp_crypto_op_params_t); /** * Crypto API session creation return code diff --git a/include/odp/api/spec/deprecated.h.in b/include/odp/api/spec/deprecated.h.in new file mode 100644 index 000000000..224f60ff3 --- /dev/null +++ b/include/odp/api/spec/deprecated.h.in @@ -0,0 +1,50 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * Macro for deprecated API definitions + */ + +#ifndef ODP_API_DEPRECATED_H_ +#define ODP_API_DEPRECATED_H_ +#include <odp/visibility_begin.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Deprecated API definitions + * + * Some API definitions may be deprecated by this or a previous API version. + * This macro controls if those are enabled (and visible to the application) + * or disabled. + * + * * 0: Deprecated API definitions are disabled (default) + * * 1: Deprecated API definitions are enabled + */ +#define ODP_DEPRECATED_API @ODP_DEPRECATED_API@ + +/** + * @def ODP_DEPRECATE + * + * Macro to deprecate API definitions + */ + +#if ODP_DEPRECATED_API +#define ODP_DEPRECATE(x) x +#else +#define ODP_DEPRECATE(x) _deprecated_ ## x +#endif + +#ifdef __cplusplus +} +#endif + +#include <odp/visibility_end.h> +#endif diff --git a/include/odp/api/spec/hints.h b/include/odp/api/spec/hints.h index 82400f073..7434c6a5c 100644 --- a/include/odp/api/spec/hints.h +++ b/include/odp/api/spec/hints.h @@ -52,11 +52,6 @@ extern "C" { #define ODP_PRINTF_FORMAT(x, y) __attribute__((format(printf, (x), (y)))) /** - * Indicate deprecated variables, functions or types - */ -#define ODP_DEPRECATED __attribute__((__deprecated__)) - -/** * Intentionally unused variables of functions */ #define ODP_UNUSED __attribute__((__unused__)) @@ -96,7 +91,6 @@ extern "C" { #define ODP_WEAK_SYMBOL #define ODP_HOT_CODE #define ODP_COLD_CODE -#define ODP_DEPRECATED #define ODP_UNUSED #define odp_likely(x) #define odp_unlikely(x) diff --git a/include/odp/api/spec/packet.h b/include/odp/api/spec/packet.h index 92c35ae6d..79d8773e9 100644 --- a/include/odp/api/spec/packet.h +++ b/include/odp/api/spec/packet.h @@ -71,6 +71,18 @@ extern "C" { * Packet is red */ +/** + * Packet API data range specifier + */ +typedef struct odp_packet_data_range { + /** Offset from beginning of packet */ + uint32_t offset; + + /** Length of data to operate on */ + uint32_t length; + +} odp_packet_data_range_t; + /* * * Alloc and free diff --git a/include/odp/api/spec/packet_io.h b/include/odp/api/spec/packet_io.h index 85cd6d184..cec1f22af 100644 --- a/include/odp/api/spec/packet_io.h +++ b/include/odp/api/spec/packet_io.h @@ -346,6 +346,39 @@ typedef union odp_pktout_config_opt_t { } odp_pktout_config_opt_t; /** + * Parser layers + */ +typedef enum odp_pktio_parser_layer_t { + /** No layers */ + ODP_PKTIO_PARSER_LAYER_NONE = 0, + + /** Layer L2 protocols (Ethernet, VLAN, ARP, etc) */ + ODP_PKTIO_PARSER_LAYER_L2, + + /** Layer L3 protocols (IPv4, IPv6, ICMP, IPsec, etc) */ + ODP_PKTIO_PARSER_LAYER_L3, + + /** Layer L4 protocols (UDP, TCP, SCTP) */ + ODP_PKTIO_PARSER_LAYER_L4, + + /** All layers */ + ODP_PKTIO_PARSER_LAYER_ALL + +} odp_pktio_parser_layer_t; + +/** + * Parser configuration + */ +typedef struct odp_pktio_parser_config_t { + /** Protocol parsing level in packet input + * + * Parse protocol layers in minimum up to this level during packet + * input. The default value is ODP_PKTIO_PARSER_LAYER_ALL. */ + odp_pktio_parser_layer_t layer; + +} odp_pktio_parser_config_t; + +/** * Packet IO configuration options * * Packet IO interface level configuration options. Use odp_pktio_capability() @@ -363,6 +396,9 @@ typedef struct odp_pktio_config_t { * Default value for all bits is zero. */ odp_pktout_config_opt_t pktout; + /** Packet input parser configuration */ + odp_pktio_parser_config_t parser; + /** Interface loopback mode * * In this mode the packets sent out through the interface is diff --git a/include/odp/api/spec/pool.h b/include/odp/api/spec/pool.h index c0de195a7..6fc5b6b4a 100644 --- a/include/odp/api/spec/pool.h +++ b/include/odp/api/spec/pool.h @@ -166,7 +166,9 @@ typedef struct odp_pool_param_t { /** Pool type */ int type; + /** Variant parameters for different pool types */ union { + /** Parameters for buffer pools */ struct { /** Number of buffers in the pool */ uint32_t num; @@ -180,6 +182,8 @@ typedef struct odp_pool_param_t { Default will always be a multiple of 8. */ uint32_t align; } buf; + + /** Parameters for packet pools */ struct { /** The number of packets that the pool must provide that are packet length 'len' bytes or smaller. @@ -211,6 +215,8 @@ typedef struct odp_pool_param_t { Specify as 0 if no user area is needed. */ uint32_t uarea_size; } pkt; + + /** Parameters for timeout pools */ struct { /** Number of timeouts in the pool */ uint32_t num; diff --git a/include/odp/api/spec/queue.h b/include/odp/api/spec/queue.h index 7972feacb..9dd0a5618 100644 --- a/include/odp/api/spec/queue.h +++ b/include/odp/api/spec/queue.h @@ -100,7 +100,9 @@ typedef enum odp_queue_op_mode_t { * Queue capabilities */ typedef struct odp_queue_capability_t { - /** Maximum number of event queues */ + /** Maximum number of event queues of any type (default size). Use + * this in addition to queue type specific 'max_num', if both queue + * types are used simultaneously. */ uint32_t max_queues; /** Maximum number of ordered locks per queue */ @@ -112,6 +114,32 @@ typedef struct odp_queue_capability_t { /** Number of scheduling priorities */ unsigned sched_prios; + /** Plain queue capabilities */ + struct { + /** Maximum number of plain queues of the default size. */ + uint32_t max_num; + + /** Maximum number of events a plain queue can store + * simultaneously. The value of zero means that plain + * queues do not have a size limit, but a single queue can + * store all available events. */ + uint32_t max_size; + + } plain; + + /** Scheduled queue capabilities */ + struct { + /** Maximum number of scheduled queues of the default size. */ + uint32_t max_num; + + /** Maximum number of events a scheduled queue can store + * simultaneously. The value of zero means that scheduled + * queues do not have a size limit, but a single queue can + * store all available events. */ + uint32_t max_size; + + } sched; + } odp_queue_capability_t; /** @@ -165,6 +193,15 @@ typedef struct odp_queue_param_t { * The implementation may use this value as a hint for the number of * context data bytes to prefetch. Default value is zero (no hint). */ uint32_t context_len; + + /** Queue size + * + * The queue must be able to store at minimum this many events + * simultaneously. The value must not exceed 'max_size' queue + * capability. The value of zero means implementation specific + * default size. */ + uint32_t size; + } odp_queue_param_t; /** diff --git a/include/odp/api/spec/system_info.h b/include/odp/api/spec/system_info.h index 0bb4f1f12..ca4dcdc7a 100644 --- a/include/odp/api/spec/system_info.h +++ b/include/odp/api/spec/system_info.h @@ -45,6 +45,15 @@ uint64_t odp_sys_page_size(void); int odp_sys_cache_line_size(void); /** + * Print system info + * + * Print out implementation defined information about the system. This + * information is intended for debugging purposes and may contain e.g. + * information about CPUs, memory and other HW configuration. + */ +void odp_sys_info_print(void); + +/** * @} */ diff --git a/include/odp/api/spec/time.h b/include/odp/api/spec/time.h index fcc94c98e..29175eb5a 100644 --- a/include/odp/api/spec/time.h +++ b/include/odp/api/spec/time.h @@ -158,19 +158,6 @@ void odp_time_wait_until(odp_time_t time); void odp_time_wait_ns(uint64_t ns); /** - * Get printable value for an odp_time_t - * - * @param time time to be printed - * - * @return uint64_t value that can be used to print/display this time - * - * @note This routine is intended to be used for diagnostic purposes - * to enable applications to generate a printable value that represents - * an odp_time_t time. - */ -uint64_t odp_time_to_u64(odp_time_t time); - -/** * @} */ diff --git a/include/odp/api/spec/traffic_mngr.h b/include/odp/api/spec/traffic_mngr.h index 71198bbdd..3a748cef7 100644 --- a/include/odp/api/spec/traffic_mngr.h +++ b/include/odp/api/spec/traffic_mngr.h @@ -471,9 +471,10 @@ typedef enum { typedef struct { odp_tm_egress_kind_t egress_kind; /**< Union discriminator */ + /** Variant parameters for different TM outputs */ union { - odp_pktio_t pktio; - odp_tm_egress_fcn_t egress_fcn; + odp_pktio_t pktio; /**< Output to PktIO */ + odp_tm_egress_fcn_t egress_fcn; /**< Output to user func */ }; } odp_tm_egress_t; diff --git a/include/odp_api.h b/include/odp_api.h index ec7fcd2e7..060ec888b 100644 --- a/include/odp_api.h +++ b/include/odp_api.h @@ -18,6 +18,7 @@ extern "C" { #endif +#include <odp/api/deprecated.h> #include <odp/api/version.h> #include <odp/api/std_types.h> #include <odp/api/compiler.h> diff --git a/pkgconfig/libodp-dpdk.pc.in b/pkgconfig/libodp-dpdk.pc.in index 6497c796d..23804b0f7 100644 --- a/pkgconfig/libodp-dpdk.pc.in +++ b/pkgconfig/libodp-dpdk.pc.in @@ -7,5 +7,5 @@ Name: libodp-dpdk Description: The ODP packet processing engine Version: @PKGCONFIG_VERSION@ Libs: -L${libdir} -lodp-dpdk -Libs.private: +Libs.private: @ATOMIC_LIBS@ Cflags: -I${includedir} diff --git a/pkgconfig/libodphelper-linux-generic.pc.in b/pkgconfig/libodphelper.pc.in index cab7be2dd..b14335e5c 100644 --- a/pkgconfig/libodphelper-linux-generic.pc.in +++ b/pkgconfig/libodphelper.pc.in @@ -3,9 +3,9 @@ exec_prefix=@exec_prefix@ libdir=@libdir@ includedir=@includedir@ -Name: libodphelper-linux-generic +Name: libodphelper Description: Helper for the ODP packet processing engine Version: @PKGCONFIG_VERSION@ -Libs: -L${libdir} -lodphelper-linux-generic +Libs: -L${libdir} -lodphelper Libs.private: Cflags: -I${includedir} diff --git a/platform/Makefile.inc b/platform/Makefile.inc index e40a14f04..25c89e91c 100644 --- a/platform/Makefile.inc +++ b/platform/Makefile.inc @@ -57,8 +57,9 @@ odpapispecinclude_HEADERS = \ $(top_srcdir)/include/odp/api/spec/ticketlock.h \ $(top_srcdir)/include/odp/api/spec/time.h \ $(top_srcdir)/include/odp/api/spec/timer.h \ - $(top_builddir)/include/odp/api/spec/version.h \ - $(top_srcdir)/include/odp/api/spec/traffic_mngr.h + $(top_srcdir)/include/odp/api/spec/traffic_mngr.h \ + $(top_builddir)/include/odp/api/spec/deprecated.h \ + $(top_builddir)/include/odp/api/spec/version.h odpapiabidefaultincludedir= $(includedir)/odp/arch/default/api/abi odpapiabidefaultinclude_HEADERS = \ @@ -97,4 +98,6 @@ EXTRA_DIST = \ arch/powerpc/odp_sysinfo_parse.c \ arch/x86/odp/api/cpu_arch.h \ arch/x86/odp_cpu_arch.c \ - arch/x86/odp_sysinfo_parse.c + arch/x86/odp_sysinfo_parse.c \ + arch/x86/cpu_flags.c \ + arch/x86/cpu_flags.h diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index dfbc57619..c1497f0f6 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -42,6 +42,7 @@ odpapiinclude_HEADERS = \ $(srcdir)/include/odp/api/cpumask.h \ $(srcdir)/include/odp/api/crypto.h \ $(srcdir)/include/odp/api/debug.h \ + $(srcdir)/include/odp/api/deprecated.h \ $(srcdir)/include/odp/api/errno.h \ $(srcdir)/include/odp/api/event.h \ $(srcdir)/include/odp/api/hash.h \ @@ -154,7 +155,7 @@ noinst_HEADERS = \ ${top_srcdir}/platform/linux-generic/include/odp_schedule_if.h \ ${top_srcdir}/platform/linux-generic/include/odp_sorted_list_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_shm_internal.h \ - ${srcdir}/include/odp_time_internal.h \ + ${top_srcdir}/platform/linux-generic/include/odp_time_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_timer_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_timer_wheel_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_traffic_mngr_internal.h \ @@ -217,12 +218,18 @@ __LIB__libodp_dpdk_la_SOURCES = \ arch/@ARCH_DIR@/odp_cpu_arch.c \ arch/@ARCH_DIR@/odp_sysinfo_parse.c +__LIB__libodp_dpdk_la_LIBADD = $(ATOMIC_LIBS) + +if ARCH_IS_X86 +__LIB__libodp_dpdk_la_SOURCES += arch/@ARCH_DIR@/cpu_flags.c +endif + # Create symlink for ABI header files. Application does not need to use the arch # specific include path for installed files. install-data-hook: - if [ -h $(prefix)/include/odp/api/abi ]; then \ - : \ + if [ -h $(DESTDIR)$(prefix)/include/odp/api/abi ]; then \ + : ; \ else \ - $(LN_S) -rf $(prefix)/include/odp/arch/@ARCH_ABI@/odp/api/abi \ - $(prefix)/include/odp/api/abi; \ + $(LN_S) -rf $(DESTDIR)$(prefix)/include/odp/arch/@ARCH_ABI@/odp/api/abi \ + $(DESTDIR)$(prefix)/include/odp/api/abi; \ fi diff --git a/platform/linux-dpdk/include/odp/api/deprecated.h b/platform/linux-dpdk/include/odp/api/deprecated.h new file mode 120000 index 000000000..42bc7ef16 --- /dev/null +++ b/platform/linux-dpdk/include/odp/api/deprecated.h @@ -0,0 +1 @@ +../../../../linux-generic/include/odp/api/deprecated.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_types.h b/platform/linux-dpdk/include/odp/api/plat/packet_types.h index c6ee3966f..5df0a41ec 100644 --- a/platform/linux-dpdk/include/odp/api/plat/packet_types.h +++ b/platform/linux-dpdk/include/odp/api/plat/packet_types.h @@ -99,8 +99,8 @@ typedef union { /** All input flags */ uint64_t all; + /** Individual input flags */ struct { - uint64_t parsed_l2:1; /**< L2 parsed */ uint64_t dst_queue:1; /**< Dst queue present */ uint64_t timestamp:1; /**< Timestamp present */ diff --git a/platform/linux-dpdk/include/odp/api/plat/time_types.h b/platform/linux-dpdk/include/odp/api/plat/time_types.h index e53ad2f97..b42bc83c2 100644..120000 --- a/platform/linux-dpdk/include/odp/api/plat/time_types.h +++ b/platform/linux-dpdk/include/odp/api/plat/time_types.h @@ -1,43 +1 @@ -/* Copyright (c) 2016, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -/** - * @file - * - * ODP time service - */ - -#ifndef ODP_TIME_TYPES_H_ -#define ODP_TIME_TYPES_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -/** @addtogroup odp_time - * @{ - **/ - -/** - * @internal Time structure used to isolate odp-linux implementation from - * the linux timespec structure, which is dependent on POSIX extension level. - */ -typedef struct odp_time_t { - int64_t tv_sec; /**< @internal Seconds or DPDK ticks */ - int64_t tv_nsec; /**< @internal Nanoseconds */ -} odp_time_t; - -#define ODP_TIME_NULL ((odp_time_t){0, 0}) - -/** - * @} - */ - -#ifdef __cplusplus -} -#endif - -#endif +../../../../../linux-generic/include/odp/api/plat/time_types.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include/odp_packet_internal.h b/platform/linux-dpdk/include/odp_packet_internal.h index d3f00847e..481e4afb1 100644 --- a/platform/linux-dpdk/include/odp_packet_internal.h +++ b/platform/linux-dpdk/include/odp_packet_internal.h @@ -84,18 +84,6 @@ ODP_STATIC_ASSERT(sizeof(output_flags_t) == sizeof(uint32_t), "OUTPUT_FLAGS_SIZE_ERROR"); /** - * Protocol stack layers - */ -typedef enum { - LAYER_NONE = 0, - LAYER_L1, - LAYER_L2, - LAYER_L3, - LAYER_L4, - LAYER_ALL -} layer_t; - -/** * Packet parser metadata */ typedef struct { @@ -106,14 +94,6 @@ typedef struct { uint32_t l2_offset; /**< offset to L2 hdr, e.g. Eth */ uint32_t l3_offset; /**< offset to L3 hdr, e.g. IPv4, IPv6 */ uint32_t l4_offset; /**< offset to L4 hdr (TCP, UDP, SCTP, also ICMP) */ - - uint32_t l3_len; /**< Layer 3 length */ - uint32_t l4_len; /**< Layer 4 length */ - - uint16_t ethtype; /**< EtherType */ - uint8_t ip_proto; /**< IP protocol */ - uint8_t parsed_layers; /**< Highest parsed protocol stack layer */ - } packet_parser_t; /** @@ -190,58 +170,25 @@ static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len) rte_pktmbuf_pkt_len(&pkt_hdr->buf_hdr.mb) = len; } -static inline int packet_parse_l2_not_done(packet_parser_t *prs) -{ - return !prs->input_flags.parsed_l2; -} - -static inline int packet_parse_not_complete(odp_packet_hdr_t *pkt_hdr) -{ - return pkt_hdr->p.parsed_layers != LAYER_ALL; -} - /* Forward declarations */ int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt); -/* Fill in parser metadata for L2 */ -static inline void packet_parse_l2(packet_parser_t *prs, uint32_t frame_len) -{ - /* Packet alloc or reset have already init other offsets and flags */ - - /* We only support Ethernet for now */ - prs->input_flags.eth = 1; - - /* Detect jumbo frames */ - if (frame_len > _ODP_ETH_LEN_MAX) - prs->input_flags.jumbo = 1; - - /* Assume valid L2 header, no CRC/FCS check in SW */ - prs->input_flags.l2 = 1; - - prs->input_flags.parsed_l2 = 1; -} +/* Perform packet parse up to a given protocol layer */ +int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, + odp_pktio_parser_layer_t layer); -static inline void _odp_packet_reset_parse(odp_packet_t pkt) +/* Reset parser metadata for a new parse */ +static inline void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) { - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - - uint32_t frame_len = rte_pktmbuf_pkt_len(&pkt_hdr->buf_hdr.mb); - - pkt_hdr->p.parsed_layers = LAYER_NONE; - pkt_hdr->p.input_flags.all = 0; + /* Reset parser metadata before new parse */ + pkt_hdr->p.error_flags.all = 0; + pkt_hdr->p.input_flags.all = 0; pkt_hdr->p.output_flags.all = 0; - pkt_hdr->p.error_flags.all = 0; - pkt_hdr->p.l2_offset = 0; - - packet_parse_l2(&pkt_hdr->p, frame_len); + pkt_hdr->p.l2_offset = 0; + pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; + pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; } -/* Perform packet parse up to a given protocol layer */ -int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, layer_t layer); - -/* Reset parser metadata for a new parse */ -void packet_parse_reset(odp_packet_hdr_t *pkt_hdr); - /* Convert a packet handle to a buffer handle */ odp_buffer_t _odp_packet_to_buffer(odp_packet_t pkt); @@ -277,7 +224,9 @@ static inline void packet_set_ts(odp_packet_hdr_t *pkt_hdr, odp_time_t *ts) } int packet_parse_common(packet_parser_t *pkt_hdr, const uint8_t *ptr, - uint32_t pkt_len, uint32_t seg_len, layer_t layer); + uint32_t pkt_len, uint32_t seg_len, + odp_pktio_parser_layer_t layer); + /* We can't enforce tailroom reservation for received packets */ ODP_STATIC_ASSERT(CONFIG_PACKET_TAILROOM == 0, diff --git a/platform/linux-dpdk/include/odp_time_internal.h b/platform/linux-dpdk/include/odp_time_internal.h deleted file mode 100644 index 8cbf81419..000000000 --- a/platform/linux-dpdk/include/odp_time_internal.h +++ /dev/null @@ -1,44 +0,0 @@ -/* Copyright (c) 2016, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -/** - * @file - * - * ODP time service - */ - -#ifndef ODP_TIME_INTERNAL_H_ -#define ODP_TIME_INTERNAL_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -typedef uint64_t (*time_to_ns_fn) (odp_time_t time); -typedef odp_time_t (*time_diff_fn) (odp_time_t t2, odp_time_t t1); -typedef odp_time_t (*time_curr_fn)(void); -typedef int (*time_cmp_fn) (odp_time_t t2, odp_time_t t1); -typedef odp_time_t (*time_sum_fn) (odp_time_t t1, odp_time_t t2); -typedef odp_time_t (*time_local_from_ns_fn) (uint64_t ns); -typedef uint64_t (*time_local_res_fn)(void); -typedef uint64_t (*time_to_u64_fn) (odp_time_t time); - -typedef struct time_handler_ { - time_to_ns_fn time_to_ns; - time_diff_fn time_diff; - time_curr_fn time_curr; - time_cmp_fn time_cmp; - time_sum_fn time_sum; - time_local_from_ns_fn time_local_from_ns; - time_local_res_fn time_local_res; - time_to_u64_fn time_to_u64; -} time_handler_t; - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/platform/linux-dpdk/m4/configure.m4 b/platform/linux-dpdk/m4/configure.m4 index fb9a91399..230ef6eff 100644 --- a/platform/linux-dpdk/m4/configure.m4 +++ b/platform/linux-dpdk/m4/configure.m4 @@ -28,6 +28,51 @@ AC_LINK_IFELSE( echo "Use newer version. For gcc > 4.7.0" exit -1) +dnl Check whether -latomic is needed +use_libatomic=no + +AC_MSG_CHECKING(whether -latomic is needed for 64-bit atomic built-ins) +AC_LINK_IFELSE( + [AC_LANG_SOURCE([[ + static int loc; + int main(void) + { + int prev = __atomic_exchange_n(&loc, 7, __ATOMIC_RELAXED); + return 0; + } + ]])], + [AC_MSG_RESULT(no)], + [AC_MSG_RESULT(yes) + AC_CHECK_LIB( + [atomic], [__atomic_exchange_8], + [use_libatomic=yes], + [AC_MSG_CHECKING([__atomic_exchange_8 is not available])]) + ]) + +AC_MSG_CHECKING(whether -latomic is needed for 128-bit atomic built-ins) +AC_LINK_IFELSE( + [AC_LANG_SOURCE([[ + static __int128 loc; + int main(void) + { + __int128 prev; + prev = __atomic_exchange_n(&loc, 7, __ATOMIC_RELAXED); + return 0; + } + ]])], + [AC_MSG_RESULT(no)], + [AC_MSG_RESULT(yes) + AC_CHECK_LIB( + [atomic], [__atomic_exchange_16], + [use_libatomic=yes], + [AC_MSG_CHECKING([cannot detect support for 128-bit atomics])]) + ]) + +if test "x$use_libatomic" = "xyes"; then + ATOMIC_LIBS="-latomic" +fi +AC_SUBST([ATOMIC_LIBS]) + # linux-generic PCAP support is not relevant as the code doesn't use # linux-generic pktio at all. And DPDK has its own PCAP support anyway AM_CONDITIONAL([HAVE_PCAP], [false]) diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index e1f7026ed..d95b15780 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -94,13 +94,15 @@ static int cipher_alg_odp_to_rte(odp_cipher_alg_t cipher_alg, cipher_xform->cipher.algo = RTE_CRYPTO_CIPHER_3DES_CBC; break; case ODP_CIPHER_ALG_AES_CBC: - /* deprecated */ +#if ODP_DEPRECATED_API case ODP_CIPHER_ALG_AES128_CBC: +#endif cipher_xform->cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC; break; case ODP_CIPHER_ALG_AES_GCM: - /* deprecated */ +#if ODP_DEPRECATED_API case ODP_CIPHER_ALG_AES128_GCM: +#endif cipher_xform->cipher.algo = RTE_CRYPTO_CIPHER_AES_GCM; break; default: @@ -121,20 +123,31 @@ static int auth_alg_odp_to_rte(odp_auth_alg_t auth_alg, auth_xform->auth.algo = RTE_CRYPTO_AUTH_NULL; break; case ODP_AUTH_ALG_MD5_HMAC: - /* deprecated */ +#if ODP_DEPRECATED_API case ODP_AUTH_ALG_MD5_96: +#endif auth_xform->auth.algo = RTE_CRYPTO_AUTH_MD5_HMAC; auth_xform->auth.digest_length = 12; break; case ODP_AUTH_ALG_SHA256_HMAC: - /* deprecated */ +#if ODP_DEPRECATED_API case ODP_AUTH_ALG_SHA256_128: +#endif auth_xform->auth.algo = RTE_CRYPTO_AUTH_SHA256_HMAC; auth_xform->auth.digest_length = 16; break; + case ODP_AUTH_ALG_SHA1_HMAC: + auth_xform->auth.algo = RTE_CRYPTO_AUTH_SHA1_HMAC; + auth_xform->auth.digest_length = 20; + break; + case ODP_AUTH_ALG_SHA512_HMAC: + auth_xform->auth.algo = RTE_CRYPTO_AUTH_SHA512_HMAC; + auth_xform->auth.digest_length = 64; + break; case ODP_AUTH_ALG_AES_GCM: - /* deprecated */ +#if ODP_DEPRECATED_API case ODP_AUTH_ALG_AES128_GCM: +#endif auth_xform->auth.algo = RTE_CRYPTO_AUTH_AES_GCM; auth_xform->auth.digest_length = 16; break; @@ -151,8 +164,10 @@ static crypto_session_entry_t *alloc_session(void) odp_spinlock_lock(&global->lock); session = global->free; - if (session) + if (session) { global->free = session->next; + session->next = NULL; + } odp_spinlock_unlock(&global->lock); return session; @@ -349,14 +364,16 @@ int odp_crypto_capability(odp_crypto_capability_t *capability) if (cap_cipher_algo == RTE_CRYPTO_CIPHER_AES_CBC) { hw_ciphers->bit.aes_cbc = 1; - /* Deprecated */ +#if ODP_DEPRECATED_API hw_ciphers->bit.aes128_cbc = 1; +#endif } if (cap_cipher_algo == RTE_CRYPTO_CIPHER_AES_GCM) { hw_ciphers->bit.aes_gcm = 1; - /* Deprecated */ +#if ODP_DEPRECATED_API hw_ciphers->bit.aes128_gcm = 1; +#endif } } @@ -373,20 +390,31 @@ int odp_crypto_capability(odp_crypto_capability_t *capability) if (cap_auth_algo == RTE_CRYPTO_AUTH_AES_GCM) { hw_auths->bit.aes_gcm = 1; - /* Deprecated */ +#if ODP_DEPRECATED_API hw_auths->bit.aes128_gcm = 1; +#endif } if (cap_auth_algo == RTE_CRYPTO_AUTH_MD5_HMAC) { hw_auths->bit.md5_hmac = 1; - /* Deprecated */ +#if ODP_DEPRECATED_API hw_auths->bit.md5_96 = 1; +#endif } if (cap_auth_algo == RTE_CRYPTO_AUTH_SHA256_HMAC) { hw_auths->bit.sha256_hmac = 1; - /* Deprecated */ +#if ODP_DEPRECATED_API hw_auths->bit.sha256_128 = 1; +#endif + } + if (cap_auth_algo == + RTE_CRYPTO_AUTH_SHA1_HMAC) { + hw_auths->bit.sha1_hmac = 1; + } + if (cap_auth_algo == + RTE_CRYPTO_AUTH_SHA512_HMAC) { + hw_auths->bit.sha512_hmac = 1; } } cap = &dev_info.capabilities[++i]; @@ -411,14 +439,16 @@ int odp_crypto_capability(odp_crypto_capability_t *capability) if (cap_cipher_algo == RTE_CRYPTO_CIPHER_AES_CBC) { ciphers->bit.aes_cbc = 1; - /* Deprecated */ +#if ODP_DEPRECATED_API ciphers->bit.aes128_cbc = 1; +#endif } if (cap_cipher_algo == RTE_CRYPTO_CIPHER_AES_GCM) { ciphers->bit.aes_gcm = 1; - /* Deprecated */ +#if ODP_DEPRECATED_API ciphers->bit.aes128_gcm = 1; +#endif } } @@ -435,20 +465,31 @@ int odp_crypto_capability(odp_crypto_capability_t *capability) if (cap_auth_algo == RTE_CRYPTO_AUTH_AES_GCM) { auths->bit.aes_gcm = 1; - /* Deprecated */ +#if ODP_DEPRECATED_API auths->bit.aes128_gcm = 1; +#endif } if (cap_auth_algo == RTE_CRYPTO_AUTH_MD5_HMAC) { auths->bit.md5_hmac = 1; - /* Deprecated */ +#if ODP_DEPRECATED_API auths->bit.md5_96 = 1; +#endif } if (cap_auth_algo == RTE_CRYPTO_AUTH_SHA256_HMAC) { auths->bit.sha256_hmac = 1; - /* Deprecated */ +#if ODP_DEPRECATED_API auths->bit.sha256_128 = 1; +#endif + } + if (cap_auth_algo == + RTE_CRYPTO_AUTH_SHA1_HMAC) { + auths->bit.sha1_hmac = 1; + } + if (cap_auth_algo == + RTE_CRYPTO_AUTH_SHA512_HMAC) { + auths->bit.sha512_hmac = 1; } } cap = &dev_info.capabilities[++i]; @@ -547,7 +588,7 @@ int odp_crypto_cipher_capability(odp_cipher_alg_t cipher, int odp_crypto_auth_capability(odp_auth_alg_t auth, odp_crypto_auth_capability_t dst[], - int num_copy) + int num_copy) { odp_crypto_auth_capability_t src[num_copy]; int idx = 0, rc = 0; @@ -740,7 +781,7 @@ static int get_crypto_dev(struct rte_crypto_sym_xform *cipher_xform, return -1; } -int odp_crypto_session_create(odp_crypto_session_params_t *params, +int odp_crypto_session_create(odp_crypto_session_param_t *param, odp_crypto_session_t *session_out, odp_crypto_ses_create_err_t *status) { @@ -771,7 +812,7 @@ int odp_crypto_session_create(odp_crypto_session_params_t *params, /* Cipher Data */ cipher_xform.cipher.key.data = rte_malloc("crypto key", - params->cipher_key.length, 0); + param->cipher_key.length, 0); if (cipher_xform.cipher.key.data == NULL) { ODP_ERR("Failed to allocate memory for cipher key\n"); /* remove the crypto_session_entry_t */ @@ -782,14 +823,14 @@ int odp_crypto_session_create(odp_crypto_session_params_t *params, cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; cipher_xform.next = NULL; - cipher_xform.cipher.key.length = params->cipher_key.length; + cipher_xform.cipher.key.length = param->cipher_key.length; memcpy(cipher_xform.cipher.key.data, - params->cipher_key.data, - params->cipher_key.length); + param->cipher_key.data, + param->cipher_key.length); /* Authentication Data */ auth_xform.auth.key.data = rte_malloc("auth key", - params->auth_key.length, 0); + param->auth_key.length, 0); if (auth_xform.auth.key.data == NULL) { ODP_ERR("Failed to allocate memory for auth key\n"); /* remove the crypto_session_entry_t */ @@ -799,16 +840,16 @@ int odp_crypto_session_create(odp_crypto_session_params_t *params, } auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; auth_xform.next = NULL; - auth_xform.auth.key.length = params->auth_key.length; + auth_xform.auth.key.length = param->auth_key.length; memcpy(auth_xform.auth.key.data, - params->auth_key.data, - params->auth_key.length); + param->auth_key.data, + param->auth_key.length); /* Derive order */ - if (ODP_CRYPTO_OP_ENCODE == params->op) - entry->do_cipher_first = params->auth_cipher_text; + if (ODP_CRYPTO_OP_ENCODE == param->op) + entry->do_cipher_first = param->auth_cipher_text; else - entry->do_cipher_first = !params->auth_cipher_text; + entry->do_cipher_first = !param->auth_cipher_text; /* Process based on cipher */ /* Derive order */ @@ -824,7 +865,7 @@ int odp_crypto_session_create(odp_crypto_session_params_t *params, first_xform->next = &cipher_xform; } - rc = cipher_alg_odp_to_rte(params->cipher_alg, &cipher_xform); + rc = cipher_alg_odp_to_rte(param->cipher_alg, &cipher_xform); /* Check result */ if (rc) { @@ -832,7 +873,7 @@ int odp_crypto_session_create(odp_crypto_session_params_t *params, return -1; } - rc = auth_alg_odp_to_rte(params->auth_alg, &auth_xform); + rc = auth_alg_odp_to_rte(param->auth_alg, &auth_xform); /* Check result */ if (rc) { @@ -845,7 +886,7 @@ int odp_crypto_session_create(odp_crypto_session_params_t *params, rc = get_crypto_dev(&cipher_xform, &auth_xform, - params->iv.length, + param->iv.length, &cdev_id); if (rc) { @@ -868,10 +909,10 @@ int odp_crypto_session_create(odp_crypto_session_params_t *params, entry->rte_session = (intptr_t)session; entry->cipher_xform = cipher_xform; entry->auth_xform = auth_xform; - entry->iv.length = params->iv.length; - entry->iv.data = params->iv.data; - entry->output_pool = params->output_pool; - entry->compl_queue = params->compl_queue; + entry->iv.length = param->iv.length; + entry->iv.data = param->iv.data; + entry->output_pool = param->output_pool; + entry->compl_queue = param->compl_queue; /* We're happy */ *session_out = (intptr_t)entry; @@ -903,7 +944,7 @@ int odp_crypto_session_destroy(odp_crypto_session_t session) return 0; } -int odp_crypto_operation(odp_crypto_op_params_t *params, +int odp_crypto_operation(odp_crypto_op_param_t *param, odp_bool_t *posted, odp_crypto_op_result_t *result) { @@ -917,10 +958,10 @@ int odp_crypto_operation(odp_crypto_op_params_t *params, uint8_t *data_addr, *aad_head; struct rte_crypto_op *op; uint16_t rc; - uint32_t plain_len, aad_len; - odp_bool_t pkt_allocated = 0; + uint32_t aad_len; + odp_bool_t allocated = false; - entry = (crypto_session_entry_t *)(intptr_t)params->session; + entry = (crypto_session_entry_t *)(intptr_t)param->session; if (entry == NULL) return -1; @@ -935,71 +976,71 @@ int odp_crypto_operation(odp_crypto_op_params_t *params, auth_xform = entry->auth_xform; /* Resolve output buffer */ - if (ODP_PACKET_INVALID == params->out_pkt && + if (ODP_PACKET_INVALID == param->out_pkt && ODP_POOL_INVALID != entry->output_pool) { - params->out_pkt = odp_packet_alloc(entry->output_pool, - odp_packet_len(params->pkt)); - pkt_allocated = 1; + param->out_pkt = odp_packet_alloc(entry->output_pool, + odp_packet_len(param->pkt)); + allocated = true; } - if (params->pkt != params->out_pkt) { - if (odp_unlikely(ODP_PACKET_INVALID == params->out_pkt)) + if (param->pkt != param->out_pkt) { + if (odp_unlikely(ODP_PACKET_INVALID == param->out_pkt)) ODP_ABORT(); - (void)odp_packet_copy_from_pkt(params->out_pkt, + int ret; + + ret = odp_packet_copy_from_pkt(param->out_pkt, 0, - params->pkt, + param->pkt, 0, - odp_packet_len(params->pkt)); - _odp_packet_copy_md_to_packet(params->pkt, params->out_pkt); - odp_packet_free(params->pkt); - params->pkt = ODP_PACKET_INVALID; + odp_packet_len(param->pkt)); + if (odp_unlikely(ret < 0)) + goto err; + + _odp_packet_copy_md_to_packet(param->pkt, param->out_pkt); + odp_packet_free(param->pkt); + param->pkt = ODP_PACKET_INVALID; } - data_addr = odp_packet_data(params->out_pkt); + data_addr = odp_packet_data(param->out_pkt); odp_spinlock_init(&global->lock); odp_spinlock_lock(&global->lock); op = rte_crypto_op_alloc(global->crypto_op_pool, RTE_CRYPTO_OP_TYPE_SYMMETRIC); if (op == NULL) { - if (pkt_allocated) - odp_packet_free(params->out_pkt); ODP_ERR("Failed to allocate crypto operation"); - return -1; + goto err; } odp_spinlock_unlock(&global->lock); /* Set crypto operation data parameters */ rte_crypto_op_attach_sym_session(op, rte_session); - op->sym->auth.digest.data = data_addr + params->hash_result_offset; + op->sym->auth.digest.data = data_addr + param->hash_result_offset; op->sym->auth.digest.phys_addr = - rte_pktmbuf_mtophys_offset((struct rte_mbuf *)params->out_pkt, - odp_packet_len(params->out_pkt) - + rte_pktmbuf_mtophys_offset((struct rte_mbuf *)param->out_pkt, + odp_packet_len(param->out_pkt) - auth_xform.auth.digest_length); op->sym->auth.digest.length = auth_xform.auth.digest_length; /* For SNOW3G algorithms, offset/length must be in bits */ if (auth_xform.auth.algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2) { - op->sym->auth.data.offset = params->auth_range.offset << 3; - op->sym->auth.data.length = params->auth_range.length << 3; + op->sym->auth.data.offset = param->auth_range.offset << 3; + op->sym->auth.data.length = param->auth_range.length << 3; } else { - op->sym->auth.data.offset = params->auth_range.offset; - op->sym->auth.data.length = params->auth_range.length; + op->sym->auth.data.offset = param->auth_range.offset; + op->sym->auth.data.length = param->auth_range.length; } - aad_head = data_addr + params->auth_range.offset; - plain_len = params->cipher_range.length; - aad_len = params->auth_range.length - plain_len; + aad_head = param->aad.ptr; + aad_len = param->aad.length; if (aad_len > 0) { op->sym->auth.aad.data = rte_malloc("aad", aad_len, 0); if (op->sym->auth.aad.data == NULL) { rte_crypto_op_free(op); - if (pkt_allocated) - odp_packet_free(params->out_pkt); ODP_ERR("Failed to allocate memory for AAD"); - return -1; + goto err; } memcpy(op->sym->auth.aad.data, aad_head, aad_len); @@ -1010,24 +1051,20 @@ int odp_crypto_operation(odp_crypto_op_params_t *params, if (entry->iv.length == 0) { rte_crypto_op_free(op); - if (pkt_allocated) - odp_packet_free(params->out_pkt); ODP_ERR("Wrong IV length"); - return -1; + goto err; } op->sym->cipher.iv.data = rte_malloc("iv", entry->iv.length, 0); if (op->sym->cipher.iv.data == NULL) { rte_crypto_op_free(op); - if (pkt_allocated) - odp_packet_free(params->out_pkt); ODP_ERR("Failed to allocate memory for IV"); - return -1; + goto err; } - if (params->override_iv_ptr) { + if (param->override_iv_ptr) { memcpy(op->sym->cipher.iv.data, - params->override_iv_ptr, + param->override_iv_ptr, entry->iv.length); } else if (entry->iv.data) { memcpy(op->sym->cipher.iv.data, @@ -1043,27 +1080,25 @@ int odp_crypto_operation(odp_crypto_op_params_t *params, /* For SNOW3G algorithms, offset/length must be in bits */ if (cipher_xform.cipher.algo == RTE_CRYPTO_CIPHER_SNOW3G_UEA2) { - op->sym->cipher.data.offset = params->cipher_range.offset << 3; - op->sym->cipher.data.length = params->cipher_range.length << 3; + op->sym->cipher.data.offset = param->cipher_range.offset << 3; + op->sym->cipher.data.length = param->cipher_range.length << 3; } else { - op->sym->cipher.data.offset = params->cipher_range.offset; - op->sym->cipher.data.length = params->cipher_range.length; + op->sym->cipher.data.offset = param->cipher_range.offset; + op->sym->cipher.data.length = param->cipher_range.length; } if (rc_cipher == ODP_CRYPTO_ALG_ERR_NONE && rc_auth == ODP_CRYPTO_ALG_ERR_NONE) { int queue_pair = odp_cpu_id(); - op->sym->m_src = (struct rte_mbuf *)params->out_pkt; + op->sym->m_src = (struct rte_mbuf *)param->out_pkt; rc = rte_cryptodev_enqueue_burst(rte_session->dev_id, queue_pair, &op, 1); if (rc == 0) { rte_crypto_op_free(op); - if (pkt_allocated) - odp_packet_free(params->out_pkt); ODP_ERR("Failed to enqueue packet"); - return -1; + goto err; } rc = rte_cryptodev_dequeue_burst(rte_session->dev_id, @@ -1071,18 +1106,16 @@ int odp_crypto_operation(odp_crypto_op_params_t *params, if (rc == 0) { rte_crypto_op_free(op); - if (pkt_allocated) - odp_packet_free(params->out_pkt); ODP_ERR("Failed to dequeue packet"); - return -1; + goto err; } - params->out_pkt = (odp_packet_t)op->sym->m_src; + param->out_pkt = (odp_packet_t)op->sym->m_src; } /* Fill in result */ - local_result.ctx = params->ctx; - local_result.pkt = params->out_pkt; + local_result.ctx = param->ctx; + local_result.pkt = param->out_pkt; local_result.cipher_status.alg_err = rc_cipher; local_result.cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE; local_result.auth_status.alg_err = rc_auth; @@ -1098,7 +1131,7 @@ int odp_crypto_operation(odp_crypto_op_params_t *params, odp_event_t completion_event; odp_crypto_generic_op_result_t *op_result; - completion_event = odp_packet_to_event(params->out_pkt); + completion_event = odp_packet_to_event(param->out_pkt); _odp_buffer_event_type_set( odp_buffer_from_event(completion_event), ODP_EVENT_CRYPTO_COMPL); @@ -1108,7 +1141,7 @@ int odp_crypto_operation(odp_crypto_op_params_t *params, op_result->result = local_result; if (odp_queue_enq(entry->compl_queue, completion_event)) { odp_event_free(completion_event); - return -1; + goto err; } /* Indicate to caller operation was async */ @@ -1116,14 +1149,21 @@ int odp_crypto_operation(odp_crypto_op_params_t *params, } else { /* Synchronous, simply return results */ if (!result) - return -1; + goto err; *result = local_result; /* Indicate to caller operation was sync */ *posted = 0; } - return 0; + +err: + if (allocated) { + odp_packet_free(param->out_pkt); + param->out_pkt = ODP_PACKET_INVALID; + } + + return -1; } int odp_crypto_term_global(void) diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index 68ad26b29..0d05e23f2 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -21,6 +21,8 @@ #include <stddef.h> #include <inttypes.h> +#include <odp/visibility_begin.h> + /* Fill in packet header field offsets for inline functions */ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { @@ -49,6 +51,8 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .rss_flag = PKT_RX_RSS_HASH }; +#include <odp/visibility_end.h> + struct rte_mbuf dummy; ODP_STATIC_ASSERT(sizeof(dummy.data_off) == sizeof(uint16_t), "data_off should be uint16_t"); @@ -77,24 +81,6 @@ static inline odp_packet_hdr_t *buf_to_packet_hdr(odp_buffer_t buf) return (odp_packet_hdr_t *)buf_hdl_to_hdr(buf); } -static inline void packet_parse_disable(odp_packet_hdr_t *pkt_hdr) -{ - pkt_hdr->p.input_flags.parsed_l2 = 1; - pkt_hdr->p.parsed_layers = LAYER_ALL; -} - -void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) -{ - /* Reset parser metadata before new parse */ - pkt_hdr->p.parsed_layers = LAYER_NONE; - pkt_hdr->p.error_flags.all = 0; - pkt_hdr->p.input_flags.all = 0; - pkt_hdr->p.output_flags.all = 0; - pkt_hdr->p.l2_offset = 0; - pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; - pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; -} - static odp_packet_t packet_alloc(pool_entry_t* pool, uint32_t len) { odp_packet_t pkt; @@ -194,7 +180,6 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) return -1; } - pkt_hdr->p.parsed_layers = LAYER_NONE; pkt_hdr->p.input_flags.all = 0; pkt_hdr->p.output_flags.all = 0; pkt_hdr->p.error_flags.all = 0; @@ -207,9 +192,6 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) pkt_hdr->input = ODP_PKTIO_INVALID; - /* Disable lazy parsing on user allocated packets */ - packet_parse_disable(pkt_hdr); - mb->port = 0xff; mb->pkt_len = len; mb->data_off = RTE_PKTMBUF_HEADROOM; @@ -623,8 +605,6 @@ void *odp_packet_l3_ptr(odp_packet_t pkt, uint32_t *len) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (pkt_hdr->p.parsed_layers < LAYER_L3) - packet_parse_layer(pkt_hdr, LAYER_L3); return packet_offset_to_ptr(pkt, len, pkt_hdr->p.l3_offset); } @@ -632,8 +612,6 @@ uint32_t odp_packet_l3_offset(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (pkt_hdr->p.parsed_layers < LAYER_L3) - packet_parse_layer(pkt_hdr, LAYER_L3); return pkt_hdr->p.l3_offset; } @@ -644,8 +622,6 @@ int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset) if (odp_unlikely(offset >= (odp_packet_len(pkt) - 1))) return -1; - if (pkt_hdr->p.parsed_layers < LAYER_L3) - packet_parse_layer(pkt_hdr, LAYER_L3); pkt_hdr->p.l3_offset = offset; return 0; } @@ -654,8 +630,6 @@ void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (pkt_hdr->p.parsed_layers < LAYER_L4) - packet_parse_layer(pkt_hdr, LAYER_L4); return packet_offset_to_ptr(pkt, len, pkt_hdr->p.l4_offset); } @@ -663,8 +637,6 @@ uint32_t odp_packet_l4_offset(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (pkt_hdr->p.parsed_layers < LAYER_L4) - packet_parse_layer(pkt_hdr, LAYER_L4); return pkt_hdr->p.l4_offset; } @@ -675,8 +647,6 @@ int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset) if (odp_unlikely(offset >= (odp_packet_len(pkt) - 1))) return -1; - if (pkt_hdr->p.parsed_layers < LAYER_L4) - packet_parse_layer(pkt_hdr, LAYER_L4); pkt_hdr->p.l4_offset = offset; return 0; } @@ -1148,12 +1118,11 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, uint8_t ihl = _ODP_IPV4HDR_IHL(ipv4->ver_ihl); uint16_t frag_offset; uint32_t dstaddr = odp_be_to_cpu_32(ipv4->dst_addr); - - prs->l3_len = odp_be_to_cpu_16(ipv4->tot_len); + uint32_t l3_len = odp_be_to_cpu_16(ipv4->tot_len); if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN) || odp_unlikely(ver != 4) || - (prs->l3_len > frame_len - *offset)) { + (l3_len > frame_len - *offset)) { prs->error_flags.ip_err = 1; return 0; } @@ -1190,13 +1159,12 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, const _odp_ipv6hdr_t *ipv6 = (const _odp_ipv6hdr_t *)*parseptr; const _odp_ipv6hdr_ext_t *ipv6ext; uint32_t dstaddr0 = odp_be_to_cpu_32(ipv6->dst_addr.u8[0]); - - prs->l3_len = odp_be_to_cpu_16(ipv6->payload_len) + - _ODP_IPV6HDR_LEN; + uint32_t l3_len = odp_be_to_cpu_16(ipv6->payload_len) + + _ODP_IPV6HDR_LEN; /* Basic sanity checks on IPv6 header */ if ((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || - prs->l3_len > frame_len - *offset) { + l3_len > frame_len - *offset) { prs->error_flags.ip_err = 1; return 0; } @@ -1257,9 +1225,6 @@ static inline void parse_tcp(packet_parser_t *prs, else if ((uint32_t)tcp->hl * 4 > sizeof(_odp_tcphdr_t)) prs->input_flags.tcpopt = 1; - prs->l4_len = prs->l3_len + - prs->l3_offset - prs->l4_offset; - if (offset) *offset += (uint32_t)tcp->hl * 4; *parseptr += (uint32_t)tcp->hl * 4; @@ -1274,13 +1239,8 @@ static inline void parse_udp(packet_parser_t *prs, const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr; uint32_t udplen = odp_be_to_cpu_16(udp->length); - if (udplen < sizeof(_odp_udphdr_t) || - udplen > (prs->l3_len + - prs->l4_offset - prs->l3_offset)) { + if (odp_unlikely(udplen < sizeof(_odp_udphdr_t))) prs->error_flags.udp_err = 1; - } - - prs->l4_len = udplen; if (offset) *offset += sizeof(_odp_udphdr_t); @@ -1294,197 +1254,172 @@ static inline void parse_udp(packet_parser_t *prs, * available from the ptr. */ int packet_parse_common(packet_parser_t *prs, const uint8_t *ptr, - uint32_t frame_len, uint32_t seg_len, layer_t layer) + uint32_t frame_len, uint32_t seg_len, + odp_pktio_parser_layer_t layer) { uint32_t offset; + uint16_t ethtype; const uint8_t *parseptr; + uint8_t ip_proto; + const _odp_ethhdr_t *eth; + uint16_t macaddr0, macaddr2, macaddr4; + const _odp_vlanhdr_t *vlan; - switch (prs->parsed_layers) { - case LAYER_NONE: - /* Fall through */ + if (layer == ODP_PKTIO_PARSER_LAYER_NONE) + return 0; - case LAYER_L2: - { - const _odp_ethhdr_t *eth; - uint16_t macaddr0, macaddr2, macaddr4; - const _odp_vlanhdr_t *vlan; - - offset = sizeof(_odp_ethhdr_t); - if (packet_parse_l2_not_done(prs)) - packet_parse_l2(prs, frame_len); - - eth = (const _odp_ethhdr_t *)ptr; - - /* Handle Ethernet broadcast/multicast addresses */ - macaddr0 = odp_be_to_cpu_16(*((const uint16_t *) - (const void *)eth)); - prs->input_flags.eth_mcast = (macaddr0 & 0x0100) == 0x0100; - - if (macaddr0 == 0xffff) { - macaddr2 = - odp_be_to_cpu_16(*((const uint16_t *) - (const void *)eth + 1)); - macaddr4 = - odp_be_to_cpu_16(*((const uint16_t *) - (const void *)eth + 2)); - prs->input_flags.eth_bcast = - (macaddr2 == 0xffff) && (macaddr4 == 0xffff); - } else { - prs->input_flags.eth_bcast = 0; - } + /* We only support Ethernet for now */ + prs->input_flags.eth = 1; + /* Assume valid L2 header, no CRC/FCS check in SW */ + prs->input_flags.l2 = 1; + /* Detect jumbo frames */ + if (frame_len > _ODP_ETH_LEN_MAX) + prs->input_flags.jumbo = 1; + + offset = sizeof(_odp_ethhdr_t); + eth = (const _odp_ethhdr_t *)ptr; + + /* Handle Ethernet broadcast/multicast addresses */ + macaddr0 = odp_be_to_cpu_16(*((const uint16_t *)(const void *)eth)); + prs->input_flags.eth_mcast = (macaddr0 & 0x0100) == 0x0100; + + if (macaddr0 == 0xffff) { + macaddr2 = + odp_be_to_cpu_16(*((const uint16_t *) + (const void *)eth + 1)); + macaddr4 = + odp_be_to_cpu_16(*((const uint16_t *) + (const void *)eth + 2)); + prs->input_flags.eth_bcast = + (macaddr2 == 0xffff) && (macaddr4 == 0xffff); + } else { + prs->input_flags.eth_bcast = 0; + } - /* Get Ethertype */ - prs->ethtype = odp_be_to_cpu_16(eth->type); - parseptr = (const uint8_t *)(eth + 1); + /* Get Ethertype */ + ethtype = odp_be_to_cpu_16(eth->type); + parseptr = (const uint8_t *)(eth + 1); - /* Check for SNAP vs. DIX */ - if (prs->ethtype < _ODP_ETH_LEN_MAX) { - prs->input_flags.snap = 1; - if (prs->ethtype > frame_len - offset) { - prs->error_flags.snap_len = 1; - goto parse_exit; - } - prs->ethtype = odp_be_to_cpu_16(*((const uint16_t *) - (uintptr_t) - (parseptr + 6))); - offset += 8; - parseptr += 8; + /* Check for SNAP vs. DIX */ + if (ethtype < _ODP_ETH_LEN_MAX) { + prs->input_flags.snap = 1; + if (ethtype > frame_len - offset) { + prs->error_flags.snap_len = 1; + goto parse_exit; } + ethtype = odp_be_to_cpu_16(*((const uint16_t *)(uintptr_t) + (parseptr + 6))); + offset += 8; + parseptr += 8; + } - /* Parse the VLAN header(s), if present */ - if (prs->ethtype == _ODP_ETHTYPE_VLAN_OUTER) { - prs->input_flags.vlan_qinq = 1; - prs->input_flags.vlan = 1; - - vlan = (const _odp_vlanhdr_t *)parseptr; - prs->ethtype = odp_be_to_cpu_16(vlan->type); - offset += sizeof(_odp_vlanhdr_t); - parseptr += sizeof(_odp_vlanhdr_t); - } + /* Parse the VLAN header(s), if present */ + if (ethtype == _ODP_ETHTYPE_VLAN_OUTER) { + prs->input_flags.vlan_qinq = 1; + prs->input_flags.vlan = 1; - if (prs->ethtype == _ODP_ETHTYPE_VLAN) { - prs->input_flags.vlan = 1; - vlan = (const _odp_vlanhdr_t *)parseptr; - prs->ethtype = odp_be_to_cpu_16(vlan->type); - offset += sizeof(_odp_vlanhdr_t); - parseptr += sizeof(_odp_vlanhdr_t); - } + vlan = (const _odp_vlanhdr_t *)parseptr; + ethtype = odp_be_to_cpu_16(vlan->type); + offset += sizeof(_odp_vlanhdr_t); + parseptr += sizeof(_odp_vlanhdr_t); + } - prs->l3_offset = offset; - prs->parsed_layers = LAYER_L2; - if (layer == LAYER_L2) - return prs->error_flags.all != 0; + if (ethtype == _ODP_ETHTYPE_VLAN) { + prs->input_flags.vlan = 1; + vlan = (const _odp_vlanhdr_t *)parseptr; + ethtype = odp_be_to_cpu_16(vlan->type); + offset += sizeof(_odp_vlanhdr_t); + parseptr += sizeof(_odp_vlanhdr_t); } - /* Fall through */ - case LAYER_L3: - { - offset = prs->l3_offset; - parseptr = (const uint8_t *)(ptr + offset); - /* Set l3_offset+flag only for known ethtypes */ - prs->input_flags.l3 = 1; - - /* Parse Layer 3 headers */ - switch (prs->ethtype) { - case _ODP_ETHTYPE_IPV4: - prs->input_flags.ipv4 = 1; - prs->ip_proto = parse_ipv4(prs, &parseptr, &offset, - frame_len); - break; + if (layer == ODP_PKTIO_PARSER_LAYER_L2) + return prs->error_flags.all != 0; - case _ODP_ETHTYPE_IPV6: - prs->input_flags.ipv6 = 1; - prs->ip_proto = parse_ipv6(prs, &parseptr, &offset, - frame_len, seg_len); - break; + /* Set l3_offset+flag only for known ethtypes */ + prs->l3_offset = offset; + prs->input_flags.l3 = 1; - case _ODP_ETHTYPE_ARP: - prs->input_flags.arp = 1; - prs->ip_proto = 255; /* Reserved invalid by IANA */ - break; + /* Parse Layer 3 headers */ + switch (ethtype) { + case _ODP_ETHTYPE_IPV4: + prs->input_flags.ipv4 = 1; + ip_proto = parse_ipv4(prs, &parseptr, &offset, frame_len); + break; - default: - prs->input_flags.l3 = 0; - prs->l3_offset = ODP_PACKET_OFFSET_INVALID; - prs->ip_proto = 255; /* Reserved invalid by IANA */ - } + case _ODP_ETHTYPE_IPV6: + prs->input_flags.ipv6 = 1; + ip_proto = parse_ipv6(prs, &parseptr, &offset, frame_len, + seg_len); + break; - /* Set l4_offset+flag only for known ip_proto */ - prs->l4_offset = offset; - prs->parsed_layers = LAYER_L3; - if (layer == LAYER_L3) - return prs->error_flags.all != 0; - } - /* Fall through */ + case _ODP_ETHTYPE_ARP: + prs->input_flags.arp = 1; + ip_proto = 255; /* Reserved invalid by IANA */ + break; - case LAYER_L4: - { - offset = prs->l4_offset; - parseptr = (const uint8_t *)(ptr + offset); - prs->input_flags.l4 = 1; + default: + prs->input_flags.l3 = 0; + prs->l3_offset = ODP_PACKET_OFFSET_INVALID; + ip_proto = 255; /* Reserved invalid by IANA */ + } - /* Parse Layer 4 headers */ - switch (prs->ip_proto) { - case _ODP_IPPROTO_ICMP: - prs->input_flags.icmp = 1; - break; + if (layer == ODP_PKTIO_PARSER_LAYER_L3) + return prs->error_flags.all != 0; - case _ODP_IPPROTO_TCP: - if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len)) - return -1; - prs->input_flags.tcp = 1; - parse_tcp(prs, &parseptr, NULL); - break; + /* Set l4_offset+flag only for known ip_proto */ + prs->l4_offset = offset; + prs->input_flags.l4 = 1; - case _ODP_IPPROTO_UDP: - if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len)) - return -1; - prs->input_flags.udp = 1; - parse_udp(prs, &parseptr, NULL); - break; + /* Parse Layer 4 headers */ + switch (ip_proto) { + case _ODP_IPPROTO_ICMPv4: + /* Fall through */ - case _ODP_IPPROTO_AH: - prs->input_flags.ipsec = 1; - prs->input_flags.ipsec_ah = 1; - break; + case _ODP_IPPROTO_ICMPv6: + prs->input_flags.icmp = 1; + break; - case _ODP_IPPROTO_ESP: - prs->input_flags.ipsec = 1; - prs->input_flags.ipsec_esp = 1; - break; + case _ODP_IPPROTO_TCP: + if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len)) + return -1; + prs->input_flags.tcp = 1; + parse_tcp(prs, &parseptr, NULL); + break; - case _ODP_IPPROTO_SCTP: - prs->input_flags.sctp = 1; - break; + case _ODP_IPPROTO_UDP: + if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len)) + return -1; + prs->input_flags.udp = 1; + parse_udp(prs, &parseptr, NULL); + break; - default: - prs->input_flags.l4 = 0; - prs->l4_offset = ODP_PACKET_OFFSET_INVALID; - break; - } + case _ODP_IPPROTO_AH: + prs->input_flags.ipsec = 1; + prs->input_flags.ipsec_ah = 1; + break; - prs->parsed_layers = LAYER_L4; + case _ODP_IPPROTO_ESP: + prs->input_flags.ipsec = 1; + prs->input_flags.ipsec_esp = 1; break; - } - case LAYER_ALL: + case _ODP_IPPROTO_SCTP: + prs->input_flags.sctp = 1; break; default: - ODP_ERR("Invalid parse layer: %d\n", (int)layer); - return -1; + prs->input_flags.l4 = 0; + prs->l4_offset = ODP_PACKET_OFFSET_INVALID; + break; } - - prs->parsed_layers = LAYER_ALL; - parse_exit: return prs->error_flags.all != 0; } - /** * Simple packet parser */ -int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, layer_t layer) +int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, + odp_pktio_parser_layer_t layer) { uint32_t seg_len = odp_packet_seg_len((odp_packet_t)pkt_hdr); uint32_t len = packet_len(pkt_hdr); diff --git a/platform/linux-dpdk/odp_packet_dpdk.c b/platform/linux-dpdk/odp_packet_dpdk.c index 26d32370b..d056a8242 100644 --- a/platform/linux-dpdk/odp_packet_dpdk.c +++ b/platform/linux-dpdk/odp_packet_dpdk.c @@ -369,6 +369,12 @@ static int recv_pkt_dpdk(pktio_entry_t *pktio_entry, int index, (struct rte_mbuf **)pkt_table, (uint16_t)RTE_MAX(len, min)); + if (pktio_entry->s.config.pktin.bit.ts_all || + pktio_entry->s.config.pktin.bit.ts_ptp) { + ts_val = odp_time_global(); + ts = &ts_val; + } + if (nb_rx == 0 && !pkt_dpdk->lockless_tx) { pool_entry_t *pool_entry = get_pool_entry(_odp_typeval(pktio_entry->s.pool)); @@ -382,14 +388,16 @@ static int recv_pkt_dpdk(pktio_entry_t *pktio_entry, int index, odp_ticketlock_unlock(&pkt_dpdk->rx_lock[index]); for (i = 0; i < nb_rx; ++i) { - _odp_packet_reset_parse(pkt_table[i]); - odp_packet_hdr(pkt_table[i])->input = pktio_entry->s.handle; - } + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt_table[i]); - if (pktio_entry->s.config.pktin.bit.ts_all || - pktio_entry->s.config.pktin.bit.ts_ptp) { - ts_val = odp_time_global(); - ts = &ts_val; + packet_parse_reset(pkt_hdr); + pkt_hdr->input = pktio_entry->s.handle; + + if (!pktio_cls_enabled(pktio_entry) && + pktio_entry->s.config.parser.layer) + packet_parse_layer(pkt_hdr, + pktio_entry->s.config.parser.layer); + packet_set_ts(pkt_hdr, ts); } if (odp_unlikely(min > len)) { diff --git a/platform/linux-dpdk/odp_packet_flags.c b/platform/linux-dpdk/odp_packet_flags.c index 20517e164..d8c68f021 100644 --- a/platform/linux-dpdk/odp_packet_flags.c +++ b/platform/linux-dpdk/odp_packet_flags.c @@ -8,26 +8,21 @@ #include <odp/api/packet_flags.h> #include <odp_packet_internal.h> -#define retflag(pkt, x, layer) do { \ +#define retflag(pkt, x) do { \ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); \ - if (pkt_hdr->p.parsed_layers < layer) \ - packet_parse_layer(pkt_hdr, layer); \ return pkt_hdr->p.x; \ } while (0) -#define setflag(pkt, x, v, layer) do { \ +#define setflag(pkt, x, v) do { \ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); \ - if (pkt_hdr->p.parsed_layers < layer) \ - packet_parse_layer(pkt_hdr, layer); \ - pkt_hdr->p.x = v & 1; \ + pkt_hdr->p.x = (v) & 1; \ } while (0) int odp_packet_has_error(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (packet_parse_not_complete(pkt_hdr)) - packet_parse_layer(pkt_hdr, LAYER_ALL); - return odp_packet_hdr(pkt)->p.error_flags.all != 0; + + return pkt_hdr->p.error_flags.all != 0; } /* Get Input Flags */ @@ -44,126 +39,117 @@ int odp_packet_has_l2_error(odp_packet_t pkt) int odp_packet_has_l3(odp_packet_t pkt) { - retflag(pkt, input_flags.l3, LAYER_L3); + retflag(pkt, input_flags.l3); } int odp_packet_has_l3_error(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (packet_parse_not_complete(pkt_hdr)) - packet_parse_layer(pkt_hdr, LAYER_ALL); - return pkt_hdr->p.error_flags.ip_err; } int odp_packet_has_l4(odp_packet_t pkt) { - retflag(pkt, input_flags.l4, LAYER_L4); + retflag(pkt, input_flags.l4); } int odp_packet_has_l4_error(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (pkt_hdr->p.parsed_layers < LAYER_L4) - packet_parse_layer(pkt_hdr, LAYER_L4); - return pkt_hdr->p.error_flags.tcp_err | pkt_hdr->p.error_flags.udp_err; } int odp_packet_has_eth_bcast(odp_packet_t pkt) { - retflag(pkt, input_flags.eth_bcast, LAYER_L2); + retflag(pkt, input_flags.eth_bcast); } int odp_packet_has_eth_mcast(odp_packet_t pkt) { - retflag(pkt, input_flags.eth_mcast, LAYER_L2); + retflag(pkt, input_flags.eth_mcast); } int odp_packet_has_vlan(odp_packet_t pkt) { - retflag(pkt, input_flags.vlan, LAYER_L2); + retflag(pkt, input_flags.vlan); } int odp_packet_has_vlan_qinq(odp_packet_t pkt) { - retflag(pkt, input_flags.vlan_qinq, LAYER_L2); + retflag(pkt, input_flags.vlan_qinq); } int odp_packet_has_arp(odp_packet_t pkt) { - retflag(pkt, input_flags.arp, LAYER_L3); + retflag(pkt, input_flags.arp); } int odp_packet_has_ipv4(odp_packet_t pkt) { - retflag(pkt, input_flags.ipv4, LAYER_L3); + retflag(pkt, input_flags.ipv4); } int odp_packet_has_ipv6(odp_packet_t pkt) { - retflag(pkt, input_flags.ipv6, LAYER_L3); + retflag(pkt, input_flags.ipv6); } int odp_packet_has_ip_bcast(odp_packet_t pkt) { - retflag(pkt, input_flags.ip_bcast, LAYER_L3); + retflag(pkt, input_flags.ip_bcast); } int odp_packet_has_ip_mcast(odp_packet_t pkt) { - retflag(pkt, input_flags.ip_mcast, LAYER_L3); + retflag(pkt, input_flags.ip_mcast); } int odp_packet_has_ipfrag(odp_packet_t pkt) { - retflag(pkt, input_flags.ipfrag, LAYER_L3); + retflag(pkt, input_flags.ipfrag); } int odp_packet_has_ipopt(odp_packet_t pkt) { - retflag(pkt, input_flags.ipopt, LAYER_L3); + retflag(pkt, input_flags.ipopt); } int odp_packet_has_ipsec(odp_packet_t pkt) { - retflag(pkt, input_flags.ipsec, LAYER_L4); + retflag(pkt, input_flags.ipsec); } int odp_packet_has_udp(odp_packet_t pkt) { - retflag(pkt, input_flags.udp, LAYER_L4); + retflag(pkt, input_flags.udp); } int odp_packet_has_tcp(odp_packet_t pkt) { - retflag(pkt, input_flags.tcp, LAYER_L4); + retflag(pkt, input_flags.tcp); } int odp_packet_has_sctp(odp_packet_t pkt) { - retflag(pkt, input_flags.sctp, LAYER_L4); + retflag(pkt, input_flags.sctp); } int odp_packet_has_icmp(odp_packet_t pkt) { - retflag(pkt, input_flags.icmp, LAYER_L4); + retflag(pkt, input_flags.icmp); } odp_packet_color_t odp_packet_color(odp_packet_t pkt) { - retflag(pkt, input_flags.color, LAYER_ALL); + retflag(pkt, input_flags.color); } void odp_packet_color_set(odp_packet_t pkt, odp_packet_color_t color) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (packet_parse_not_complete(pkt_hdr)) - packet_parse_layer(pkt_hdr, LAYER_ALL); - pkt_hdr->p.input_flags.color = color; } @@ -171,29 +157,23 @@ odp_bool_t odp_packet_drop_eligible(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (packet_parse_not_complete(pkt_hdr)) - packet_parse_layer(pkt_hdr, LAYER_ALL); - return !pkt_hdr->p.input_flags.nodrop; } void odp_packet_drop_eligible_set(odp_packet_t pkt, odp_bool_t drop) { - setflag(pkt, input_flags.nodrop, !drop, LAYER_ALL); + setflag(pkt, input_flags.nodrop, !drop); } int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt) { - retflag(pkt, output_flags.shaper_len_adj, LAYER_ALL); + retflag(pkt, output_flags.shaper_len_adj); } void odp_packet_shaper_len_adjust_set(odp_packet_t pkt, int8_t adj) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (packet_parse_not_complete(pkt_hdr)) - packet_parse_layer(pkt_hdr, LAYER_ALL); - pkt_hdr->p.output_flags.shaper_len_adj = adj; } @@ -201,107 +181,107 @@ void odp_packet_shaper_len_adjust_set(odp_packet_t pkt, int8_t adj) void odp_packet_has_l2_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.l2, val, LAYER_L2); + setflag(pkt, input_flags.l2, val); } void odp_packet_has_l3_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.l3, val, LAYER_L3); + setflag(pkt, input_flags.l3, val); } void odp_packet_has_l4_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.l4, val, LAYER_L4); + setflag(pkt, input_flags.l4, val); } void odp_packet_has_eth_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.eth, val, LAYER_L2); + setflag(pkt, input_flags.eth, val); } void odp_packet_has_eth_bcast_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.eth_bcast, val, LAYER_L2); + setflag(pkt, input_flags.eth_bcast, val); } void odp_packet_has_eth_mcast_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.eth_mcast, val, LAYER_L2); + setflag(pkt, input_flags.eth_mcast, val); } void odp_packet_has_jumbo_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.jumbo, val, LAYER_L2); + setflag(pkt, input_flags.jumbo, val); } void odp_packet_has_vlan_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.vlan, val, LAYER_L2); + setflag(pkt, input_flags.vlan, val); } void odp_packet_has_vlan_qinq_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.vlan_qinq, val, LAYER_L2); + setflag(pkt, input_flags.vlan_qinq, val); } void odp_packet_has_arp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.arp, val, LAYER_L3); + setflag(pkt, input_flags.arp, val); } void odp_packet_has_ipv4_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.ipv4, val, LAYER_L3); + setflag(pkt, input_flags.ipv4, val); } void odp_packet_has_ipv6_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.ipv6, val, LAYER_L3); + setflag(pkt, input_flags.ipv6, val); } void odp_packet_has_ip_bcast_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.ip_bcast, val, LAYER_L3); + setflag(pkt, input_flags.ip_bcast, val); } void odp_packet_has_ip_mcast_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.ip_mcast, val, LAYER_L3); + setflag(pkt, input_flags.ip_mcast, val); } void odp_packet_has_ipfrag_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.ipfrag, val, LAYER_L3); + setflag(pkt, input_flags.ipfrag, val); } void odp_packet_has_ipopt_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.ipopt, val, LAYER_L3); + setflag(pkt, input_flags.ipopt, val); } void odp_packet_has_ipsec_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.ipsec, val, LAYER_L4); + setflag(pkt, input_flags.ipsec, val); } void odp_packet_has_udp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.udp, val, LAYER_L4); + setflag(pkt, input_flags.udp, val); } void odp_packet_has_tcp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.tcp, val, LAYER_L4); + setflag(pkt, input_flags.tcp, val); } void odp_packet_has_sctp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.sctp, val, LAYER_L4); + setflag(pkt, input_flags.sctp, val); } void odp_packet_has_icmp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.icmp, val, LAYER_L4); + setflag(pkt, input_flags.icmp, val); } void odp_packet_has_ts_clr(odp_packet_t pkt) diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index e6d415954..0821cdcac 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -240,7 +240,8 @@ static int check_params(odp_pool_param_t *params) { odp_pool_capability_t capa; - odp_pool_capability(&capa); + if (odp_pool_capability(&capa) < 0) + return -1; switch (params->type) { case ODP_POOL_BUFFER: @@ -552,6 +553,8 @@ static odp_buffer_t buffer_alloc(pool_entry_t *pool) odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl) { + ODP_ASSERT(ODP_POOL_INVALID != pool_hdl); + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); return buffer_alloc(pool); @@ -560,6 +563,9 @@ odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl) int odp_buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int num) { int i; + + ODP_ASSERT(ODP_POOL_INVALID != pool_hdl); + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); for (i = 0; i < num; i++) { diff --git a/platform/linux-dpdk/odp_time.c b/platform/linux-dpdk/odp_time.c index c1ad973cc..6c066a5b5 100644 --- a/platform/linux-dpdk/odp_time.c +++ b/platform/linux-dpdk/odp_time.c @@ -13,105 +13,182 @@ #include <odp_time_internal.h> #include <rte_cycles.h> #include <string.h> +#include <inttypes.h> + +typedef uint64_t (*time_to_ns_fn) (odp_time_t time); +typedef odp_time_t (*time_cur_fn)(void); +typedef odp_time_t (*time_from_ns_fn) (uint64_t ns); +typedef uint64_t (*time_res_fn)(void); + +typedef struct time_handler_ { + time_to_ns_fn time_to_ns; + time_cur_fn time_cur; + time_from_ns_fn time_from_ns; + time_res_fn time_res; +} time_handler_t; + +typedef struct time_global_t { + struct timespec spec_start; + int use_hw; + uint64_t hw_start; + uint64_t hw_freq_hz; + /* DPDK specific */ + time_handler_t handler; + double tick_per_nsec; + double nsec_per_tick; +} time_global_t; + +static time_global_t global; + +/* + * Posix timespec based functions + */ + +static inline uint64_t time_spec_diff_nsec(struct timespec *t2, + struct timespec *t1) +{ + struct timespec diff; + uint64_t nsec; + + diff.tv_sec = t2->tv_sec - t1->tv_sec; + diff.tv_nsec = t2->tv_nsec - t1->tv_nsec; -typedef union { - odp_time_t ex; - struct timespec in; -} _odp_time_t; + if (diff.tv_nsec < 0) { + diff.tv_nsec += ODP_TIME_SEC_IN_NS; + diff.tv_sec -= 1; + } + + nsec = (diff.tv_sec * ODP_TIME_SEC_IN_NS) + diff.tv_nsec; -static odp_time_t start_time; -static time_handler_t time_handler; -double tick_per_nsec; -double nsec_per_tick; + return nsec; +} -static inline uint64_t time_local_res_dpdk(void) +static inline odp_time_t time_spec_cur(void) { - return rte_get_timer_hz(); + int ret; + odp_time_t time; + struct timespec sys_time; + + ret = clock_gettime(CLOCK_MONOTONIC_RAW, &sys_time); + if (odp_unlikely(ret != 0)) + ODP_ABORT("clock_gettime failed\n"); + + time.nsec = time_spec_diff_nsec(&sys_time, &global.spec_start); + + return time; } -static inline -uint64_t time_to_ns(odp_time_t time) +static inline uint64_t time_spec_res(void) { - uint64_t ns; + int ret; + struct timespec tres; - ns = time.tv_sec * ODP_TIME_SEC_IN_NS; - ns += time.tv_nsec; + ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); + if (odp_unlikely(ret != 0)) + ODP_ABORT("clock_getres failed\n"); - return ns; + return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec; } -static inline uint64_t time_to_ns_dpdk(odp_time_t time) +static inline uint64_t time_spec_to_ns(odp_time_t time) { - return (time.tv_sec * nsec_per_tick); + return time.nsec; } -static inline odp_time_t time_diff(odp_time_t t2, odp_time_t t1) +static inline odp_time_t time_spec_from_ns(uint64_t ns) { odp_time_t time; - time.tv_sec = t2.tv_sec - t1.tv_sec; - time.tv_nsec = t2.tv_nsec - t1.tv_nsec; - - if (time.tv_nsec < 0) { - time.tv_nsec += ODP_TIME_SEC_IN_NS; - --time.tv_sec; - } + time.nsec = ns; return time; } -static inline odp_time_t time_diff_dpdk(odp_time_t t2, odp_time_t t1) +/* + * HW time counter based functions + */ + +static inline odp_time_t time_hw_cur(void) { odp_time_t time; - time.tv_sec = t2.tv_sec - t1.tv_sec; + time.count = cpu_global_time() - global.hw_start; + return time; } -static inline odp_time_t time_curr(void) +static inline uint64_t time_hw_res(void) { - int ret; - _odp_time_t time; + /* Promise a bit lower resolution than average cycle counter + * frequency */ + return global.hw_freq_hz / 10; +} - ret = clock_gettime(CLOCK_MONOTONIC_RAW, &time.in); - if (odp_unlikely(ret != 0)) - ODP_ABORT("clock_gettime failed\n"); - return time.ex; +static inline uint64_t time_hw_to_ns(odp_time_t time) +{ + uint64_t nsec; + uint64_t freq_hz = global.hw_freq_hz; + uint64_t count = time.count; + uint64_t sec = 0; + + if (count >= freq_hz) { + sec = count / freq_hz; + count = count - sec * freq_hz; + } + + nsec = (ODP_TIME_SEC_IN_NS * count) / freq_hz; + + return (sec * ODP_TIME_SEC_IN_NS) + nsec; } -static inline odp_time_t time_curr_dpdk(void) +static inline odp_time_t time_hw_from_ns(uint64_t ns) { odp_time_t time; + uint64_t count; + uint64_t freq_hz = global.hw_freq_hz; + uint64_t sec = 0; + + if (ns >= ODP_TIME_SEC_IN_NS) { + sec = ns / ODP_TIME_SEC_IN_NS; + ns = ns - sec * ODP_TIME_SEC_IN_NS; + } + + count = sec * freq_hz; + count += (ns * freq_hz) / ODP_TIME_SEC_IN_NS; + + time.count = count; - time.tv_sec = rte_get_timer_cycles(); return time; } -static inline odp_time_t time_local(void) +/* + * Common functions + */ + +static inline odp_time_t time_cur(void) { - odp_time_t time; + if (global.use_hw) + return time_hw_cur(); - time = time_handler.time_curr(); - return time_handler.time_diff(time, start_time); + return time_spec_cur(); } -static inline int time_cmp(odp_time_t t2, odp_time_t t1) +static inline uint64_t time_res(void) { - if (t2.tv_sec < t1.tv_sec) - return -1; - - if (t2.tv_sec > t1.tv_sec) - return 1; + if (global.use_hw) + return time_hw_res(); - return t2.tv_nsec - t1.tv_nsec; + return time_spec_res(); } -static inline int time_cmp_dpdk(odp_time_t t2, odp_time_t t1) +static inline int time_cmp(odp_time_t t2, odp_time_t t1) { - if (t2.tv_sec < t1.tv_sec) - return -1; - if (t2.tv_sec > t1.tv_sec) + if (odp_likely(t2.u64 > t1.u64)) return 1; + + if (t2.u64 < t1.u64) + return -1; + return 0; } @@ -119,61 +196,67 @@ static inline odp_time_t time_sum(odp_time_t t1, odp_time_t t2) { odp_time_t time; - time.tv_sec = t2.tv_sec + t1.tv_sec; - time.tv_nsec = t2.tv_nsec + t1.tv_nsec; - - if (time.tv_nsec >= (long)ODP_TIME_SEC_IN_NS) { - time.tv_nsec -= ODP_TIME_SEC_IN_NS; - ++time.tv_sec; - } + time.u64 = t1.u64 + t2.u64; return time; } -static inline odp_time_t time_sum_dpdk(odp_time_t t1, odp_time_t t2) +static inline uint64_t time_to_ns(odp_time_t time) { - odp_time_t time; + if (global.use_hw) + return time_hw_to_ns(time); - time.tv_sec = t2.tv_sec + t1.tv_sec; - return time; + return time_spec_to_ns(time); +} + +static inline odp_time_t time_from_ns(uint64_t ns) +{ + if (global.use_hw) + return time_hw_from_ns(ns); + + return time_spec_from_ns(ns); +} + +static inline uint64_t time_res_dpdk(void) +{ + return rte_get_timer_hz(); } -static inline odp_time_t time_local_from_ns(uint64_t ns) +static inline uint64_t time_to_ns_dpdk(odp_time_t time) +{ + return (time.u64 * global.nsec_per_tick); +} + +static inline odp_time_t time_cur_dpdk(void) { odp_time_t time; - time.tv_sec = ns / ODP_TIME_SEC_IN_NS; - time.tv_nsec = ns - time.tv_sec * ODP_TIME_SEC_IN_NS; + time.u64 = rte_get_timer_cycles() - global.hw_start; return time; } -static inline odp_time_t time_local_from_ns_dpdk(uint64_t ns) +static inline odp_time_t time_from_ns_dpdk(uint64_t ns) { odp_time_t time; - time.tv_sec = ns * tick_per_nsec; + + time.u64 = ns * global.tick_per_nsec; + return time; } +static inline odp_time_t time_local(void) +{ + return global.handler.time_cur(); +} + static inline void time_wait_until(odp_time_t time) { odp_time_t cur; do { cur = time_local(); - } while (time_handler.time_cmp(time, cur) > 0); -} - -static inline uint64_t time_local_res(void) -{ - int ret; - struct timespec tres; - - ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); - if (odp_unlikely(ret != 0)) - ODP_ABORT("clock_getres failed\n"); - - return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec; + } while (time_cmp(time, cur) > 0); } odp_time_t odp_time_local(void) @@ -188,49 +271,53 @@ odp_time_t odp_time_global(void) odp_time_t odp_time_diff(odp_time_t t2, odp_time_t t1) { - return time_handler.time_diff(t2, t1); + odp_time_t time; + + time.u64 = t2.u64 - t1.u64; + + return time; } uint64_t odp_time_to_ns(odp_time_t time) { - return time_handler.time_to_ns(time); + return global.handler.time_to_ns(time); } odp_time_t odp_time_local_from_ns(uint64_t ns) { - return time_handler.time_local_from_ns(ns); + return global.handler.time_from_ns(ns); } odp_time_t odp_time_global_from_ns(uint64_t ns) { - return time_handler.time_local_from_ns(ns); + return global.handler.time_from_ns(ns); } int odp_time_cmp(odp_time_t t2, odp_time_t t1) { - return time_handler.time_cmp(t2, t1); + return time_cmp(t2, t1); } odp_time_t odp_time_sum(odp_time_t t1, odp_time_t t2) { - return time_handler.time_sum(t1, t2); + return time_sum(t1, t2); } uint64_t odp_time_local_res(void) { - return time_handler.time_local_res(); + return global.handler.time_res(); } uint64_t odp_time_global_res(void) { - return time_handler.time_local_res(); + return global.handler.time_res(); } void odp_time_wait_ns(uint64_t ns) { odp_time_t cur = time_local(); - odp_time_t wait = time_handler.time_local_from_ns(ns); - odp_time_t end_time = time_handler.time_sum(cur, wait); + odp_time_t wait = global.handler.time_from_ns(ns); + odp_time_t end_time = time_sum(cur, wait); time_wait_until(end_time); } @@ -240,31 +327,6 @@ void odp_time_wait_until(odp_time_t time) return time_wait_until(time); } -static uint64_t time_to_u64(odp_time_t time) -{ - int ret; - struct timespec tres; - uint64_t resolution; - - ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); - if (odp_unlikely(ret != 0)) - ODP_ABORT("clock_getres failed\n"); - - resolution = (uint64_t)tres.tv_nsec; - - return time_handler.time_to_ns(time) / resolution; -} - -static uint64_t time_to_u64_dpdk(odp_time_t time) -{ - return time.tv_sec; -} - -uint64_t odp_time_to_u64(odp_time_t time) -{ - return time_handler.time_to_u64(time); -} - static odp_bool_t is_invariant_tsc_supported(void) { FILE *file; @@ -311,37 +373,52 @@ static inline odp_bool_t is_dpdk_timer_cycles_support(void) int odp_time_init_global(void) { + int ret = 0; + + memset(&global, 0, sizeof(time_global_t)); + if (is_dpdk_timer_cycles_support()) { - time_handler.time_to_ns = time_to_ns_dpdk; - time_handler.time_diff = time_diff_dpdk; - time_handler.time_curr = time_curr_dpdk; - time_handler.time_cmp = time_cmp_dpdk; - time_handler.time_sum = time_sum_dpdk; - time_handler.time_local_from_ns = time_local_from_ns_dpdk; - time_handler.time_local_res = time_local_res_dpdk; - time_handler.time_to_u64 = time_to_u64_dpdk; - tick_per_nsec = (double)time_local_res_dpdk() / + global.handler.time_to_ns = time_to_ns_dpdk; + global.handler.time_cur = time_cur_dpdk; + global.handler.time_from_ns = time_from_ns_dpdk; + global.handler.time_res = time_res_dpdk; + global.tick_per_nsec = (double)time_res_dpdk() / (double)ODP_TIME_SEC_IN_NS; - nsec_per_tick = (double)ODP_TIME_SEC_IN_NS / - (double)time_local_res_dpdk(); - } else { - time_handler.time_to_ns = time_to_ns; - time_handler.time_diff = time_diff; - time_handler.time_curr = time_curr; - time_handler.time_cmp = time_cmp; - time_handler.time_sum = time_sum; - time_handler.time_local_from_ns = time_local_from_ns; - time_handler.time_local_res = time_local_res; - time_handler.time_to_u64 = time_to_u64; + global.nsec_per_tick = (double)ODP_TIME_SEC_IN_NS / + (double)time_res_dpdk(); + + global.hw_start = rte_get_timer_cycles(); + if (global.hw_start == 0) + return -1; + else + return 0; } - start_time = time_handler.time_curr(); - if (time_handler.time_cmp(start_time, ODP_TIME_NULL) == 0) { - ODP_ABORT("initiate odp time failed\n"); - return -1; - } else { + global.handler.time_to_ns = time_to_ns; + global.handler.time_cur = time_cur; + global.handler.time_from_ns = time_from_ns; + global.handler.time_res = time_res; + + if (cpu_has_global_time()) { + global.use_hw = 1; + global.hw_freq_hz = cpu_global_time_freq(); + + if (global.hw_freq_hz == 0) + return -1; + + printf("HW time counter freq: %" PRIu64 " hz\n\n", + global.hw_freq_hz); + + global.hw_start = cpu_global_time(); return 0; } + + global.spec_start.tv_sec = 0; + global.spec_start.tv_nsec = 0; + + ret = clock_gettime(CLOCK_MONOTONIC_RAW, &global.spec_start); + + return ret; } int odp_time_term_global(void) diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 9e8a0064e..8dcdebd29 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -9,6 +9,7 @@ AM_CFLAGS += -I$(top_srcdir)/include AM_CFLAGS += -I$(top_srcdir)/include/odp/arch/@ARCH_ABI@ AM_CFLAGS += -I$(top_builddir)/include AM_CFLAGS += -Iinclude +AM_CFLAGS += -D_ODP_PKTIO_IPC include_HEADERS = \ $(top_srcdir)/include/odp.h \ @@ -32,6 +33,7 @@ odpapiinclude_HEADERS = \ $(srcdir)/include/odp/api/cpumask.h \ $(srcdir)/include/odp/api/crypto.h \ $(srcdir)/include/odp/api/debug.h \ + $(srcdir)/include/odp/api/deprecated.h \ $(srcdir)/include/odp/api/errno.h \ $(srcdir)/include/odp/api/event.h \ $(srcdir)/include/odp/api/hash.h \ @@ -145,6 +147,7 @@ noinst_HEADERS = \ ${srcdir}/include/odp_schedule_if.h \ ${srcdir}/include/odp_sorted_list_internal.h \ ${srcdir}/include/odp_shm_internal.h \ + ${srcdir}/include/odp_time_internal.h \ ${srcdir}/include/odp_timer_internal.h \ ${srcdir}/include/odp_timer_wheel_internal.h \ ${srcdir}/include/odp_traffic_mngr_internal.h \ @@ -218,6 +221,12 @@ __LIB__libodp_linux_la_SOURCES = \ arch/@ARCH_DIR@/odp_cpu_arch.c \ arch/@ARCH_DIR@/odp_sysinfo_parse.c +__LIB__libodp_linux_la_LIBADD = $(ATOMIC_LIBS) + +if ARCH_IS_X86 +__LIB__libodp_linux_la_SOURCES += arch/@ARCH_DIR@/cpu_flags.c +endif + if HAVE_PCAP __LIB__libodp_linux_la_SOURCES += pktio/pcap.c endif @@ -225,9 +234,9 @@ endif # Create symlink for ABI header files. Application does not need to use the arch # specific include path for installed files. install-data-hook: - if [ -h $(prefix)/include/odp/api/abi ]; then \ - : \ + if [ -h $(DESTDIR)$(prefix)/include/odp/api/abi ]; then \ + : ; \ else \ - $(LN_S) -rf $(prefix)/include/odp/arch/@ARCH_ABI@/odp/api/abi \ - $(prefix)/include/odp/api/abi; \ + $(LN_S) -rf $(DESTDIR)$(prefix)/include/odp/arch/@ARCH_ABI@/odp/api/abi \ + $(DESTDIR)$(prefix)/include/odp/api/abi; \ fi diff --git a/platform/linux-generic/arch/arm/odp_cpu_arch.c b/platform/linux-generic/arch/arm/odp_cpu_arch.c index 2ac223e07..c31f90844 100644 --- a/platform/linux-generic/arch/arm/odp_cpu_arch.c +++ b/platform/linux-generic/arch/arm/odp_cpu_arch.c @@ -13,6 +13,7 @@ #include <odp/api/hints.h> #include <odp/api/system_info.h> #include <odp_debug_internal.h> +#include <odp_time_internal.h> #define GIGA 1000000000 @@ -46,3 +47,18 @@ uint64_t odp_cpu_cycles_resolution(void) { return 1; } + +int cpu_has_global_time(void) +{ + return 0; +} + +uint64_t cpu_global_time(void) +{ + return 0; +} + +uint64_t cpu_global_time_freq(void) +{ + return 0; +} diff --git a/platform/linux-generic/arch/arm/odp_sysinfo_parse.c b/platform/linux-generic/arch/arm/odp_sysinfo_parse.c index 53e2aaeaf..8ae2022ac 100644 --- a/platform/linux-generic/arch/arm/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/arm/odp_sysinfo_parse.c @@ -25,3 +25,7 @@ uint64_t odp_cpu_hz_current(int id ODP_UNUSED) { return 0; } + +void sys_info_print_arch(void) +{ +} diff --git a/platform/linux-generic/arch/default/odp_cpu_arch.c b/platform/linux-generic/arch/default/odp_cpu_arch.c index 2ac223e07..c31f90844 100644 --- a/platform/linux-generic/arch/default/odp_cpu_arch.c +++ b/platform/linux-generic/arch/default/odp_cpu_arch.c @@ -13,6 +13,7 @@ #include <odp/api/hints.h> #include <odp/api/system_info.h> #include <odp_debug_internal.h> +#include <odp_time_internal.h> #define GIGA 1000000000 @@ -46,3 +47,18 @@ uint64_t odp_cpu_cycles_resolution(void) { return 1; } + +int cpu_has_global_time(void) +{ + return 0; +} + +uint64_t cpu_global_time(void) +{ + return 0; +} + +uint64_t cpu_global_time_freq(void) +{ + return 0; +} diff --git a/platform/linux-generic/arch/default/odp_sysinfo_parse.c b/platform/linux-generic/arch/default/odp_sysinfo_parse.c index 53e2aaeaf..8ae2022ac 100644 --- a/platform/linux-generic/arch/default/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/default/odp_sysinfo_parse.c @@ -25,3 +25,7 @@ uint64_t odp_cpu_hz_current(int id ODP_UNUSED) { return 0; } + +void sys_info_print_arch(void) +{ +} diff --git a/platform/linux-generic/arch/mips64/odp_cpu_arch.c b/platform/linux-generic/arch/mips64/odp_cpu_arch.c index 646acf9c1..f7eafa0fb 100644 --- a/platform/linux-generic/arch/mips64/odp_cpu_arch.c +++ b/platform/linux-generic/arch/mips64/odp_cpu_arch.c @@ -7,6 +7,7 @@ #include <odp/api/cpu.h> #include <odp/api/hints.h> #include <odp/api/system_info.h> +#include <odp_time_internal.h> uint64_t odp_cpu_cycles(void) { @@ -29,3 +30,18 @@ uint64_t odp_cpu_cycles_resolution(void) { return 1; } + +int cpu_has_global_time(void) +{ + return 0; +} + +uint64_t cpu_global_time(void) +{ + return 0; +} + +uint64_t cpu_global_time_freq(void) +{ + return 0; +} diff --git a/platform/linux-generic/arch/mips64/odp_sysinfo_parse.c b/platform/linux-generic/arch/mips64/odp_sysinfo_parse.c index 407264b7f..d6f75f283 100644 --- a/platform/linux-generic/arch/mips64/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/mips64/odp_sysinfo_parse.c @@ -62,3 +62,7 @@ uint64_t odp_cpu_hz_current(int id ODP_UNUSED) { return 0; } + +void sys_info_print_arch(void) +{ +} diff --git a/platform/linux-generic/arch/powerpc/odp_cpu_arch.c b/platform/linux-generic/arch/powerpc/odp_cpu_arch.c index 2ac223e07..c31f90844 100644 --- a/platform/linux-generic/arch/powerpc/odp_cpu_arch.c +++ b/platform/linux-generic/arch/powerpc/odp_cpu_arch.c @@ -13,6 +13,7 @@ #include <odp/api/hints.h> #include <odp/api/system_info.h> #include <odp_debug_internal.h> +#include <odp_time_internal.h> #define GIGA 1000000000 @@ -46,3 +47,18 @@ uint64_t odp_cpu_cycles_resolution(void) { return 1; } + +int cpu_has_global_time(void) +{ + return 0; +} + +uint64_t cpu_global_time(void) +{ + return 0; +} + +uint64_t cpu_global_time_freq(void) +{ + return 0; +} diff --git a/platform/linux-generic/arch/powerpc/odp_sysinfo_parse.c b/platform/linux-generic/arch/powerpc/odp_sysinfo_parse.c index 3b88d55b6..bd4b9b429 100644 --- a/platform/linux-generic/arch/powerpc/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/powerpc/odp_sysinfo_parse.c @@ -61,3 +61,7 @@ uint64_t odp_cpu_hz_current(int id ODP_UNUSED) { return 0; } + +void sys_info_print_arch(void) +{ +} diff --git a/platform/linux-generic/arch/x86/cpu_flags.c b/platform/linux-generic/arch/x86/cpu_flags.c new file mode 100644 index 000000000..a492a35bf --- /dev/null +++ b/platform/linux-generic/arch/x86/cpu_flags.c @@ -0,0 +1,368 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <arch/x86/cpu_flags.h> +#include <odp_debug_internal.h> +#include <odp_time_internal.h> +#include <stdio.h> +#include <stdint.h> + +enum rte_cpu_flag_t { + /* (EAX 01h) ECX features*/ + RTE_CPUFLAG_SSE3 = 0, /**< SSE3 */ + RTE_CPUFLAG_PCLMULQDQ, /**< PCLMULQDQ */ + RTE_CPUFLAG_DTES64, /**< DTES64 */ + RTE_CPUFLAG_MONITOR, /**< MONITOR */ + RTE_CPUFLAG_DS_CPL, /**< DS_CPL */ + RTE_CPUFLAG_VMX, /**< VMX */ + RTE_CPUFLAG_SMX, /**< SMX */ + RTE_CPUFLAG_EIST, /**< EIST */ + RTE_CPUFLAG_TM2, /**< TM2 */ + RTE_CPUFLAG_SSSE3, /**< SSSE3 */ + RTE_CPUFLAG_CNXT_ID, /**< CNXT_ID */ + RTE_CPUFLAG_FMA, /**< FMA */ + RTE_CPUFLAG_CMPXCHG16B, /**< CMPXCHG16B */ + RTE_CPUFLAG_XTPR, /**< XTPR */ + RTE_CPUFLAG_PDCM, /**< PDCM */ + RTE_CPUFLAG_PCID, /**< PCID */ + RTE_CPUFLAG_DCA, /**< DCA */ + RTE_CPUFLAG_SSE4_1, /**< SSE4_1 */ + RTE_CPUFLAG_SSE4_2, /**< SSE4_2 */ + RTE_CPUFLAG_X2APIC, /**< X2APIC */ + RTE_CPUFLAG_MOVBE, /**< MOVBE */ + RTE_CPUFLAG_POPCNT, /**< POPCNT */ + RTE_CPUFLAG_TSC_DEADLINE, /**< TSC_DEADLINE */ + RTE_CPUFLAG_AES, /**< AES */ + RTE_CPUFLAG_XSAVE, /**< XSAVE */ + RTE_CPUFLAG_OSXSAVE, /**< OSXSAVE */ + RTE_CPUFLAG_AVX, /**< AVX */ + RTE_CPUFLAG_F16C, /**< F16C */ + RTE_CPUFLAG_RDRAND, /**< RDRAND */ + + /* (EAX 01h) EDX features */ + RTE_CPUFLAG_FPU, /**< FPU */ + RTE_CPUFLAG_VME, /**< VME */ + RTE_CPUFLAG_DE, /**< DE */ + RTE_CPUFLAG_PSE, /**< PSE */ + RTE_CPUFLAG_TSC, /**< TSC */ + RTE_CPUFLAG_MSR, /**< MSR */ + RTE_CPUFLAG_PAE, /**< PAE */ + RTE_CPUFLAG_MCE, /**< MCE */ + RTE_CPUFLAG_CX8, /**< CX8 */ + RTE_CPUFLAG_APIC, /**< APIC */ + RTE_CPUFLAG_SEP, /**< SEP */ + RTE_CPUFLAG_MTRR, /**< MTRR */ + RTE_CPUFLAG_PGE, /**< PGE */ + RTE_CPUFLAG_MCA, /**< MCA */ + RTE_CPUFLAG_CMOV, /**< CMOV */ + RTE_CPUFLAG_PAT, /**< PAT */ + RTE_CPUFLAG_PSE36, /**< PSE36 */ + RTE_CPUFLAG_PSN, /**< PSN */ + RTE_CPUFLAG_CLFSH, /**< CLFSH */ + RTE_CPUFLAG_DS, /**< DS */ + RTE_CPUFLAG_ACPI, /**< ACPI */ + RTE_CPUFLAG_MMX, /**< MMX */ + RTE_CPUFLAG_FXSR, /**< FXSR */ + RTE_CPUFLAG_SSE, /**< SSE */ + RTE_CPUFLAG_SSE2, /**< SSE2 */ + RTE_CPUFLAG_SS, /**< SS */ + RTE_CPUFLAG_HTT, /**< HTT */ + RTE_CPUFLAG_TM, /**< TM */ + RTE_CPUFLAG_PBE, /**< PBE */ + + /* (EAX 06h) EAX features */ + RTE_CPUFLAG_DIGTEMP, /**< DIGTEMP */ + RTE_CPUFLAG_TRBOBST, /**< TRBOBST */ + RTE_CPUFLAG_ARAT, /**< ARAT */ + RTE_CPUFLAG_PLN, /**< PLN */ + RTE_CPUFLAG_ECMD, /**< ECMD */ + RTE_CPUFLAG_PTM, /**< PTM */ + + /* (EAX 06h) ECX features */ + RTE_CPUFLAG_MPERF_APERF_MSR, /**< MPERF_APERF_MSR */ + RTE_CPUFLAG_ACNT2, /**< ACNT2 */ + RTE_CPUFLAG_ENERGY_EFF, /**< ENERGY_EFF */ + + /* (EAX 07h, ECX 0h) EBX features */ + RTE_CPUFLAG_FSGSBASE, /**< FSGSBASE */ + RTE_CPUFLAG_BMI1, /**< BMI1 */ + RTE_CPUFLAG_HLE, /**< Hardware Lock elision */ + RTE_CPUFLAG_AVX2, /**< AVX2 */ + RTE_CPUFLAG_SMEP, /**< SMEP */ + RTE_CPUFLAG_BMI2, /**< BMI2 */ + RTE_CPUFLAG_ERMS, /**< ERMS */ + RTE_CPUFLAG_INVPCID, /**< INVPCID */ + RTE_CPUFLAG_RTM, /**< Transactional memory */ + RTE_CPUFLAG_AVX512F, /**< AVX512F */ + + /* (EAX 80000001h) ECX features */ + RTE_CPUFLAG_LAHF_SAHF, /**< LAHF_SAHF */ + RTE_CPUFLAG_LZCNT, /**< LZCNT */ + + /* (EAX 80000001h) EDX features */ + RTE_CPUFLAG_SYSCALL, /**< SYSCALL */ + RTE_CPUFLAG_XD, /**< XD */ + RTE_CPUFLAG_1GB_PG, /**< 1GB_PG */ + RTE_CPUFLAG_RDTSCP, /**< RDTSCP */ + RTE_CPUFLAG_EM64T, /**< EM64T */ + + /* (EAX 80000007h) EDX features */ + RTE_CPUFLAG_INVTSC, /**< INVTSC */ + + /* The last item */ + RTE_CPUFLAG_NUMFLAGS, /**< This should always be the last! */ +}; + +enum cpu_register_t { + RTE_REG_EAX = 0, + RTE_REG_EBX, + RTE_REG_ECX, + RTE_REG_EDX, +}; + +typedef uint32_t cpuid_registers_t[4]; + +/** + * Struct to hold a processor feature entry + */ +struct feature_entry { + uint32_t leaf; /**< cpuid leaf */ + uint32_t subleaf; /**< cpuid subleaf */ + uint32_t reg; /**< cpuid register */ + uint32_t bit; /**< cpuid register bit */ +#define CPU_FLAG_NAME_MAX_LEN 64 + char name[CPU_FLAG_NAME_MAX_LEN]; /**< String for printing */ +}; + +#define FEAT_DEF(name, leaf, subleaf, reg, bit) \ + [RTE_CPUFLAG_##name] = {leaf, subleaf, reg, bit, #name }, + +static const struct feature_entry cpu_feature_table[] = { + FEAT_DEF(SSE3, 0x00000001, 0, RTE_REG_ECX, 0) + FEAT_DEF(PCLMULQDQ, 0x00000001, 0, RTE_REG_ECX, 1) + FEAT_DEF(DTES64, 0x00000001, 0, RTE_REG_ECX, 2) + FEAT_DEF(MONITOR, 0x00000001, 0, RTE_REG_ECX, 3) + FEAT_DEF(DS_CPL, 0x00000001, 0, RTE_REG_ECX, 4) + FEAT_DEF(VMX, 0x00000001, 0, RTE_REG_ECX, 5) + FEAT_DEF(SMX, 0x00000001, 0, RTE_REG_ECX, 6) + FEAT_DEF(EIST, 0x00000001, 0, RTE_REG_ECX, 7) + FEAT_DEF(TM2, 0x00000001, 0, RTE_REG_ECX, 8) + FEAT_DEF(SSSE3, 0x00000001, 0, RTE_REG_ECX, 9) + FEAT_DEF(CNXT_ID, 0x00000001, 0, RTE_REG_ECX, 10) + FEAT_DEF(FMA, 0x00000001, 0, RTE_REG_ECX, 12) + FEAT_DEF(CMPXCHG16B, 0x00000001, 0, RTE_REG_ECX, 13) + FEAT_DEF(XTPR, 0x00000001, 0, RTE_REG_ECX, 14) + FEAT_DEF(PDCM, 0x00000001, 0, RTE_REG_ECX, 15) + FEAT_DEF(PCID, 0x00000001, 0, RTE_REG_ECX, 17) + FEAT_DEF(DCA, 0x00000001, 0, RTE_REG_ECX, 18) + FEAT_DEF(SSE4_1, 0x00000001, 0, RTE_REG_ECX, 19) + FEAT_DEF(SSE4_2, 0x00000001, 0, RTE_REG_ECX, 20) + FEAT_DEF(X2APIC, 0x00000001, 0, RTE_REG_ECX, 21) + FEAT_DEF(MOVBE, 0x00000001, 0, RTE_REG_ECX, 22) + FEAT_DEF(POPCNT, 0x00000001, 0, RTE_REG_ECX, 23) + FEAT_DEF(TSC_DEADLINE, 0x00000001, 0, RTE_REG_ECX, 24) + FEAT_DEF(AES, 0x00000001, 0, RTE_REG_ECX, 25) + FEAT_DEF(XSAVE, 0x00000001, 0, RTE_REG_ECX, 26) + FEAT_DEF(OSXSAVE, 0x00000001, 0, RTE_REG_ECX, 27) + FEAT_DEF(AVX, 0x00000001, 0, RTE_REG_ECX, 28) + FEAT_DEF(F16C, 0x00000001, 0, RTE_REG_ECX, 29) + FEAT_DEF(RDRAND, 0x00000001, 0, RTE_REG_ECX, 30) + + FEAT_DEF(FPU, 0x00000001, 0, RTE_REG_EDX, 0) + FEAT_DEF(VME, 0x00000001, 0, RTE_REG_EDX, 1) + FEAT_DEF(DE, 0x00000001, 0, RTE_REG_EDX, 2) + FEAT_DEF(PSE, 0x00000001, 0, RTE_REG_EDX, 3) + FEAT_DEF(TSC, 0x00000001, 0, RTE_REG_EDX, 4) + FEAT_DEF(MSR, 0x00000001, 0, RTE_REG_EDX, 5) + FEAT_DEF(PAE, 0x00000001, 0, RTE_REG_EDX, 6) + FEAT_DEF(MCE, 0x00000001, 0, RTE_REG_EDX, 7) + FEAT_DEF(CX8, 0x00000001, 0, RTE_REG_EDX, 8) + FEAT_DEF(APIC, 0x00000001, 0, RTE_REG_EDX, 9) + FEAT_DEF(SEP, 0x00000001, 0, RTE_REG_EDX, 11) + FEAT_DEF(MTRR, 0x00000001, 0, RTE_REG_EDX, 12) + FEAT_DEF(PGE, 0x00000001, 0, RTE_REG_EDX, 13) + FEAT_DEF(MCA, 0x00000001, 0, RTE_REG_EDX, 14) + FEAT_DEF(CMOV, 0x00000001, 0, RTE_REG_EDX, 15) + FEAT_DEF(PAT, 0x00000001, 0, RTE_REG_EDX, 16) + FEAT_DEF(PSE36, 0x00000001, 0, RTE_REG_EDX, 17) + FEAT_DEF(PSN, 0x00000001, 0, RTE_REG_EDX, 18) + FEAT_DEF(CLFSH, 0x00000001, 0, RTE_REG_EDX, 19) + FEAT_DEF(DS, 0x00000001, 0, RTE_REG_EDX, 21) + FEAT_DEF(ACPI, 0x00000001, 0, RTE_REG_EDX, 22) + FEAT_DEF(MMX, 0x00000001, 0, RTE_REG_EDX, 23) + FEAT_DEF(FXSR, 0x00000001, 0, RTE_REG_EDX, 24) + FEAT_DEF(SSE, 0x00000001, 0, RTE_REG_EDX, 25) + FEAT_DEF(SSE2, 0x00000001, 0, RTE_REG_EDX, 26) + FEAT_DEF(SS, 0x00000001, 0, RTE_REG_EDX, 27) + FEAT_DEF(HTT, 0x00000001, 0, RTE_REG_EDX, 28) + FEAT_DEF(TM, 0x00000001, 0, RTE_REG_EDX, 29) + FEAT_DEF(PBE, 0x00000001, 0, RTE_REG_EDX, 31) + + FEAT_DEF(DIGTEMP, 0x00000006, 0, RTE_REG_EAX, 0) + FEAT_DEF(TRBOBST, 0x00000006, 0, RTE_REG_EAX, 1) + FEAT_DEF(ARAT, 0x00000006, 0, RTE_REG_EAX, 2) + FEAT_DEF(PLN, 0x00000006, 0, RTE_REG_EAX, 4) + FEAT_DEF(ECMD, 0x00000006, 0, RTE_REG_EAX, 5) + FEAT_DEF(PTM, 0x00000006, 0, RTE_REG_EAX, 6) + + FEAT_DEF(MPERF_APERF_MSR, 0x00000006, 0, RTE_REG_ECX, 0) + FEAT_DEF(ACNT2, 0x00000006, 0, RTE_REG_ECX, 1) + FEAT_DEF(ENERGY_EFF, 0x00000006, 0, RTE_REG_ECX, 3) + + FEAT_DEF(FSGSBASE, 0x00000007, 0, RTE_REG_EBX, 0) + FEAT_DEF(BMI1, 0x00000007, 0, RTE_REG_EBX, 2) + FEAT_DEF(HLE, 0x00000007, 0, RTE_REG_EBX, 4) + FEAT_DEF(AVX2, 0x00000007, 0, RTE_REG_EBX, 5) + FEAT_DEF(SMEP, 0x00000007, 0, RTE_REG_EBX, 6) + FEAT_DEF(BMI2, 0x00000007, 0, RTE_REG_EBX, 7) + FEAT_DEF(ERMS, 0x00000007, 0, RTE_REG_EBX, 8) + FEAT_DEF(INVPCID, 0x00000007, 0, RTE_REG_EBX, 10) + FEAT_DEF(RTM, 0x00000007, 0, RTE_REG_EBX, 11) + FEAT_DEF(AVX512F, 0x00000007, 0, RTE_REG_EBX, 16) + + FEAT_DEF(LAHF_SAHF, 0x80000001, 0, RTE_REG_ECX, 0) + FEAT_DEF(LZCNT, 0x80000001, 0, RTE_REG_ECX, 4) + + FEAT_DEF(SYSCALL, 0x80000001, 0, RTE_REG_EDX, 11) + FEAT_DEF(XD, 0x80000001, 0, RTE_REG_EDX, 20) + FEAT_DEF(1GB_PG, 0x80000001, 0, RTE_REG_EDX, 26) + FEAT_DEF(RDTSCP, 0x80000001, 0, RTE_REG_EDX, 27) + FEAT_DEF(EM64T, 0x80000001, 0, RTE_REG_EDX, 29) + + FEAT_DEF(INVTSC, 0x80000007, 0, RTE_REG_EDX, 8) +}; + +/* + * Execute CPUID instruction and get contents of a specific register + * + * This function, when compiled with GCC, will generate architecture-neutral + * code, as per GCC manual. + */ +static void cpu_get_features(uint32_t leaf, uint32_t subleaf, + cpuid_registers_t out) +{ +#if defined(__i386__) && defined(__PIC__) + /* %ebx is a forbidden register if we compile with -fPIC or -fPIE */ + __asm__ __volatile__("movl %%ebx,%0 ; cpuid ; xchgl %%ebx,%0" + : "=r" (out[RTE_REG_EBX]), + "=a" (out[RTE_REG_EAX]), + "=c" (out[RTE_REG_ECX]), + "=d" (out[RTE_REG_EDX]) + : "a" (leaf), "c" (subleaf)); +#else + __asm__ __volatile__("cpuid" + : "=a" (out[RTE_REG_EAX]), + "=b" (out[RTE_REG_EBX]), + "=c" (out[RTE_REG_ECX]), + "=d" (out[RTE_REG_EDX]) + : "a" (leaf), "c" (subleaf)); +#endif +} + +static int cpu_get_flag_enabled(enum rte_cpu_flag_t feature) +{ + const struct feature_entry *feat; + cpuid_registers_t regs; + + if (feature >= RTE_CPUFLAG_NUMFLAGS) + /* Flag does not match anything in the feature tables */ + return -1; + + feat = &cpu_feature_table[feature]; + + if (!feat->leaf) + /* This entry in the table wasn't filled out! */ + return -1; + + cpu_get_features(feat->leaf & 0xffff0000, 0, regs); + if (((regs[RTE_REG_EAX] ^ feat->leaf) & 0xffff0000) || + regs[RTE_REG_EAX] < feat->leaf) + return 0; + + /* get the cpuid leaf containing the desired feature */ + cpu_get_features(feat->leaf, feat->subleaf, regs); + + /* check if the feature is enabled */ + return (regs[feat->reg] >> feat->bit) & 1; +} + +static const char *cpu_get_flag_name(enum rte_cpu_flag_t feature) +{ + if (feature >= RTE_CPUFLAG_NUMFLAGS) + return NULL; + return cpu_feature_table[feature].name; +} + +void cpu_flags_print_all(void) +{ + int len, i; + int max_str = 1024; + int max_len = max_str - 1; + char str[max_str]; + + len = snprintf(str, max_len, "\nCPU features supported:\n"); + + for (i = 0; i < RTE_CPUFLAG_NUMFLAGS; i++) { + if (cpu_get_flag_enabled(i) > 0) + len += snprintf(&str[len], max_len - len, "%s ", + cpu_get_flag_name(i)); + } + + len += snprintf(&str[len], max_len - len, + "\n\nCPU features NOT supported:\n"); + + for (i = 0; i < RTE_CPUFLAG_NUMFLAGS; i++) { + if (cpu_get_flag_enabled(i) <= 0) + len += snprintf(&str[len], max_len - len, "%s ", + cpu_get_flag_name(i)); + } + + len += snprintf(&str[len], max_len - len, "\n\n"); + + str[len] = '\0'; + ODP_PRINT("%s", str); +} + +int cpu_has_global_time(void) +{ + if (cpu_get_flag_enabled(RTE_CPUFLAG_INVTSC) > 0) + return 1; + + return 0; +} diff --git a/platform/linux-generic/arch/x86/cpu_flags.h b/platform/linux-generic/arch/x86/cpu_flags.h new file mode 100644 index 000000000..f709ca08b --- /dev/null +++ b/platform/linux-generic/arch/x86/cpu_flags.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_CPU_FLAGS_H_ +#define ODP_PLAT_CPU_FLAGS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +void cpu_flags_print_all(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/x86/odp_cpu_arch.c b/platform/linux-generic/arch/x86/odp_cpu_arch.c index c8cf27b65..b1da428b9 100644 --- a/platform/linux-generic/arch/x86/odp_cpu_arch.c +++ b/platform/linux-generic/arch/x86/odp_cpu_arch.c @@ -3,7 +3,14 @@ * * SPDX-License-Identifier: BSD-3-Clause */ + +#include <odp_posix_extensions.h> + #include <odp/api/cpu.h> +#include <odp_time_internal.h> +#include <odp_debug_internal.h> + +#include <time.h> uint64_t odp_cpu_cycles(void) { @@ -31,3 +38,64 @@ uint64_t odp_cpu_cycles_resolution(void) { return 1; } + +uint64_t cpu_global_time(void) +{ + return odp_cpu_cycles(); +} + +#define SEC_IN_NS 1000000000ULL + +/* Measure TSC frequency. Frequency information registers are defined for x86, + * but those are often not enumerated. */ +uint64_t cpu_global_time_freq(void) +{ + struct timespec sleep, ts1, ts2; + uint64_t t1, t2, ts_nsec, cycles, hz; + int i; + uint64_t avg = 0; + int rounds = 3; + int warm_up = 1; + + for (i = 0; i < rounds; i++) { + sleep.tv_sec = 0; + + if (warm_up) + sleep.tv_nsec = SEC_IN_NS / 1000; + else + sleep.tv_nsec = SEC_IN_NS / 4; + + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts1)) { + ODP_DBG("clock_gettime failed\n"); + return 0; + } + + t1 = cpu_global_time(); + + if (nanosleep(&sleep, NULL) < 0) { + ODP_DBG("nanosleep failed\n"); + return 0; + } + + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts2)) { + ODP_DBG("clock_gettime failed\n"); + return 0; + } + + t2 = cpu_global_time(); + + ts_nsec = (ts2.tv_sec - ts1.tv_sec) * SEC_IN_NS; + ts_nsec += ts2.tv_nsec - ts1.tv_nsec; + + cycles = t2 - t1; + + hz = (cycles * SEC_IN_NS) / ts_nsec; + + if (warm_up) + warm_up = 0; + else + avg += hz; + } + + return avg / (rounds - 1); +} diff --git a/platform/linux-generic/arch/x86/odp_sysinfo_parse.c b/platform/linux-generic/arch/x86/odp_sysinfo_parse.c index 96127ec67..d77165a41 100644 --- a/platform/linux-generic/arch/x86/odp_sysinfo_parse.c +++ b/platform/linux-generic/arch/x86/odp_sysinfo_parse.c @@ -5,6 +5,7 @@ */ #include <odp_internal.h> +#include <arch/x86/cpu_flags.h> #include <string.h> int cpuinfo_parser(FILE *file, system_info_t *sysinfo) @@ -73,3 +74,8 @@ uint64_t odp_cpu_hz_current(int id) return 0; } + +void sys_info_print_arch(void) +{ + cpu_flags_print_all(); +} diff --git a/platform/linux-generic/include/odp/api/debug.h b/platform/linux-generic/include/odp/api/debug.h index 7db143395..bef2fd0eb 100644 --- a/platform/linux-generic/include/odp/api/debug.h +++ b/platform/linux-generic/include/odp/api/debug.h @@ -19,17 +19,44 @@ extern "C" { #include <odp/api/spec/debug.h> -#if defined(__GNUC__) && !defined(__clang__) - -#if __GNUC__ < 4 || (__GNUC__ == 4 && (__GNUC_MINOR__ < 6)) - /** - * @internal _Static_assert was only added in GCC 4.6. Provide a weak replacement - * for previous versions. + * @internal _Static_assert was only added in GCC 4.6 and the C++ version + * static_assert for g++ 6 and above. Provide a weak replacement for previous + * versions. */ -#define _Static_assert(e, s) (extern int (*static_assert_checker(void)) \ - [sizeof(struct { unsigned int error_if_negative:(e) ? 1 : -1; })]) +#define _odp_merge(a, b) a##b +/** @internal */ +#define _odp_label(a) _odp_merge(_ODP_SASSERT_, a) +/** @internal */ +#define _ODP_SASSERT _odp_label(__COUNTER__) +/** @internal */ +#define _ODP_SASSERT_ENUM(e) { _ODP_SASSERT = 1 / !!(e) } +/** @internal */ +#define _odp_static_assert(e, s) enum _ODP_SASSERT_ENUM(e) + +#if defined(__clang__) +#if defined(__cplusplus) +#if !__has_feature(cxx_static_assert) && !defined(static_assert) +/** @internal */ +#define static_assert(e, s) _odp_static_assert(e, s) +#endif +#elif !__has_feature(c_static_assert) && !defined(_Static_assert) +/** @internal */ +#define _Static_assert(e, s) _odp_static_assert(e, s) +#endif +#elif defined(__GNUC__) +#if __GNUC__ < 4 || (__GNUC__ == 4 && (__GNUC_MINOR__ < 6)) || \ + (__GNUC__ < 6 && defined(__cplusplus)) +#if defined(__cplusplus) +#if !defined(static_assert) +/** @intenral */ +#define static_assert(e, s) _odp_static_assert(e, s) +#endif +#elif !defined(_Static_assert) +/** @internal */ +#define _Static_assert(e, s) _odp_static_assert(e, s) +#endif #endif #endif @@ -39,9 +66,11 @@ extern "C" { * if condition 'cond' is false. Macro definition is empty when compiler is not * supported or the compiler does not support static assertion. */ -#define ODP_STATIC_ASSERT(cond, msg) _Static_assert(cond, msg) +#ifndef __cplusplus +#define ODP_STATIC_ASSERT(cond, msg) _Static_assert(cond, msg) -#ifdef __cplusplus +#else +#define ODP_STATIC_ASSERT(cond, msg) static_assert(cond, msg) } #endif diff --git a/platform/linux-generic/include/odp/api/deprecated.h b/platform/linux-generic/include/odp/api/deprecated.h new file mode 100644 index 000000000..82797ebc4 --- /dev/null +++ b/platform/linux-generic/include/odp/api/deprecated.h @@ -0,0 +1,26 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * Control deprecated API definitions + */ + +#ifndef ODP_PLAT_DEPRECATED_H_ +#define ODP_PLAT_DEPRECATED_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/spec/deprecated.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/api/plat/packet_types.h b/platform/linux-generic/include/odp/api/plat/packet_types.h index 7e3c51e6c..a209c759f 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_types.h @@ -105,8 +105,8 @@ typedef union { /** All input flags */ uint64_t all; + /** Individual input flags */ struct { - uint64_t parsed_l2:1; /**< L2 parsed */ uint64_t dst_queue:1; /**< Dst queue present */ uint64_t flow_hash:1; /**< Flow hash present */ diff --git a/platform/linux-generic/include/odp/api/plat/time_types.h b/platform/linux-generic/include/odp/api/plat/time_types.h index 4847f3b1f..e7111c8c5 100644 --- a/platform/linux-generic/include/odp/api/plat/time_types.h +++ b/platform/linux-generic/include/odp/api/plat/time_types.h @@ -22,15 +22,25 @@ extern "C" { **/ /** - * @internal Time structure used to isolate odp-linux implementation from - * the linux timespec structure, which is dependent on POSIX extension level. + * @internal Time structure used for both POSIX timespec and HW counter + * implementations. */ typedef struct odp_time_t { - int64_t tv_sec; /**< @internal Seconds */ - int64_t tv_nsec; /**< @internal Nanoseconds */ + /** @internal Variant mappings for time type */ + union { + /** @internal Used with generic 64 bit operations */ + uint64_t u64; + + /** @internal Nanoseconds */ + uint64_t nsec; + + /** @internal HW timer counter value */ + uint64_t count; + + }; } odp_time_t; -#define ODP_TIME_NULL ((odp_time_t){0, 0}) +#define ODP_TIME_NULL ((odp_time_t){.u64 = 0}) /** * @} diff --git a/platform/linux-generic/include/odp/api/plat/traffic_mngr_types.h b/platform/linux-generic/include/odp/api/plat/traffic_mngr_types.h index b766afecd..f47a13f6f 100644 --- a/platform/linux-generic/include/odp/api/plat/traffic_mngr_types.h +++ b/platform/linux-generic/include/odp/api/plat/traffic_mngr_types.h @@ -168,7 +168,7 @@ typedef odp_tm_handle_t odp_tm_wred_t; */ #define ODP_TM_ROOT ((odp_tm_handle_t)-1) -/** Get printable format of odp_queue_t */ +/** @internal Get printable format of odp_tm_handle_t @param hdl @return */ static inline uint64_t odp_tm_handle_to_u64(odp_tm_handle_t hdl) { return hdl; diff --git a/platform/linux-generic/include/odp_crypto_internal.h b/platform/linux-generic/include/odp_crypto_internal.h index f85b76eaa..c3b70b231 100644 --- a/platform/linux-generic/include/odp_crypto_internal.h +++ b/platform/linux-generic/include/odp_crypto_internal.h @@ -11,8 +11,6 @@ extern "C" { #endif -#include <openssl/des.h> -#include <openssl/aes.h> #include <openssl/evp.h> #define MAX_IV_LEN 64 @@ -42,34 +40,17 @@ struct odp_crypto_generic_session { struct { /* Copy of session IV data */ uint8_t iv_data[MAX_IV_LEN]; + uint8_t key_data[EVP_MAX_KEY_LENGTH]; - union { - struct { - DES_key_schedule ks1; - DES_key_schedule ks2; - DES_key_schedule ks3; - } des; - struct { - AES_KEY key; - } aes; - struct { - EVP_CIPHER_CTX *ctx; - } aes_gcm; - } data; + const EVP_CIPHER *evp_cipher; crypto_func_t func; } cipher; struct { - union { - struct { - uint8_t key[16]; - uint32_t bytes; - } md5; - struct { - uint8_t key[32]; - uint32_t bytes; - } sha256; - } data; + uint8_t key[EVP_MAX_KEY_LENGTH]; + uint32_t key_length; + uint32_t bytes; + const EVP_MD *evp_md; crypto_func_t func; } auth; }; diff --git a/platform/linux-generic/include/odp_internal.h b/platform/linux-generic/include/odp_internal.h index e1267cff7..8bae028d9 100644 --- a/platform/linux-generic/include/odp_internal.h +++ b/platform/linux-generic/include/odp_internal.h @@ -128,6 +128,7 @@ int _odp_ishm_term_local(void); int cpuinfo_parser(FILE *file, system_info_t *sysinfo); uint64_t odp_cpu_hz_current(int id); +void sys_info_print_arch(void); #ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 0a9f17799..a480a7484 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -80,18 +80,6 @@ ODP_STATIC_ASSERT(sizeof(output_flags_t) == sizeof(uint32_t), "OUTPUT_FLAGS_SIZE_ERROR"); /** - * Protocol stack layers - */ -typedef enum { - LAYER_NONE = 0, - LAYER_L1, - LAYER_L2, - LAYER_L3, - LAYER_L4, - LAYER_ALL -} layer_t; - -/** * Packet parser metadata */ typedef struct { @@ -102,14 +90,6 @@ typedef struct { uint32_t l2_offset; /**< offset to L2 hdr, e.g. Eth */ uint32_t l3_offset; /**< offset to L3 hdr, e.g. IPv4, IPv6 */ uint32_t l4_offset; /**< offset to L4 hdr (TCP, UDP, SCTP, also ICMP) */ - - uint32_t l3_len; /**< Layer 3 length */ - uint32_t l4_len; /**< Layer 4 length */ - - uint16_t ethtype; /**< EtherType */ - uint8_t ip_proto; /**< IP protocol */ - uint8_t parsed_layers; /**< Highest parsed protocol stack layer */ - } packet_parser_t; /** @@ -203,16 +183,6 @@ static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len) pkt_hdr->frame_len = len; } -static inline int packet_parse_l2_not_done(packet_parser_t *prs) -{ - return !prs->input_flags.parsed_l2; -} - -static inline int packet_parse_not_complete(odp_packet_hdr_t *pkt_hdr) -{ - return pkt_hdr->p.parsed_layers != LAYER_ALL; -} - /* Forward declarations */ int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt); @@ -220,11 +190,9 @@ int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt); int packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, odp_packet_t pkt[], int max_num); -/* Fill in parser metadata for L2 */ -void packet_parse_l2(packet_parser_t *prs, uint32_t frame_len); - /* Perform packet parse up to a given protocol layer */ -int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, layer_t layer); +int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, + odp_pktio_parser_layer_t layer); /* Reset parser metadata for a new parse */ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr); @@ -264,10 +232,17 @@ static inline void packet_set_ts(odp_packet_hdr_t *pkt_hdr, odp_time_t *ts) } int packet_parse_common(packet_parser_t *pkt_hdr, const uint8_t *ptr, - uint32_t pkt_len, uint32_t seg_len, layer_t layer); + uint32_t pkt_len, uint32_t seg_len, + odp_pktio_parser_layer_t layer); int _odp_cls_parse(odp_packet_hdr_t *pkt_hdr, const uint8_t *parseptr); +int _odp_packet_set_data(odp_packet_t pkt, uint32_t offset, + uint8_t c, uint32_t len); + +int _odp_packet_cmp_data(odp_packet_t pkt, uint32_t offset, + const void *s, uint32_t len); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_packet_socket.h b/platform/linux-generic/include/odp_packet_socket.h index dbfc9f1f8..0e61f6f0c 100644 --- a/platform/linux-generic/include/odp_packet_socket.h +++ b/platform/linux-generic/include/odp_packet_socket.h @@ -27,11 +27,6 @@ * Packet socket config: */ -/** Max receive (Rx) burst size*/ -#define ODP_PACKET_SOCKET_MAX_BURST_RX 32 -/** Max transmit (Tx) burst size*/ -#define ODP_PACKET_SOCKET_MAX_BURST_TX 32 - /* * This makes sure that building for kernels older than 3.1 works * and a fanout requests fails (for invalid packet socket option) @@ -47,8 +42,6 @@ typedef struct { odp_pool_t pool; /**< pool to alloc packets from */ uint32_t mtu; /**< maximum transmission unit */ unsigned char if_mac[ETH_ALEN]; /**< IF eth mac addr */ - uint8_t *cache_ptr[ODP_PACKET_SOCKET_MAX_BURST_RX]; - odp_shm_t shm; } pkt_sock_t; /** packet mmap ring */ diff --git a/platform/linux-generic/include/odp_ring_internal.h b/platform/linux-generic/include/odp_ring_internal.h index 55fedeb3a..44b83c603 100644 --- a/platform/linux-generic/include/odp_ring_internal.h +++ b/platform/linux-generic/include/odp_ring_internal.h @@ -57,7 +57,7 @@ static inline uint32_t ring_deq(ring_t *ring, uint32_t mask) /* Move reader head. This thread owns data at the new head. */ do { - tail = odp_atomic_load_u32(&ring->w_tail); + tail = odp_atomic_load_acq_u32(&ring->w_tail); if (head == tail) return RING_EMPTY; @@ -90,7 +90,7 @@ static inline uint32_t ring_deq_multi(ring_t *ring, uint32_t mask, /* Move reader head. This thread owns data at the new head. */ do { - tail = odp_atomic_load_u32(&ring->w_tail); + tail = odp_atomic_load_acq_u32(&ring->w_tail); /* Ring is empty */ if (head == tail) diff --git a/platform/linux-generic/include/odp_time_internal.h b/platform/linux-generic/include/odp_time_internal.h new file mode 100644 index 000000000..99ac79772 --- /dev/null +++ b/platform/linux-generic/include/odp_time_internal.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_TIME_INTERNAL_H_ +#define ODP_TIME_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +int cpu_has_global_time(void); +uint64_t cpu_global_time(void); +uint64_t cpu_global_time_freq(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/protocols/ip.h b/platform/linux-generic/include/protocols/ip.h index 20041f1c0..2b34a753f 100644 --- a/platform/linux-generic/include/protocols/ip.h +++ b/platform/linux-generic/include/protocols/ip.h @@ -157,13 +157,14 @@ typedef struct ODP_PACKED { * IP protocol values (IPv4:'proto' or IPv6:'next_hdr') * @{*/ #define _ODP_IPPROTO_HOPOPTS 0x00 /**< IPv6 hop-by-hop options */ -#define _ODP_IPPROTO_ICMP 0x01 /**< Internet Control Message Protocol (1) */ +#define _ODP_IPPROTO_ICMPv4 0x01 /**< Internet Control Message Protocol (1) */ #define _ODP_IPPROTO_TCP 0x06 /**< Transmission Control Protocol (6) */ #define _ODP_IPPROTO_UDP 0x11 /**< User Datagram Protocol (17) */ #define _ODP_IPPROTO_ROUTE 0x2B /**< IPv6 Routing header (43) */ #define _ODP_IPPROTO_FRAG 0x2C /**< IPv6 Fragment (44) */ #define _ODP_IPPROTO_AH 0x33 /**< Authentication Header (51) */ #define _ODP_IPPROTO_ESP 0x32 /**< Encapsulating Security Payload (50) */ +#define _ODP_IPPROTO_ICMPv6 0x3A /**< Internet Control Message Protocol (58) */ #define _ODP_IPPROTO_SCTP 0x84 /**< Stream Control Transmission protocol (132) */ #define _ODP_IPPROTO_INVALID 0xFF /**< Reserved invalid by IANA */ diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4 index d3e5528c1..e1197f606 100644 --- a/platform/linux-generic/m4/configure.m4 +++ b/platform/linux-generic/m4/configure.m4 @@ -28,12 +28,56 @@ AC_LINK_IFELSE( echo "Use newer version. For gcc > 4.7.0" exit -1) +dnl Check whether -latomic is needed +use_libatomic=no + +AC_MSG_CHECKING(whether -latomic is needed for 64-bit atomic built-ins) +AC_LINK_IFELSE( + [AC_LANG_SOURCE([[ + static int loc; + int main(void) + { + int prev = __atomic_exchange_n(&loc, 7, __ATOMIC_RELAXED); + return 0; + } + ]])], + [AC_MSG_RESULT(no)], + [AC_MSG_RESULT(yes) + AC_CHECK_LIB( + [atomic], [__atomic_exchange_8], + [use_libatomic=yes], + [AC_MSG_FAILURE([__atomic_exchange_8 is not available])]) + ]) + +AC_MSG_CHECKING(whether -latomic is needed for 128-bit atomic built-ins) +AC_LINK_IFELSE( + [AC_LANG_SOURCE([[ + static __int128 loc; + int main(void) + { + __int128 prev; + prev = __atomic_exchange_n(&loc, 7, __ATOMIC_RELAXED); + return 0; + } + ]])], + [AC_MSG_RESULT(no)], + [AC_MSG_RESULT(yes) + AC_CHECK_LIB( + [atomic], [__atomic_exchange_16], + [use_libatomic=yes], + [AC_MSG_CHECKING([cannot detect support for 128-bit atomics])]) + ]) + +if test "x$use_libatomic" = "xyes"; then + ATOMIC_LIBS="-latomic" +fi +AC_SUBST([ATOMIC_LIBS]) + m4_include([platform/linux-generic/m4/odp_pthread.m4]) m4_include([platform/linux-generic/m4/odp_openssl.m4]) m4_include([platform/linux-generic/m4/odp_pcap.m4]) m4_include([platform/linux-generic/m4/odp_netmap.m4]) m4_include([platform/linux-generic/m4/odp_dpdk.m4]) -m4_include([platform/linux-generic/m4/odp_ipc.m4]) m4_include([platform/linux-generic/m4/odp_schedule.m4]) AC_CONFIG_FILES([platform/linux-generic/Makefile diff --git a/platform/linux-generic/m4/odp_dpdk.m4 b/platform/linux-generic/m4/odp_dpdk.m4 index 30347dce8..58d14727b 100644 --- a/platform/linux-generic/m4/odp_dpdk.m4 +++ b/platform/linux-generic/m4/odp_dpdk.m4 @@ -2,22 +2,10 @@ # Enable DPDK support ########################################################################## pktio_dpdk_support=no -AC_ARG_ENABLE([dpdk_support], - [ --enable-dpdk-support include dpdk IO support], - [if test x$enableval = xyes; then - pktio_dpdk_support=yes - fi]) - -########################################################################## -# Set optional DPDK path -########################################################################## AC_ARG_WITH([dpdk-path], -AC_HELP_STRING([--with-dpdk-path=DIR path to dpdk build directory], - [(or in the default path if not specified).]), +AC_HELP_STRING([--with-dpdk-path=DIR path to dpdk build directory]), [DPDK_PATH=$withval AM_CPPFLAGS="$AM_CPPFLAGS -msse4.2 -isystem $DPDK_PATH/include" - AM_LDFLAGS="$AM_LDFLAGS -L$DPDK_PATH/lib" - LIBS="$LIBS -ldpdk -ldl -lpcap" pktio_dpdk_support=yes],[]) ########################################################################## @@ -28,12 +16,31 @@ CPPFLAGS="$AM_CPPFLAGS $CPPFLAGS" ########################################################################## # Check for DPDK availability +# +# DPDK pmd drivers are not linked unless the --whole-archive option is +# used. No spaces are allowed between the --whole-arhive flags. ########################################################################## if test x$pktio_dpdk_support = xyes then AC_CHECK_HEADERS([rte_config.h], [], [AC_MSG_FAILURE(["can't find DPDK header"])]) + + DPDK_PMD=--whole-archive, + for filename in $with_dpdk_path/lib/*.a; do + cur_driver=`echo $(basename "$filename" .a) | \ + sed -n 's/^\(librte_pmd_\)/-lrte_pmd_/p' | sed -n 's/$/,/p'` + # rte_pmd_nfp has external dependencies which break linking + if test "$cur_driver" = "-lrte_pmd_nfp,"; then + echo "skip linking rte_pmd_nfp" + else + DPDK_PMD+=$cur_driver + fi + done + DPDK_PMD+=--no-whole-archive + ODP_CFLAGS="$ODP_CFLAGS -DODP_PKTIO_DPDK" + AM_LDFLAGS="$AM_LDFLAGS -L$DPDK_PATH/lib -Wl,$DPDK_PMD" + LIBS="$LIBS -ldpdk -ldl -lpcap" else pktio_dpdk_support=no fi diff --git a/platform/linux-generic/m4/odp_ipc.m4 b/platform/linux-generic/m4/odp_ipc.m4 deleted file mode 100644 index 78217e221..000000000 --- a/platform/linux-generic/m4/odp_ipc.m4 +++ /dev/null @@ -1,9 +0,0 @@ -########################################################################## -# Enable IPC pktio support -########################################################################## -AC_ARG_ENABLE([pktio_ipc_support], - [ --enable-pktio_ipc-support include ipc IO support], - [if test x$enableval = xyes; then - pktio_ipc_support=yes - ODP_CFLAGS="$ODP_CFLAGS -D_ODP_PKTIO_IPC" - fi]) diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index 5d96b00b3..7ebc47d7d 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -790,10 +790,6 @@ static inline cos_t *cls_select_cos(pktio_entry_t *entry, cls = &entry->s.cls; default_cos = cls->default_cos; - /* Check for lazy parse needed */ - if (packet_parse_not_complete(pkt_hdr)) - packet_parse_layer(pkt_hdr, LAYER_ALL); - /* Return error cos for error packet */ if (pkt_hdr->p.error_flags.all) return cls->error_cos; @@ -838,7 +834,8 @@ int cls_classify_packet(pktio_entry_t *entry, const uint8_t *base, packet_parse_reset(pkt_hdr); packet_set_len(pkt_hdr, pkt_len); - packet_parse_common(&pkt_hdr->p, base, pkt_len, seg_len, LAYER_ALL); + packet_parse_common(&pkt_hdr->p, base, pkt_len, seg_len, + ODP_PKTIO_PARSER_LAYER_ALL); cos = cls_select_cos(entry, base, pkt_hdr); if (cos == NULL) diff --git a/platform/linux-generic/odp_crypto.c b/platform/linux-generic/odp_crypto.c index 54b222fd2..6fc1907d9 100644 --- a/platform/linux-generic/odp_crypto.c +++ b/platform/linux-generic/odp_crypto.c @@ -34,6 +34,9 @@ * * Keep sorted: first by key length, then by IV length */ +static const odp_crypto_cipher_capability_t cipher_capa_null[] = { +{.key_len = 0, .iv_len = 0} }; + static const odp_crypto_cipher_capability_t cipher_capa_des[] = { {.key_len = 24, .iv_len = 8} }; @@ -51,11 +54,24 @@ static const odp_crypto_cipher_capability_t cipher_capa_aes_gcm[] = { * * Keep sorted: first by digest length, then by key length */ +static const odp_crypto_auth_capability_t auth_capa_null[] = { +{.digest_len = 0, .key_len = 0, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; + static const odp_crypto_auth_capability_t auth_capa_md5_hmac[] = { -{.digest_len = 12, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; +{.digest_len = 12, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0} }, +{.digest_len = 16, .key_len = 16, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; + +static const odp_crypto_auth_capability_t auth_capa_sha1_hmac[] = { +{.digest_len = 12, .key_len = 20, .aad_len = {.min = 0, .max = 0, .inc = 0} }, +{.digest_len = 20, .key_len = 20, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; static const odp_crypto_auth_capability_t auth_capa_sha256_hmac[] = { -{.digest_len = 16, .key_len = 32, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; +{.digest_len = 16, .key_len = 32, .aad_len = {.min = 0, .max = 0, .inc = 0} }, +{.digest_len = 32, .key_len = 32, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; + +static const odp_crypto_auth_capability_t auth_capa_sha512_hmac[] = { +{.digest_len = 32, .key_len = 64, .aad_len = {.min = 0, .max = 0, .inc = 0} }, +{.digest_len = 64, .key_len = 64, .aad_len = {.min = 0, .max = 0, .inc = 0} } }; static const odp_crypto_auth_capability_t auth_capa_aes_gcm[] = { {.digest_len = 16, .key_len = 0, .aad_len = {.min = 8, .max = 12, .inc = 4} } }; @@ -86,8 +102,10 @@ odp_crypto_generic_session_t *alloc_session(void) odp_spinlock_lock(&global->lock); session = global->free; - if (session) + if (session) { global->free = session->next; + session->next = NULL; + } odp_spinlock_unlock(&global->lock); return session; @@ -110,128 +128,74 @@ null_crypto_routine(odp_crypto_op_param_t *param ODP_UNUSED, } static -odp_crypto_alg_err_t md5_gen(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session) +void packet_hmac(odp_crypto_op_param_t *param, + odp_crypto_generic_session_t *session, + uint8_t *hash) { - uint8_t *data = odp_packet_data(param->out_pkt); - uint8_t *icv = data; + odp_packet_t pkt = param->out_pkt; + uint32_t offset = param->auth_range.offset; uint32_t len = param->auth_range.length; - uint8_t hash[EVP_MAX_MD_SIZE]; + HMAC_CTX ctx; - /* Adjust pointer for beginning of area to auth */ - data += param->auth_range.offset; - icv += param->hash_result_offset; + ODP_ASSERT(offset + len <= odp_packet_len(pkt)); /* Hash it */ - HMAC(EVP_md5(), - session->auth.data.md5.key, - 16, - data, - len, - hash, - NULL); - - /* Copy to the output location */ - memcpy(icv, hash, session->auth.data.md5.bytes); - - return ODP_CRYPTO_ALG_ERR_NONE; -} - -static -odp_crypto_alg_err_t md5_check(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session) -{ - uint8_t *data = odp_packet_data(param->out_pkt); - uint8_t *icv = data; - uint32_t len = param->auth_range.length; - uint32_t bytes = session->auth.data.md5.bytes; - uint8_t hash_in[EVP_MAX_MD_SIZE]; - uint8_t hash_out[EVP_MAX_MD_SIZE]; - - /* Adjust pointer for beginning of area to auth */ - data += param->auth_range.offset; - icv += param->hash_result_offset; - - /* Copy current value out and clear it before authentication */ - memset(hash_in, 0, sizeof(hash_in)); - memcpy(hash_in, icv, bytes); - memset(icv, 0, bytes); - memset(hash_out, 0, sizeof(hash_out)); - - /* Hash it */ - HMAC(EVP_md5(), - session->auth.data.md5.key, - 16, - data, - len, - hash_out, - NULL); - - /* Verify match */ - if (0 != memcmp(hash_in, hash_out, bytes)) - return ODP_CRYPTO_ALG_ERR_ICV_CHECK; + HMAC_CTX_init(&ctx); + HMAC_Init_ex(&ctx, + session->auth.key, + session->auth.key_length, + session->auth.evp_md, + NULL); + + while (len > 0) { + uint32_t seglen = 0; /* GCC */ + void *mapaddr = odp_packet_offset(pkt, offset, &seglen, NULL); + uint32_t maclen = len > seglen ? seglen : len; + + HMAC_Update(&ctx, mapaddr, maclen); + offset += maclen; + len -= maclen; + } - /* Matched */ - return ODP_CRYPTO_ALG_ERR_NONE; + HMAC_Final(&ctx, hash, NULL); + HMAC_CTX_cleanup(&ctx); } static -odp_crypto_alg_err_t sha256_gen(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session) +odp_crypto_alg_err_t auth_gen(odp_crypto_op_param_t *param, + odp_crypto_generic_session_t *session) { - uint8_t *data = odp_packet_data(param->out_pkt); - uint8_t *icv = data; - uint32_t len = param->auth_range.length; uint8_t hash[EVP_MAX_MD_SIZE]; - /* Adjust pointer for beginning of area to auth */ - data += param->auth_range.offset; - icv += param->hash_result_offset; - /* Hash it */ - HMAC(EVP_sha256(), - session->auth.data.sha256.key, - 32, - data, - len, - hash, - NULL); + packet_hmac(param, session, hash); /* Copy to the output location */ - memcpy(icv, hash, session->auth.data.sha256.bytes); + odp_packet_copy_from_mem(param->out_pkt, + param->hash_result_offset, + session->auth.bytes, + hash); return ODP_CRYPTO_ALG_ERR_NONE; } static -odp_crypto_alg_err_t sha256_check(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session) +odp_crypto_alg_err_t auth_check(odp_crypto_op_param_t *param, + odp_crypto_generic_session_t *session) { - uint8_t *data = odp_packet_data(param->out_pkt); - uint8_t *icv = data; - uint32_t len = param->auth_range.length; - uint32_t bytes = session->auth.data.sha256.bytes; + uint32_t bytes = session->auth.bytes; uint8_t hash_in[EVP_MAX_MD_SIZE]; uint8_t hash_out[EVP_MAX_MD_SIZE]; - /* Adjust pointer for beginning of area to auth */ - data += param->auth_range.offset; - icv += param->hash_result_offset; - /* Copy current value out and clear it before authentication */ - memset(hash_in, 0, sizeof(hash_in)); - memcpy(hash_in, icv, bytes); - memset(icv, 0, bytes); - memset(hash_out, 0, sizeof(hash_out)); + odp_packet_copy_to_mem(param->out_pkt, param->hash_result_offset, + bytes, hash_in); + + _odp_packet_set_data(param->out_pkt, param->hash_result_offset, + 0, bytes); /* Hash it */ - HMAC(EVP_sha256(), - session->auth.data.sha256.key, - 32, - data, - len, - hash_out, - NULL); + packet_hmac(param, session, hash_out); /* Verify match */ if (0 != memcmp(hash_in, hash_out, bytes)) @@ -242,102 +206,122 @@ odp_crypto_alg_err_t sha256_check(odp_crypto_op_param_t *param, } static -odp_crypto_alg_err_t aes_encrypt(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session) +int internal_encrypt(EVP_CIPHER_CTX *ctx, odp_crypto_op_param_t *param) { - uint8_t *data = odp_packet_data(param->out_pkt); - uint32_t len = param->cipher_range.length; - unsigned char iv_enc[AES_BLOCK_SIZE]; - void *iv_ptr; + odp_packet_t pkt = param->out_pkt; + unsigned in_pos = param->cipher_range.offset; + unsigned out_pos = param->cipher_range.offset; + unsigned in_len = param->cipher_range.length; + uint8_t block[2 * EVP_MAX_BLOCK_LENGTH]; + unsigned block_len = EVP_CIPHER_block_size(EVP_CIPHER_CTX_cipher(ctx)); + int cipher_len; + int ret; - if (param->override_iv_ptr) - iv_ptr = param->override_iv_ptr; - else if (session->p.iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; + while (in_len > 0) { + uint32_t seglen = 0; /* GCC */ + uint8_t *insegaddr = odp_packet_offset(pkt, in_pos, + &seglen, NULL); + unsigned inseglen = in_len < seglen ? in_len : seglen; - /* - * Create a copy of the IV. The DES library modifies IV - * and if we are processing packets on parallel threads - * we could get corruption. - */ - memcpy(iv_enc, iv_ptr, AES_BLOCK_SIZE); + /* There should be at least 1 additional block in out buffer */ + if (inseglen > block_len) { + unsigned part = inseglen - block_len; - /* Adjust pointer for beginning of area to cipher */ - data += param->cipher_range.offset; - /* Encrypt it */ - AES_cbc_encrypt(data, data, len, &session->cipher.data.aes.key, - iv_enc, AES_ENCRYPT); + EVP_EncryptUpdate(ctx, insegaddr, &cipher_len, + insegaddr, part); + in_pos += part; + in_len -= part; + insegaddr += part; + inseglen -= part; - return ODP_CRYPTO_ALG_ERR_NONE; + out_pos += cipher_len; + } + + /* Use temporal storage */ + if (inseglen > 0) { + unsigned part = inseglen; + + EVP_EncryptUpdate(ctx, block, &cipher_len, + insegaddr, part); + in_pos += part; + in_len -= part; + insegaddr += part; + inseglen -= part; + + odp_packet_copy_from_mem(pkt, out_pos, + cipher_len, block); + out_pos += cipher_len; + } + } + + ret = EVP_EncryptFinal_ex(ctx, block, &cipher_len); + odp_packet_copy_from_mem(pkt, out_pos, cipher_len, block); + + return ret; } static -odp_crypto_alg_err_t aes_decrypt(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session) +int internal_decrypt(EVP_CIPHER_CTX *ctx, odp_crypto_op_param_t *param) { - uint8_t *data = odp_packet_data(param->out_pkt); - uint32_t len = param->cipher_range.length; - unsigned char iv_enc[AES_BLOCK_SIZE]; - void *iv_ptr; + odp_packet_t pkt = param->out_pkt; + unsigned in_pos = param->cipher_range.offset; + unsigned out_pos = param->cipher_range.offset; + unsigned in_len = param->cipher_range.length; + uint8_t block[2 * EVP_MAX_BLOCK_LENGTH]; + unsigned block_len = EVP_CIPHER_block_size(EVP_CIPHER_CTX_cipher(ctx)); + int cipher_len; + int ret; - if (param->override_iv_ptr) - iv_ptr = param->override_iv_ptr; - else if (session->p.iv.data) - iv_ptr = session->cipher.iv_data; - else - return ODP_CRYPTO_ALG_ERR_IV_INVALID; + while (in_len > 0) { + uint32_t seglen = 0; /* GCC */ + uint8_t *insegaddr = odp_packet_offset(pkt, in_pos, + &seglen, NULL); + unsigned inseglen = in_len < seglen ? in_len : seglen; - /* - * Create a copy of the IV. The DES library modifies IV - * and if we are processing packets on parallel threads - * we could get corruption. - */ - memcpy(iv_enc, iv_ptr, AES_BLOCK_SIZE); + /* There should be at least 1 additional block in out buffer */ + if (inseglen > block_len) { + unsigned part = inseglen - block_len; - /* Adjust pointer for beginning of area to cipher */ - data += param->cipher_range.offset; - /* Encrypt it */ - AES_cbc_encrypt(data, data, len, &session->cipher.data.aes.key, - iv_enc, AES_DECRYPT); + EVP_DecryptUpdate(ctx, insegaddr, &cipher_len, + insegaddr, part); + in_pos += part; + in_len -= part; + insegaddr += part; + inseglen -= part; - return ODP_CRYPTO_ALG_ERR_NONE; -} + out_pos += cipher_len; + } -static int process_aes_param(odp_crypto_generic_session_t *session) -{ - /* Verify IV len is either 0 or 16 */ - if (!((0 == session->p.iv.length) || (16 == session->p.iv.length))) - return -1; + /* Use temporal storage */ + if (inseglen > 0) { + unsigned part = inseglen; - /* Set function */ - if (ODP_CRYPTO_OP_ENCODE == session->p.op) { - session->cipher.func = aes_encrypt; - AES_set_encrypt_key(session->p.cipher_key.data, 128, - &session->cipher.data.aes.key); - } else { - session->cipher.func = aes_decrypt; - AES_set_decrypt_key(session->p.cipher_key.data, 128, - &session->cipher.data.aes.key); + EVP_DecryptUpdate(ctx, block, &cipher_len, + insegaddr, part); + in_pos += part; + in_len -= part; + insegaddr += part; + inseglen -= part; + + odp_packet_copy_from_mem(pkt, out_pos, + cipher_len, block); + out_pos += cipher_len; + } } - return 0; + ret = EVP_DecryptFinal_ex(ctx, block, &cipher_len); + odp_packet_copy_from_mem(pkt, out_pos, cipher_len, block); + + return ret; } static -odp_crypto_alg_err_t aes_gcm_encrypt(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session) +odp_crypto_alg_err_t cipher_encrypt(odp_crypto_op_param_t *param, + odp_crypto_generic_session_t *session) { - uint8_t *data = odp_packet_data(param->out_pkt); - uint32_t plain_len = param->cipher_range.length; - uint8_t *aad_head = data + param->auth_range.offset; - uint8_t *aad_tail = data + param->cipher_range.offset + - param->cipher_range.length; - uint32_t auth_len = param->auth_range.length; - unsigned char iv_enc[AES_BLOCK_SIZE]; + EVP_CIPHER_CTX *ctx; void *iv_ptr; - uint8_t *tag = data + param->hash_result_offset; + int ret; if (param->override_iv_ptr) iv_ptr = param->override_iv_ptr; @@ -346,63 +330,28 @@ odp_crypto_alg_err_t aes_gcm_encrypt(odp_crypto_op_param_t *param, else return ODP_CRYPTO_ALG_ERR_IV_INVALID; - /* All cipher data must be part of the authentication */ - if (param->auth_range.offset > param->cipher_range.offset || - param->auth_range.offset + auth_len < - param->cipher_range.offset + plain_len) - return ODP_CRYPTO_ALG_ERR_DATA_SIZE; - - /* - * Create a copy of the IV. The DES library modifies IV - * and if we are processing packets on parallel threads - * we could get corruption. - */ - memcpy(iv_enc, iv_ptr, AES_BLOCK_SIZE); - - /* Adjust pointer for beginning of area to cipher/auth */ - uint8_t *plaindata = data + param->cipher_range.offset; - /* Encrypt it */ - EVP_CIPHER_CTX *ctx = session->cipher.data.aes_gcm.ctx; - int cipher_len = 0; - - EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_enc); - - /* Authenticate header data (if any) without encrypting them */ - if (aad_head < plaindata) { - EVP_EncryptUpdate(ctx, NULL, &cipher_len, - aad_head, plaindata - aad_head); - } + ctx = EVP_CIPHER_CTX_new(); + EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL, + session->cipher.key_data, NULL); + EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_CIPHER_CTX_set_padding(ctx, 0); - EVP_EncryptUpdate(ctx, plaindata, &cipher_len, - plaindata, plain_len); - cipher_len = plain_len; + ret = internal_encrypt(ctx, param); - /* Authenticate footer data (if any) without encrypting them */ - if (aad_head + auth_len > plaindata + plain_len) { - EVP_EncryptUpdate(ctx, NULL, NULL, aad_tail, - auth_len - (aad_tail - aad_head)); - } + EVP_CIPHER_CTX_free(ctx); - EVP_EncryptFinal_ex(ctx, plaindata + cipher_len, &cipher_len); - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag); - - return ODP_CRYPTO_ALG_ERR_NONE; + return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : + ODP_CRYPTO_ALG_ERR_NONE; } static -odp_crypto_alg_err_t aes_gcm_decrypt(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session) +odp_crypto_alg_err_t cipher_decrypt(odp_crypto_op_param_t *param, + odp_crypto_generic_session_t *session) { - uint8_t *data = odp_packet_data(param->out_pkt); - uint32_t cipher_len = param->cipher_range.length; - uint8_t *aad_head = data + param->auth_range.offset; - uint8_t *aad_tail = data + param->cipher_range.offset + - param->cipher_range.length; - uint32_t auth_len = param->auth_range.length; - unsigned char iv_enc[AES_BLOCK_SIZE]; + EVP_CIPHER_CTX *ctx; void *iv_ptr; - uint8_t *tag = data + param->hash_result_offset; + int ret; if (param->override_iv_ptr) iv_ptr = param->override_iv_ptr; @@ -411,90 +360,59 @@ odp_crypto_alg_err_t aes_gcm_decrypt(odp_crypto_op_param_t *param, else return ODP_CRYPTO_ALG_ERR_IV_INVALID; - /* All cipher data must be part of the authentication */ - if (param->auth_range.offset > param->cipher_range.offset || - param->auth_range.offset + auth_len < - param->cipher_range.offset + cipher_len) - return ODP_CRYPTO_ALG_ERR_DATA_SIZE; - - /* - * Create a copy of the IV. The DES library modifies IV - * and if we are processing packets on parallel threads - * we could get corruption. - */ - memcpy(iv_enc, iv_ptr, AES_BLOCK_SIZE); - - /* Adjust pointer for beginning of area to cipher/auth */ - uint8_t *cipherdata = data + param->cipher_range.offset; - /* Encrypt it */ - EVP_CIPHER_CTX *ctx = session->cipher.data.aes_gcm.ctx; - int plain_len = 0; - - EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_enc); - - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag); - - /* Authenticate header data (if any) without encrypting them */ - if (aad_head < cipherdata) { - EVP_DecryptUpdate(ctx, NULL, &plain_len, - aad_head, cipherdata - aad_head); - } - - EVP_DecryptUpdate(ctx, cipherdata, &plain_len, - cipherdata, cipher_len); - plain_len = cipher_len; + /* Decrypt it */ + ctx = EVP_CIPHER_CTX_new(); + EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL, + session->cipher.key_data, NULL); + EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_CIPHER_CTX_set_padding(ctx, 0); - /* Authenticate footer data (if any) without encrypting them */ - if (aad_head + auth_len > cipherdata + cipher_len) { - EVP_DecryptUpdate(ctx, NULL, NULL, aad_tail, - auth_len - (aad_tail - aad_head)); - } + ret = internal_decrypt(ctx, param); - if (EVP_DecryptFinal_ex(ctx, cipherdata + cipher_len, &plain_len) < 0) - return ODP_CRYPTO_ALG_ERR_ICV_CHECK; + EVP_CIPHER_CTX_free(ctx); - return ODP_CRYPTO_ALG_ERR_NONE; + return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : + ODP_CRYPTO_ALG_ERR_NONE; } -static int process_aes_gcm_param(odp_crypto_generic_session_t *session) +static int process_cipher_param(odp_crypto_generic_session_t *session, + const EVP_CIPHER *cipher) { - /* Verify Key len is 16 */ - if (session->p.cipher_key.length != 16) + /* Verify Key len is valid */ + if ((uint32_t)EVP_CIPHER_key_length(cipher) != + session->p.cipher_key.length) return -1; - /* Set function */ - EVP_CIPHER_CTX *ctx = - session->cipher.data.aes_gcm.ctx = EVP_CIPHER_CTX_new(); + /* Verify IV len is correct */ + if (!((0 == session->p.iv.length) || + ((uint32_t)EVP_CIPHER_iv_length(cipher) == session->p.iv.length))) + return -1; - if (ODP_CRYPTO_OP_ENCODE == session->p.op) { - session->cipher.func = aes_gcm_encrypt; - EVP_EncryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL); - } else { - session->cipher.func = aes_gcm_decrypt; - EVP_DecryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL); - } + session->cipher.evp_cipher = cipher; - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, - session->p.iv.length, NULL); - if (ODP_CRYPTO_OP_ENCODE == session->p.op) { - EVP_EncryptInit_ex(ctx, NULL, NULL, - session->p.cipher_key.data, NULL); - } else { - EVP_DecryptInit_ex(ctx, NULL, NULL, - session->p.cipher_key.data, NULL); - } + memcpy(session->cipher.key_data, session->p.cipher_key.data, + session->p.cipher_key.length); + + /* Set function */ + if (ODP_CRYPTO_OP_ENCODE == session->p.op) + session->cipher.func = cipher_encrypt; + else + session->cipher.func = cipher_decrypt; return 0; } static -odp_crypto_alg_err_t des_encrypt(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session) +odp_crypto_alg_err_t aes_gcm_encrypt(odp_crypto_op_param_t *param, + odp_crypto_generic_session_t *session) { - uint8_t *data = odp_packet_data(param->out_pkt); - uint32_t len = param->cipher_range.length; - DES_cblock iv; + EVP_CIPHER_CTX *ctx; + const uint8_t *aad_head = param->aad.ptr; + uint32_t aad_len = param->aad.length; void *iv_ptr; + int dummy_len = 0; + uint8_t block[EVP_MAX_MD_SIZE]; + int ret; if (param->override_iv_ptr) iv_ptr = param->override_iv_ptr; @@ -503,36 +421,44 @@ odp_crypto_alg_err_t des_encrypt(odp_crypto_op_param_t *param, else return ODP_CRYPTO_ALG_ERR_IV_INVALID; - /* - * Create a copy of the IV. The DES library modifies IV - * and if we are processing packets on parallel threads - * we could get corruption. - */ - memcpy(iv, iv_ptr, sizeof(iv)); - - /* Adjust pointer for beginning of area to cipher */ - data += param->cipher_range.offset; /* Encrypt it */ - DES_ede3_cbc_encrypt(data, - data, - len, - &session->cipher.data.des.ks1, - &session->cipher.data.des.ks2, - &session->cipher.data.des.ks3, - &iv, - 1); + ctx = EVP_CIPHER_CTX_new(); + EVP_EncryptInit_ex(ctx, session->cipher.evp_cipher, NULL, + session->cipher.key_data, NULL); + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, + session->p.iv.length, NULL); + EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_CIPHER_CTX_set_padding(ctx, 0); - return ODP_CRYPTO_ALG_ERR_NONE; + /* Authenticate header data (if any) without encrypting them */ + if (aad_len > 0) + EVP_EncryptUpdate(ctx, NULL, &dummy_len, + aad_head, aad_len); + + ret = internal_encrypt(ctx, param); + + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, + session->p.auth_digest_len, block); + odp_packet_copy_from_mem(param->out_pkt, param->hash_result_offset, + session->p.auth_digest_len, block); + + EVP_CIPHER_CTX_free(ctx); + + return ret <= 0 ? ODP_CRYPTO_ALG_ERR_DATA_SIZE : + ODP_CRYPTO_ALG_ERR_NONE; } static -odp_crypto_alg_err_t des_decrypt(odp_crypto_op_param_t *param, - odp_crypto_generic_session_t *session) +odp_crypto_alg_err_t aes_gcm_decrypt(odp_crypto_op_param_t *param, + odp_crypto_generic_session_t *session) { - uint8_t *data = odp_packet_data(param->out_pkt); - uint32_t len = param->cipher_range.length; - DES_cblock iv; + EVP_CIPHER_CTX *ctx; + const uint8_t *aad_head = param->aad.ptr; + uint32_t aad_len = param->aad.length; + int dummy_len = 0; void *iv_ptr; + uint8_t block[EVP_MAX_MD_SIZE]; + int ret; if (param->override_iv_ptr) iv_ptr = param->override_iv_ptr; @@ -541,84 +467,76 @@ odp_crypto_alg_err_t des_decrypt(odp_crypto_op_param_t *param, else return ODP_CRYPTO_ALG_ERR_IV_INVALID; - /* - * Create a copy of the IV. The DES library modifies IV - * and if we are processing packets on parallel threads - * we could get corruption. - */ - memcpy(iv, iv_ptr, sizeof(iv)); + /* Decrypt it */ + ctx = EVP_CIPHER_CTX_new(); + EVP_DecryptInit_ex(ctx, session->cipher.evp_cipher, NULL, + session->cipher.key_data, NULL); + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, + session->p.iv.length, NULL); + EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv_ptr); + EVP_CIPHER_CTX_set_padding(ctx, 0); - /* Adjust pointer for beginning of area to cipher */ - data += param->cipher_range.offset; + odp_packet_copy_to_mem(param->out_pkt, param->hash_result_offset, + session->p.auth_digest_len, block); + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, + session->p.auth_digest_len, block); - /* Decrypt it */ - DES_ede3_cbc_encrypt(data, - data, - len, - &session->cipher.data.des.ks1, - &session->cipher.data.des.ks2, - &session->cipher.data.des.ks3, - &iv, - 0); + /* Authenticate header data (if any) without encrypting them */ + if (aad_len > 0) + EVP_DecryptUpdate(ctx, NULL, &dummy_len, + aad_head, aad_len); - return ODP_CRYPTO_ALG_ERR_NONE; + ret = internal_decrypt(ctx, param); + + EVP_CIPHER_CTX_free(ctx); + + return ret <= 0 ? ODP_CRYPTO_ALG_ERR_ICV_CHECK : + ODP_CRYPTO_ALG_ERR_NONE; } -static int process_des_param(odp_crypto_generic_session_t *session) +static int process_aes_gcm_param(odp_crypto_generic_session_t *session, + const EVP_CIPHER *cipher) { - /* Verify IV len is either 0 or 8 */ - if (!((0 == session->p.iv.length) || (8 == session->p.iv.length))) + /* Verify Key len is valid */ + if ((uint32_t)EVP_CIPHER_key_length(cipher) != + session->p.cipher_key.length) return -1; - /* Set function */ - if (ODP_CRYPTO_OP_ENCODE == session->p.op) - session->cipher.func = des_encrypt; - else - session->cipher.func = des_decrypt; + memcpy(session->cipher.key_data, session->p.cipher_key.data, + session->p.cipher_key.length); - /* Convert keys */ - DES_set_key((DES_cblock *)&session->p.cipher_key.data[0], - &session->cipher.data.des.ks1); - DES_set_key((DES_cblock *)&session->p.cipher_key.data[8], - &session->cipher.data.des.ks2); - DES_set_key((DES_cblock *)&session->p.cipher_key.data[16], - &session->cipher.data.des.ks3); + session->cipher.evp_cipher = cipher; - return 0; -} - -static int process_md5_param(odp_crypto_generic_session_t *session, - uint32_t bits) -{ /* Set function */ if (ODP_CRYPTO_OP_ENCODE == session->p.op) - session->auth.func = md5_gen; + session->cipher.func = aes_gcm_encrypt; else - session->auth.func = md5_check; - - /* Number of valid bytes */ - session->auth.data.md5.bytes = bits / 8; - - /* Convert keys */ - memcpy(session->auth.data.md5.key, session->p.auth_key.data, 16); + session->cipher.func = aes_gcm_decrypt; return 0; } -static int process_sha256_param(odp_crypto_generic_session_t *session, - uint32_t bits) +static int process_auth_param(odp_crypto_generic_session_t *session, + uint32_t key_length, + const EVP_MD *evp_md) { /* Set function */ if (ODP_CRYPTO_OP_ENCODE == session->p.op) - session->auth.func = sha256_gen; + session->auth.func = auth_gen; else - session->auth.func = sha256_check; + session->auth.func = auth_check; + + session->auth.evp_md = evp_md; /* Number of valid bytes */ - session->auth.data.sha256.bytes = bits / 8; + session->auth.bytes = session->p.auth_digest_len; + if (session->auth.bytes < (unsigned)EVP_MD_size(evp_md) / 2) + return -1; /* Convert keys */ - memcpy(session->auth.data.sha256.key, session->p.auth_key.data, 32); + session->auth.key_length = key_length; + memcpy(session->auth.key, session->p.auth_key.data, + session->auth.key_length); return 0; } @@ -639,15 +557,18 @@ int odp_crypto_capability(odp_crypto_capability_t *capa) capa->auths.bit.null = 1; capa->auths.bit.md5_hmac = 1; + capa->auths.bit.sha1_hmac = 1; capa->auths.bit.sha256_hmac = 1; + capa->auths.bit.sha512_hmac = 1; capa->auths.bit.aes_gcm = 1; - /* Deprecated */ +#if ODP_DEPRECATED_API capa->ciphers.bit.aes128_cbc = 1; capa->ciphers.bit.aes128_gcm = 1; capa->auths.bit.md5_96 = 1; capa->auths.bit.sha256_128 = 1; capa->auths.bit.aes128_gcm = 1; +#endif capa->max_sessions = MAX_SESSIONS; @@ -664,8 +585,8 @@ int odp_crypto_cipher_capability(odp_cipher_alg_t cipher, switch (cipher) { case ODP_CIPHER_ALG_NULL: - src = NULL; - num = 0; + src = cipher_capa_null; + num = sizeof(cipher_capa_null) / size; break; case ODP_CIPHER_ALG_DES: src = cipher_capa_des; @@ -704,17 +625,25 @@ int odp_crypto_auth_capability(odp_auth_alg_t auth, switch (auth) { case ODP_AUTH_ALG_NULL: - src = NULL; - num = 0; + src = auth_capa_null; + num = sizeof(auth_capa_null) / size; break; case ODP_AUTH_ALG_MD5_HMAC: src = auth_capa_md5_hmac; num = sizeof(auth_capa_md5_hmac) / size; break; + case ODP_AUTH_ALG_SHA1_HMAC: + src = auth_capa_sha1_hmac; + num = sizeof(auth_capa_sha1_hmac) / size; + break; case ODP_AUTH_ALG_SHA256_HMAC: src = auth_capa_sha256_hmac; num = sizeof(auth_capa_sha256_hmac) / size; break; + case ODP_AUTH_ALG_SHA512_HMAC: + src = auth_capa_sha512_hmac; + num = sizeof(auth_capa_sha512_hmac) / size; + break; case ODP_AUTH_ALG_AES_GCM: src = auth_capa_aes_gcm; num = sizeof(auth_capa_aes_gcm) / size; @@ -738,6 +667,7 @@ odp_crypto_session_create(odp_crypto_session_param_t *param, { int rc; odp_crypto_generic_session_t *session; + int aes_gcm = 0; /* Default to successful result */ *status = ODP_CRYPTO_SES_CREATE_ERR_NONE; @@ -752,16 +682,16 @@ odp_crypto_session_create(odp_crypto_session_param_t *param, /* Copy parameters */ session->p = *param; - /* Copy IV data */ - if (session->p.iv.data) { - if (session->p.iv.length > MAX_IV_LEN) { - ODP_DBG("Maximum IV length exceeded\n"); - return -1; - } + if (session->p.iv.length > MAX_IV_LEN) { + ODP_DBG("Maximum IV length exceeded\n"); + free_session(session); + return -1; + } + /* Copy IV data */ + if (session->p.iv.data) memcpy(session->cipher.iv_data, session->p.iv.data, session->p.iv.length); - } /* Derive order */ if (ODP_CRYPTO_OP_ENCODE == param->op) @@ -777,21 +707,25 @@ odp_crypto_session_create(odp_crypto_session_param_t *param, break; case ODP_CIPHER_ALG_DES: case ODP_CIPHER_ALG_3DES_CBC: - rc = process_des_param(session); + rc = process_cipher_param(session, EVP_des_ede3_cbc()); break; case ODP_CIPHER_ALG_AES_CBC: - /* deprecated */ +#if ODP_DEPRECATED_API case ODP_CIPHER_ALG_AES128_CBC: - rc = process_aes_param(session); +#endif + rc = process_cipher_param(session, EVP_aes_128_cbc()); break; - case ODP_CIPHER_ALG_AES_GCM: - /* deprecated */ +#if ODP_DEPRECATED_API case ODP_CIPHER_ALG_AES128_GCM: + if (param->auth_alg == ODP_AUTH_ALG_AES128_GCM) + aes_gcm = 1; + /* Fallthrough */ +#endif + case ODP_CIPHER_ALG_AES_GCM: /* AES-GCM requires to do both auth and * cipher at the same time */ - if (param->auth_alg == ODP_AUTH_ALG_AES_GCM || - param->auth_alg == ODP_AUTH_ALG_AES128_GCM) - rc = process_aes_gcm_param(session); + if (param->auth_alg == ODP_AUTH_ALG_AES_GCM || aes_gcm) + rc = process_aes_gcm_param(session, EVP_aes_128_gcm()); else rc = -1; break; @@ -802,32 +736,54 @@ odp_crypto_session_create(odp_crypto_session_param_t *param, /* Check result */ if (rc) { *status = ODP_CRYPTO_SES_CREATE_ERR_INV_CIPHER; + free_session(session); return -1; } + aes_gcm = 0; + /* Process based on auth */ switch (param->auth_alg) { case ODP_AUTH_ALG_NULL: session->auth.func = null_crypto_routine; rc = 0; break; - case ODP_AUTH_ALG_MD5_HMAC: - /* deprecated */ +#if ODP_DEPRECATED_API case ODP_AUTH_ALG_MD5_96: - rc = process_md5_param(session, 96); + /* Fixed digest tag length with deprecated algo */ + session->p.auth_digest_len = 96 / 8; + /* Fallthrough */ +#endif + case ODP_AUTH_ALG_MD5_HMAC: + rc = process_auth_param(session, 16, EVP_md5()); break; - case ODP_AUTH_ALG_SHA256_HMAC: - /* deprecated */ + case ODP_AUTH_ALG_SHA1_HMAC: + rc = process_auth_param(session, 20, EVP_sha1()); + break; +#if ODP_DEPRECATED_API case ODP_AUTH_ALG_SHA256_128: - rc = process_sha256_param(session, 128); + /* Fixed digest tag length with deprecated algo */ + session->p.auth_digest_len = 128 / 8; + /* Fallthrough */ +#endif + case ODP_AUTH_ALG_SHA256_HMAC: + rc = process_auth_param(session, 32, EVP_sha256()); break; - case ODP_AUTH_ALG_AES_GCM: - /* deprecated */ + case ODP_AUTH_ALG_SHA512_HMAC: + rc = process_auth_param(session, 64, EVP_sha512()); + break; +#if ODP_DEPRECATED_API case ODP_AUTH_ALG_AES128_GCM: + if (param->cipher_alg == ODP_CIPHER_ALG_AES128_GCM) + aes_gcm = 1; + /* Fixed digest tag length with deprecated algo */ + session->p.auth_digest_len = 16; + /* Fallthrough */ +#endif + case ODP_AUTH_ALG_AES_GCM: /* AES-GCM requires to do both auth and * cipher at the same time */ - if (param->cipher_alg == ODP_CIPHER_ALG_AES_GCM || - param->cipher_alg == ODP_CIPHER_ALG_AES128_GCM) { + if (param->cipher_alg == ODP_CIPHER_ALG_AES_GCM || aes_gcm) { session->auth.func = null_crypto_routine; rc = 0; } else { @@ -841,6 +797,7 @@ odp_crypto_session_create(odp_crypto_session_param_t *param, /* Check result */ if (rc) { *status = ODP_CRYPTO_SES_CREATE_ERR_INV_AUTH; + free_session(session); return -1; } @@ -854,9 +811,6 @@ int odp_crypto_session_destroy(odp_crypto_session_t session) odp_crypto_generic_session_t *generic; generic = (odp_crypto_generic_session_t *)(intptr_t)session; - if (generic->p.cipher_alg == ODP_CIPHER_ALG_AES128_GCM || - generic->p.cipher_alg == ODP_CIPHER_ALG_AES_GCM) - EVP_CIPHER_CTX_free(generic->cipher.data.aes_gcm.ctx); memset(generic, 0, sizeof(*generic)); free_session(generic); return 0; @@ -871,14 +825,17 @@ odp_crypto_operation(odp_crypto_op_param_t *param, odp_crypto_alg_err_t rc_auth = ODP_CRYPTO_ALG_ERR_NONE; odp_crypto_generic_session_t *session; odp_crypto_op_result_t local_result; + odp_bool_t allocated = false; session = (odp_crypto_generic_session_t *)(intptr_t)param->session; /* Resolve output buffer */ if (ODP_PACKET_INVALID == param->out_pkt && - ODP_POOL_INVALID != session->p.output_pool) + ODP_POOL_INVALID != session->p.output_pool) { param->out_pkt = odp_packet_alloc(session->p.output_pool, odp_packet_len(param->pkt)); + allocated = true; + } if (odp_unlikely(ODP_PACKET_INVALID == param->out_pkt)) { ODP_DBG("Alloc failed.\n"); @@ -886,11 +843,16 @@ odp_crypto_operation(odp_crypto_op_param_t *param, } if (param->pkt != param->out_pkt) { - (void)odp_packet_copy_from_pkt(param->out_pkt, + int ret; + + ret = odp_packet_copy_from_pkt(param->out_pkt, 0, param->pkt, 0, odp_packet_len(param->pkt)); + if (odp_unlikely(ret < 0)) + goto err; + _odp_packet_copy_md_to_packet(param->pkt, param->out_pkt); odp_packet_free(param->pkt); param->pkt = ODP_PACKET_INVALID; @@ -932,7 +894,7 @@ odp_crypto_operation(odp_crypto_op_param_t *param, op_result->result = local_result; if (odp_queue_enq(session->p.compl_queue, completion_event)) { odp_event_free(completion_event); - return -1; + goto err; } /* Indicate to caller operation was async */ @@ -940,13 +902,21 @@ odp_crypto_operation(odp_crypto_op_param_t *param, } else { /* Synchronous, simply return results */ if (!result) - return -1; + goto err; *result = local_result; /* Indicate to caller operation was sync */ *posted = 0; } return 0; + +err: + if (allocated) { + odp_packet_free(param->out_pkt); + param->out_pkt = ODP_PACKET_INVALID; + } + + return -1; } static void ODP_UNUSED openssl_thread_id(CRYPTO_THREADID ODP_UNUSED *id) diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 17a51b069..eb66af2d3 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -24,6 +24,8 @@ /* Initial packet segment data length */ #define BASE_LEN CONFIG_PACKET_MAX_SEG_LEN +#include <odp/visibility_begin.h> + /* Fill in packet header field offsets for inline functions */ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .data = offsetof(odp_packet_hdr_t, buf_hdr.seg[0].data), @@ -43,6 +45,8 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { }; +#include <odp/visibility_end.h> + static inline odp_packet_hdr_t *packet_hdr(odp_packet_t pkt) { return (odp_packet_hdr_t *)(uintptr_t)pkt; @@ -216,16 +220,9 @@ static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, return addr; } -static inline void packet_parse_disable(odp_packet_hdr_t *pkt_hdr) -{ - pkt_hdr->p.input_flags.parsed_l2 = 1; - pkt_hdr->p.parsed_layers = LAYER_ALL; -} - void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) { /* Reset parser metadata before new parse */ - pkt_hdr->p.parsed_layers = LAYER_NONE; pkt_hdr->p.error_flags.all = 0; pkt_hdr->p.input_flags.all = 0; pkt_hdr->p.output_flags.all = 0; @@ -237,8 +234,7 @@ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) /** * Initialize packet */ -static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len, - int parse) +static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len) { uint32_t seg_len; int num = pkt_hdr->buf_hdr.segcount; @@ -253,7 +249,6 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len, pkt_hdr->buf_hdr.seg[num - 1].len = seg_len; } - pkt_hdr->p.parsed_layers = LAYER_NONE; pkt_hdr->p.input_flags.all = 0; pkt_hdr->p.output_flags.all = 0; pkt_hdr->p.error_flags.all = 0; @@ -262,10 +257,6 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len, pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; - /* Disable lazy parsing on user allocated packets */ - if (!parse) - packet_parse_disable(pkt_hdr); - /* * Packet headroom is set from the pool's headroom * Packet tailroom is rounded up to fill the last @@ -481,7 +472,7 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, } static inline int packet_alloc(pool_t *pool, uint32_t len, int max_pkt, - int num_seg, odp_packet_t *pkt, int parse) + int num_seg, odp_packet_t *pkt) { int num_buf, i; int num = max_pkt; @@ -514,7 +505,7 @@ static inline int packet_alloc(pool_t *pool, uint32_t len, int max_pkt, pkt[i] = packet_handle(hdr); init_segments(&pkt_hdr[i * num_seg], num_seg); - packet_init(hdr, len, parse); + packet_init(hdr, len); } return num; @@ -527,7 +518,7 @@ int packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, int num, num_seg; num_seg = num_segments(len); - num = packet_alloc(pool, len, max_num, num_seg, pkt, 1); + num = packet_alloc(pool, len, max_num, num_seg, pkt); return num; } @@ -547,7 +538,7 @@ odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, uint32_t len) return ODP_PACKET_INVALID; num_seg = num_segments(len); - num = packet_alloc(pool, len, 1, num_seg, &pkt, 0); + num = packet_alloc(pool, len, 1, num_seg, &pkt); if (odp_unlikely(num == 0)) return ODP_PACKET_INVALID; @@ -570,7 +561,7 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, return -1; num_seg = num_segments(len); - num = packet_alloc(pool, len, max_num, num_seg, pkt, 0); + num = packet_alloc(pool, len, max_num, num_seg, pkt); return num; } @@ -590,11 +581,16 @@ void odp_packet_free(odp_packet_t pkt) void odp_packet_free_multi(const odp_packet_t pkt[], int num) { + odp_buffer_t buf[num * CONFIG_PACKET_MAX_SEGS]; + int i; + if (CONFIG_PACKET_MAX_SEGS == 1) { - buffer_free_multi((const odp_buffer_t * const)pkt, num); + for (i = 0; i < num; i++) + buf[i] = buffer_handle(packet_hdr(pkt[i])); + + buffer_free_multi(buf, num); } else { - odp_buffer_t buf[num * CONFIG_PACKET_MAX_SEGS]; - int i, j; + int j; int bufs = 0; for (i = 0; i < num; i++) { @@ -626,7 +622,7 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) if (len > pool->headroom + pool->data_size + pool->tailroom) return -1; - packet_init(pkt_hdr, len, 0); + packet_init(pkt_hdr, len); return 0; } @@ -795,7 +791,8 @@ static inline int move_data_to_head(odp_packet_hdr_t *pkt_hdr, int segs) free_len = BASE_LEN - len; - for (src_seg = dst_seg + 1; src_seg < segs; src_seg++) { + for (src_seg = dst_seg + 1; CONFIG_PACKET_MAX_SEGS > 1 && + src_seg < segs; src_seg++) { len = fill_seg_head(pkt_hdr, dst_seg, src_seg, free_len); moved += len; @@ -919,7 +916,7 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, pkt_hdr = new_hdr; *pkt = packet_handle(pkt_hdr); - } else if (free_segs) { + } else if (CONFIG_PACKET_MAX_SEGS > 1 && free_segs) { new_hdr = pkt_hdr->buf_hdr.seg[free_segs].hdr; packet_seg_copy_md(new_hdr, pkt_hdr); @@ -1237,8 +1234,6 @@ void *odp_packet_l3_ptr(odp_packet_t pkt, uint32_t *len) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (pkt_hdr->p.parsed_layers < LAYER_L3) - packet_parse_layer(pkt_hdr, LAYER_L3); return packet_map(pkt_hdr, pkt_hdr->p.l3_offset, len, NULL); } @@ -1246,8 +1241,6 @@ uint32_t odp_packet_l3_offset(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (pkt_hdr->p.parsed_layers < LAYER_L3) - packet_parse_layer(pkt_hdr, LAYER_L3); return pkt_hdr->p.l3_offset; } @@ -1258,8 +1251,6 @@ int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset) if (offset >= pkt_hdr->frame_len) return -1; - if (pkt_hdr->p.parsed_layers < LAYER_L3) - packet_parse_layer(pkt_hdr, LAYER_L3); pkt_hdr->p.l3_offset = offset; return 0; } @@ -1268,8 +1259,6 @@ void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (pkt_hdr->p.parsed_layers < LAYER_L4) - packet_parse_layer(pkt_hdr, LAYER_L4); return packet_map(pkt_hdr, pkt_hdr->p.l4_offset, len, NULL); } @@ -1277,8 +1266,6 @@ uint32_t odp_packet_l4_offset(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (pkt_hdr->p.parsed_layers < LAYER_L4) - packet_parse_layer(pkt_hdr, LAYER_L4); return pkt_hdr->p.l4_offset; } @@ -1289,8 +1276,6 @@ int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset) if (offset >= pkt_hdr->frame_len) return -1; - if (pkt_hdr->p.parsed_layers < LAYER_L4) - packet_parse_layer(pkt_hdr, LAYER_L4); pkt_hdr->p.l4_offset = offset; return 0; } @@ -1655,6 +1640,54 @@ int odp_packet_move_data(odp_packet_t pkt, uint32_t dst_offset, pkt, src_offset, len); } +int _odp_packet_set_data(odp_packet_t pkt, uint32_t offset, + uint8_t c, uint32_t len) +{ + void *mapaddr; + uint32_t seglen = 0; /* GCC */ + uint32_t setlen; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + if (offset + len > pkt_hdr->frame_len) + return -1; + + while (len > 0) { + mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); + setlen = len > seglen ? seglen : len; + memset(mapaddr, c, setlen); + offset += setlen; + len -= setlen; + } + + return 0; +} + +int _odp_packet_cmp_data(odp_packet_t pkt, uint32_t offset, + const void *s, uint32_t len) +{ + const uint8_t *ptr = s; + void *mapaddr; + uint32_t seglen = 0; /* GCC */ + uint32_t cmplen; + int ret; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + ODP_ASSERT(offset + len <= pkt_hdr->frame_len); + + while (len > 0) { + mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); + cmplen = len > seglen ? seglen : len; + ret = memcmp(mapaddr, ptr, cmplen); + if (ret != 0) + return ret; + offset += cmplen; + len -= cmplen; + ptr += cmplen; + } + + return 0; +} + /* * * Debugging @@ -1771,12 +1804,11 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, uint8_t ihl = _ODP_IPV4HDR_IHL(ipv4->ver_ihl); uint16_t frag_offset; uint32_t dstaddr = odp_be_to_cpu_32(ipv4->dst_addr); - - prs->l3_len = odp_be_to_cpu_16(ipv4->tot_len); + uint32_t l3_len = odp_be_to_cpu_16(ipv4->tot_len); if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN) || odp_unlikely(ver != 4) || - (prs->l3_len > frame_len - *offset)) { + (l3_len > frame_len - *offset)) { prs->error_flags.ip_err = 1; return 0; } @@ -1813,13 +1845,12 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, const _odp_ipv6hdr_t *ipv6 = (const _odp_ipv6hdr_t *)*parseptr; const _odp_ipv6hdr_ext_t *ipv6ext; uint32_t dstaddr0 = odp_be_to_cpu_32(ipv6->dst_addr.u8[0]); - - prs->l3_len = odp_be_to_cpu_16(ipv6->payload_len) + - _ODP_IPV6HDR_LEN; + uint32_t l3_len = odp_be_to_cpu_16(ipv6->payload_len) + + _ODP_IPV6HDR_LEN; /* Basic sanity checks on IPv6 header */ if ((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 || - prs->l3_len > frame_len - *offset) { + l3_len > frame_len - *offset) { prs->error_flags.ip_err = 1; return 0; } @@ -1880,9 +1911,6 @@ static inline void parse_tcp(packet_parser_t *prs, else if ((uint32_t)tcp->hl * 4 > sizeof(_odp_tcphdr_t)) prs->input_flags.tcpopt = 1; - prs->l4_len = prs->l3_len + - prs->l3_offset - prs->l4_offset; - if (offset) *offset += (uint32_t)tcp->hl * 4; *parseptr += (uint32_t)tcp->hl * 4; @@ -1897,13 +1925,8 @@ static inline void parse_udp(packet_parser_t *prs, const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr; uint32_t udplen = odp_be_to_cpu_16(udp->length); - if (udplen < sizeof(_odp_udphdr_t) || - udplen > (prs->l3_len + - prs->l4_offset - prs->l3_offset)) { + if (odp_unlikely(udplen < sizeof(_odp_udphdr_t))) prs->error_flags.udp_err = 1; - } - - prs->l4_len = udplen; if (offset) *offset += sizeof(_odp_udphdr_t); @@ -1911,215 +1934,170 @@ static inline void parse_udp(packet_parser_t *prs, } /** - * Initialize L2 related parser flags and metadata - */ -void packet_parse_l2(packet_parser_t *prs, uint32_t frame_len) -{ - /* Packet alloc or reset have already init other offsets and flags */ - - /* We only support Ethernet for now */ - prs->input_flags.eth = 1; - - /* Detect jumbo frames */ - if (frame_len > _ODP_ETH_LEN_MAX) - prs->input_flags.jumbo = 1; - - /* Assume valid L2 header, no CRC/FCS check in SW */ - prs->input_flags.l2 = 1; - - prs->input_flags.parsed_l2 = 1; -} - -/** * Parse common packet headers up to given layer * * The function expects at least PACKET_PARSE_SEG_LEN bytes of data to be * available from the ptr. */ int packet_parse_common(packet_parser_t *prs, const uint8_t *ptr, - uint32_t frame_len, uint32_t seg_len, layer_t layer) + uint32_t frame_len, uint32_t seg_len, + odp_pktio_parser_layer_t layer) { uint32_t offset; + uint16_t ethtype; const uint8_t *parseptr; + uint8_t ip_proto; + const _odp_ethhdr_t *eth; + uint16_t macaddr0, macaddr2, macaddr4; + const _odp_vlanhdr_t *vlan; - switch (prs->parsed_layers) { - case LAYER_NONE: - /* Fall through */ - - case LAYER_L2: - { - const _odp_ethhdr_t *eth; - uint16_t macaddr0, macaddr2, macaddr4; - const _odp_vlanhdr_t *vlan; - - offset = sizeof(_odp_ethhdr_t); - if (packet_parse_l2_not_done(prs)) - packet_parse_l2(prs, frame_len); - - eth = (const _odp_ethhdr_t *)ptr; - - /* Handle Ethernet broadcast/multicast addresses */ - macaddr0 = odp_be_to_cpu_16(*((const uint16_t *) - (const void *)eth)); - prs->input_flags.eth_mcast = (macaddr0 & 0x0100) == 0x0100; - - if (macaddr0 == 0xffff) { - macaddr2 = - odp_be_to_cpu_16(*((const uint16_t *) - (const void *)eth + 1)); - macaddr4 = - odp_be_to_cpu_16(*((const uint16_t *) - (const void *)eth + 2)); - prs->input_flags.eth_bcast = - (macaddr2 == 0xffff) && (macaddr4 == 0xffff); - } else { - prs->input_flags.eth_bcast = 0; - } + if (layer == ODP_PKTIO_PARSER_LAYER_NONE) + return 0; - /* Get Ethertype */ - prs->ethtype = odp_be_to_cpu_16(eth->type); - parseptr = (const uint8_t *)(eth + 1); + /* We only support Ethernet for now */ + prs->input_flags.eth = 1; + /* Assume valid L2 header, no CRC/FCS check in SW */ + prs->input_flags.l2 = 1; + /* Detect jumbo frames */ + if (frame_len > _ODP_ETH_LEN_MAX) + prs->input_flags.jumbo = 1; - /* Check for SNAP vs. DIX */ - if (prs->ethtype < _ODP_ETH_LEN_MAX) { - prs->input_flags.snap = 1; - if (prs->ethtype > frame_len - offset) { - prs->error_flags.snap_len = 1; - goto parse_exit; - } - prs->ethtype = odp_be_to_cpu_16(*((const uint16_t *) - (uintptr_t) - (parseptr + 6))); - offset += 8; - parseptr += 8; - } + offset = sizeof(_odp_ethhdr_t); + eth = (const _odp_ethhdr_t *)ptr; + + /* Handle Ethernet broadcast/multicast addresses */ + macaddr0 = odp_be_to_cpu_16(*((const uint16_t *)(const void *)eth)); + prs->input_flags.eth_mcast = (macaddr0 & 0x0100) == 0x0100; + + if (macaddr0 == 0xffff) { + macaddr2 = + odp_be_to_cpu_16(*((const uint16_t *) + (const void *)eth + 1)); + macaddr4 = + odp_be_to_cpu_16(*((const uint16_t *) + (const void *)eth + 2)); + prs->input_flags.eth_bcast = + (macaddr2 == 0xffff) && (macaddr4 == 0xffff); + } else { + prs->input_flags.eth_bcast = 0; + } - /* Parse the VLAN header(s), if present */ - if (prs->ethtype == _ODP_ETHTYPE_VLAN_OUTER) { - prs->input_flags.vlan_qinq = 1; - prs->input_flags.vlan = 1; + /* Get Ethertype */ + ethtype = odp_be_to_cpu_16(eth->type); + parseptr = (const uint8_t *)(eth + 1); - vlan = (const _odp_vlanhdr_t *)parseptr; - prs->ethtype = odp_be_to_cpu_16(vlan->type); - offset += sizeof(_odp_vlanhdr_t); - parseptr += sizeof(_odp_vlanhdr_t); + /* Check for SNAP vs. DIX */ + if (ethtype < _ODP_ETH_LEN_MAX) { + prs->input_flags.snap = 1; + if (ethtype > frame_len - offset) { + prs->error_flags.snap_len = 1; + goto parse_exit; } + ethtype = odp_be_to_cpu_16(*((const uint16_t *)(uintptr_t) + (parseptr + 6))); + offset += 8; + parseptr += 8; + } - if (prs->ethtype == _ODP_ETHTYPE_VLAN) { - prs->input_flags.vlan = 1; - vlan = (const _odp_vlanhdr_t *)parseptr; - prs->ethtype = odp_be_to_cpu_16(vlan->type); - offset += sizeof(_odp_vlanhdr_t); - parseptr += sizeof(_odp_vlanhdr_t); - } + /* Parse the VLAN header(s), if present */ + if (ethtype == _ODP_ETHTYPE_VLAN_OUTER) { + prs->input_flags.vlan_qinq = 1; + prs->input_flags.vlan = 1; - prs->l3_offset = offset; - prs->parsed_layers = LAYER_L2; - if (layer == LAYER_L2) - return prs->error_flags.all != 0; + vlan = (const _odp_vlanhdr_t *)parseptr; + ethtype = odp_be_to_cpu_16(vlan->type); + offset += sizeof(_odp_vlanhdr_t); + parseptr += sizeof(_odp_vlanhdr_t); } - /* Fall through */ - case LAYER_L3: - { - offset = prs->l3_offset; - parseptr = (const uint8_t *)(ptr + offset); - /* Set l3_offset+flag only for known ethtypes */ - prs->input_flags.l3 = 1; - - /* Parse Layer 3 headers */ - switch (prs->ethtype) { - case _ODP_ETHTYPE_IPV4: - prs->input_flags.ipv4 = 1; - prs->ip_proto = parse_ipv4(prs, &parseptr, &offset, - frame_len); - break; + if (ethtype == _ODP_ETHTYPE_VLAN) { + prs->input_flags.vlan = 1; + vlan = (const _odp_vlanhdr_t *)parseptr; + ethtype = odp_be_to_cpu_16(vlan->type); + offset += sizeof(_odp_vlanhdr_t); + parseptr += sizeof(_odp_vlanhdr_t); + } - case _ODP_ETHTYPE_IPV6: - prs->input_flags.ipv6 = 1; - prs->ip_proto = parse_ipv6(prs, &parseptr, &offset, - frame_len, seg_len); - break; + if (layer == ODP_PKTIO_PARSER_LAYER_L2) + return prs->error_flags.all != 0; - case _ODP_ETHTYPE_ARP: - prs->input_flags.arp = 1; - prs->ip_proto = 255; /* Reserved invalid by IANA */ - break; + /* Set l3_offset+flag only for known ethtypes */ + prs->l3_offset = offset; + prs->input_flags.l3 = 1; - default: - prs->input_flags.l3 = 0; - prs->l3_offset = ODP_PACKET_OFFSET_INVALID; - prs->ip_proto = 255; /* Reserved invalid by IANA */ - } + /* Parse Layer 3 headers */ + switch (ethtype) { + case _ODP_ETHTYPE_IPV4: + prs->input_flags.ipv4 = 1; + ip_proto = parse_ipv4(prs, &parseptr, &offset, frame_len); + break; - /* Set l4_offset+flag only for known ip_proto */ - prs->l4_offset = offset; - prs->parsed_layers = LAYER_L3; - if (layer == LAYER_L3) - return prs->error_flags.all != 0; - } - /* Fall through */ + case _ODP_ETHTYPE_IPV6: + prs->input_flags.ipv6 = 1; + ip_proto = parse_ipv6(prs, &parseptr, &offset, frame_len, + seg_len); + break; - case LAYER_L4: - { - offset = prs->l4_offset; - parseptr = (const uint8_t *)(ptr + offset); - prs->input_flags.l4 = 1; + case _ODP_ETHTYPE_ARP: + prs->input_flags.arp = 1; + ip_proto = 255; /* Reserved invalid by IANA */ + break; - /* Parse Layer 4 headers */ - switch (prs->ip_proto) { - case _ODP_IPPROTO_ICMP: - prs->input_flags.icmp = 1; - break; + default: + prs->input_flags.l3 = 0; + prs->l3_offset = ODP_PACKET_OFFSET_INVALID; + ip_proto = 255; /* Reserved invalid by IANA */ + } - case _ODP_IPPROTO_TCP: - if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len)) - return -1; - prs->input_flags.tcp = 1; - parse_tcp(prs, &parseptr, NULL); - break; + if (layer == ODP_PKTIO_PARSER_LAYER_L3) + return prs->error_flags.all != 0; - case _ODP_IPPROTO_UDP: - if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len)) - return -1; - prs->input_flags.udp = 1; - parse_udp(prs, &parseptr, NULL); - break; + /* Set l4_offset+flag only for known ip_proto */ + prs->l4_offset = offset; + prs->input_flags.l4 = 1; - case _ODP_IPPROTO_AH: - prs->input_flags.ipsec = 1; - prs->input_flags.ipsec_ah = 1; - break; + /* Parse Layer 4 headers */ + switch (ip_proto) { + case _ODP_IPPROTO_ICMPv4: + /* Fall through */ - case _ODP_IPPROTO_ESP: - prs->input_flags.ipsec = 1; - prs->input_flags.ipsec_esp = 1; - break; + case _ODP_IPPROTO_ICMPv6: + prs->input_flags.icmp = 1; + break; - case _ODP_IPPROTO_SCTP: - prs->input_flags.sctp = 1; - break; + case _ODP_IPPROTO_TCP: + if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len)) + return -1; + prs->input_flags.tcp = 1; + parse_tcp(prs, &parseptr, NULL); + break; - default: - prs->input_flags.l4 = 0; - prs->l4_offset = ODP_PACKET_OFFSET_INVALID; - break; - } + case _ODP_IPPROTO_UDP: + if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len)) + return -1; + prs->input_flags.udp = 1; + parse_udp(prs, &parseptr, NULL); + break; - prs->parsed_layers = LAYER_L4; + case _ODP_IPPROTO_AH: + prs->input_flags.ipsec = 1; + prs->input_flags.ipsec_ah = 1; + break; + + case _ODP_IPPROTO_ESP: + prs->input_flags.ipsec = 1; + prs->input_flags.ipsec_esp = 1; break; - } - case LAYER_ALL: + case _ODP_IPPROTO_SCTP: + prs->input_flags.sctp = 1; break; default: - ODP_ERR("Invalid parse layer: %d\n", (int)layer); - return -1; + prs->input_flags.l4 = 0; + prs->l4_offset = ODP_PACKET_OFFSET_INVALID; + break; } - - prs->parsed_layers = LAYER_ALL; - parse_exit: return prs->error_flags.all != 0; } @@ -2127,7 +2105,8 @@ parse_exit: /** * Simple packet parser */ -int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, layer_t layer) +int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, + odp_pktio_parser_layer_t layer) { uint32_t seg_len = packet_first_seg_len(pkt_hdr); void *base = packet_data(pkt_hdr); diff --git a/platform/linux-generic/odp_packet_flags.c b/platform/linux-generic/odp_packet_flags.c index ea9a22710..72df1ecfe 100644 --- a/platform/linux-generic/odp_packet_flags.c +++ b/platform/linux-generic/odp_packet_flags.c @@ -8,27 +8,21 @@ #include <odp/api/packet_flags.h> #include <odp_packet_internal.h> -#define retflag(pkt, x, layer) do { \ +#define retflag(pkt, x) do { \ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); \ - if (pkt_hdr->p.parsed_layers < layer) \ - packet_parse_layer(pkt_hdr, layer); \ return pkt_hdr->p.x; \ } while (0) -#define setflag(pkt, x, v, layer) do { \ +#define setflag(pkt, x, v) do { \ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); \ - if (pkt_hdr->p.parsed_layers < layer) \ - packet_parse_layer(pkt_hdr, layer); \ - pkt_hdr->p.x = v & 1; \ + pkt_hdr->p.x = (v) & 1; \ } while (0) int odp_packet_has_error(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (packet_parse_not_complete(pkt_hdr)) - packet_parse_layer(pkt_hdr, LAYER_ALL); - return odp_packet_hdr(pkt)->p.error_flags.all != 0; + return pkt_hdr->p.error_flags.all != 0; } /* Get Input Flags */ @@ -45,126 +39,117 @@ int odp_packet_has_l2_error(odp_packet_t pkt) int odp_packet_has_l3(odp_packet_t pkt) { - retflag(pkt, input_flags.l3, LAYER_L3); + retflag(pkt, input_flags.l3); } int odp_packet_has_l3_error(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (pkt_hdr->p.parsed_layers < LAYER_L3) - packet_parse_layer(pkt_hdr, LAYER_L3); - return pkt_hdr->p.error_flags.ip_err; } int odp_packet_has_l4(odp_packet_t pkt) { - retflag(pkt, input_flags.l4, LAYER_L4); + retflag(pkt, input_flags.l4); } int odp_packet_has_l4_error(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (pkt_hdr->p.parsed_layers < LAYER_L4) - packet_parse_layer(pkt_hdr, LAYER_L4); - return pkt_hdr->p.error_flags.tcp_err | pkt_hdr->p.error_flags.udp_err; } int odp_packet_has_eth_bcast(odp_packet_t pkt) { - retflag(pkt, input_flags.eth_bcast, LAYER_L2); + retflag(pkt, input_flags.eth_bcast); } int odp_packet_has_eth_mcast(odp_packet_t pkt) { - retflag(pkt, input_flags.eth_mcast, LAYER_L2); + retflag(pkt, input_flags.eth_mcast); } int odp_packet_has_vlan(odp_packet_t pkt) { - retflag(pkt, input_flags.vlan, LAYER_L2); + retflag(pkt, input_flags.vlan); } int odp_packet_has_vlan_qinq(odp_packet_t pkt) { - retflag(pkt, input_flags.vlan_qinq, LAYER_L2); + retflag(pkt, input_flags.vlan_qinq); } int odp_packet_has_arp(odp_packet_t pkt) { - retflag(pkt, input_flags.arp, LAYER_L3); + retflag(pkt, input_flags.arp); } int odp_packet_has_ipv4(odp_packet_t pkt) { - retflag(pkt, input_flags.ipv4, LAYER_L3); + retflag(pkt, input_flags.ipv4); } int odp_packet_has_ipv6(odp_packet_t pkt) { - retflag(pkt, input_flags.ipv6, LAYER_L3); + retflag(pkt, input_flags.ipv6); } int odp_packet_has_ip_bcast(odp_packet_t pkt) { - retflag(pkt, input_flags.ip_bcast, LAYER_L3); + retflag(pkt, input_flags.ip_bcast); } int odp_packet_has_ip_mcast(odp_packet_t pkt) { - retflag(pkt, input_flags.ip_mcast, LAYER_L3); + retflag(pkt, input_flags.ip_mcast); } int odp_packet_has_ipfrag(odp_packet_t pkt) { - retflag(pkt, input_flags.ipfrag, LAYER_L3); + retflag(pkt, input_flags.ipfrag); } int odp_packet_has_ipopt(odp_packet_t pkt) { - retflag(pkt, input_flags.ipopt, LAYER_L3); + retflag(pkt, input_flags.ipopt); } int odp_packet_has_ipsec(odp_packet_t pkt) { - retflag(pkt, input_flags.ipsec, LAYER_L4); + retflag(pkt, input_flags.ipsec); } int odp_packet_has_udp(odp_packet_t pkt) { - retflag(pkt, input_flags.udp, LAYER_L4); + retflag(pkt, input_flags.udp); } int odp_packet_has_tcp(odp_packet_t pkt) { - retflag(pkt, input_flags.tcp, LAYER_L4); + retflag(pkt, input_flags.tcp); } int odp_packet_has_sctp(odp_packet_t pkt) { - retflag(pkt, input_flags.sctp, LAYER_L4); + retflag(pkt, input_flags.sctp); } int odp_packet_has_icmp(odp_packet_t pkt) { - retflag(pkt, input_flags.icmp, LAYER_L4); + retflag(pkt, input_flags.icmp); } odp_packet_color_t odp_packet_color(odp_packet_t pkt) { - retflag(pkt, input_flags.color, LAYER_ALL); + retflag(pkt, input_flags.color); } void odp_packet_color_set(odp_packet_t pkt, odp_packet_color_t color) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (packet_parse_not_complete(pkt_hdr)) - packet_parse_layer(pkt_hdr, LAYER_ALL); - pkt_hdr->p.input_flags.color = color; } @@ -172,29 +157,23 @@ odp_bool_t odp_packet_drop_eligible(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (packet_parse_not_complete(pkt_hdr)) - packet_parse_layer(pkt_hdr, LAYER_ALL); - return !pkt_hdr->p.input_flags.nodrop; } void odp_packet_drop_eligible_set(odp_packet_t pkt, odp_bool_t drop) { - setflag(pkt, input_flags.nodrop, !drop, LAYER_ALL); + setflag(pkt, input_flags.nodrop, !drop); } int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt) { - retflag(pkt, output_flags.shaper_len_adj, LAYER_ALL); + retflag(pkt, output_flags.shaper_len_adj); } void odp_packet_shaper_len_adjust_set(odp_packet_t pkt, int8_t adj) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (packet_parse_not_complete(pkt_hdr)) - packet_parse_layer(pkt_hdr, LAYER_ALL); - pkt_hdr->p.output_flags.shaper_len_adj = adj; } @@ -202,107 +181,107 @@ void odp_packet_shaper_len_adjust_set(odp_packet_t pkt, int8_t adj) void odp_packet_has_l2_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.l2, val, LAYER_L2); + setflag(pkt, input_flags.l2, val); } void odp_packet_has_l3_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.l3, val, LAYER_L3); + setflag(pkt, input_flags.l3, val); } void odp_packet_has_l4_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.l4, val, LAYER_L4); + setflag(pkt, input_flags.l4, val); } void odp_packet_has_eth_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.eth, val, LAYER_L2); + setflag(pkt, input_flags.eth, val); } void odp_packet_has_eth_bcast_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.eth_bcast, val, LAYER_L2); + setflag(pkt, input_flags.eth_bcast, val); } void odp_packet_has_eth_mcast_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.eth_mcast, val, LAYER_L2); + setflag(pkt, input_flags.eth_mcast, val); } void odp_packet_has_jumbo_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.jumbo, val, LAYER_L2); + setflag(pkt, input_flags.jumbo, val); } void odp_packet_has_vlan_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.vlan, val, LAYER_L2); + setflag(pkt, input_flags.vlan, val); } void odp_packet_has_vlan_qinq_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.vlan_qinq, val, LAYER_L2); + setflag(pkt, input_flags.vlan_qinq, val); } void odp_packet_has_arp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.arp, val, LAYER_L3); + setflag(pkt, input_flags.arp, val); } void odp_packet_has_ipv4_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.ipv4, val, LAYER_L3); + setflag(pkt, input_flags.ipv4, val); } void odp_packet_has_ipv6_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.ipv6, val, LAYER_L3); + setflag(pkt, input_flags.ipv6, val); } void odp_packet_has_ip_bcast_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.ip_bcast, val, LAYER_L3); + setflag(pkt, input_flags.ip_bcast, val); } void odp_packet_has_ip_mcast_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.ip_mcast, val, LAYER_L3); + setflag(pkt, input_flags.ip_mcast, val); } void odp_packet_has_ipfrag_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.ipfrag, val, LAYER_L3); + setflag(pkt, input_flags.ipfrag, val); } void odp_packet_has_ipopt_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.ipopt, val, LAYER_L3); + setflag(pkt, input_flags.ipopt, val); } void odp_packet_has_ipsec_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.ipsec, val, LAYER_L4); + setflag(pkt, input_flags.ipsec, val); } void odp_packet_has_udp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.udp, val, LAYER_L4); + setflag(pkt, input_flags.udp, val); } void odp_packet_has_tcp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.tcp, val, LAYER_L4); + setflag(pkt, input_flags.tcp, val); } void odp_packet_has_sctp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.sctp, val, LAYER_L4); + setflag(pkt, input_flags.sctp, val); } void odp_packet_has_icmp_set(odp_packet_t pkt, int val) { - setflag(pkt, input_flags.icmp, val, LAYER_L4); + setflag(pkt, input_flags.icmp, val); } void odp_packet_has_flow_hash_clr(odp_packet_t pkt) diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 98460a566..877978ba2 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -206,6 +206,8 @@ static odp_pktio_t setup_pktio_entry(const char *name, odp_pool_t pool, memcpy(&pktio_entry->s.param, param, sizeof(odp_pktio_param_t)); pktio_entry->s.handle = hdl; + odp_pktio_config_init(&pktio_entry->s.config); + for (pktio_if = 0; pktio_if_ops[pktio_if]; ++pktio_if) { ret = pktio_if_ops[pktio_if]->open(hdl, pktio_entry, name, pool); @@ -586,6 +588,10 @@ int pktout_enqueue(queue_entry_t *qentry, odp_buffer_hdr_t *buf_hdr) int len = 1; int nbr; + if (sched_fn->ord_enq_multi(qentry->s.index, (void **)buf_hdr, len, + &nbr)) + return (nbr == len ? 0 : -1); + nbr = odp_pktout_send(qentry->s.pktout, &pkt, len); return (nbr == len ? 0 : -1); } @@ -603,6 +609,10 @@ int pktout_enq_multi(queue_entry_t *qentry, odp_buffer_hdr_t *buf_hdr[], int nbr; int i; + if (sched_fn->ord_enq_multi(qentry->s.index, (void **)buf_hdr, num, + &nbr)) + return nbr; + for (i = 0; i < num; ++i) pkt_tbl[i] = _odp_packet_from_buffer(buf_hdr[i]->handle.handle); @@ -923,6 +933,8 @@ void odp_pktout_queue_param_init(odp_pktout_queue_param_t *param) void odp_pktio_config_init(odp_pktio_config_t *config) { memset(config, 0, sizeof(odp_pktio_config_t)); + + config->parser.layer = ODP_PKTIO_PARSER_LAYER_ALL; } int odp_pktio_info(odp_pktio_t hdl, odp_pktio_info_t *info) @@ -1098,6 +1110,7 @@ int odp_pktio_term_global(void) int odp_pktio_capability(odp_pktio_t pktio, odp_pktio_capability_t *capa) { pktio_entry_t *entry; + int ret; entry = get_pktio_entry(pktio); if (entry == NULL) { @@ -1106,9 +1119,15 @@ int odp_pktio_capability(odp_pktio_t pktio, odp_pktio_capability_t *capa) } if (entry->s.ops->capability) - return entry->s.ops->capability(entry, capa); + ret = entry->s.ops->capability(entry, capa); + else + ret = single_capability(capa); - return single_capability(capa); + /* The same parser is used for all pktios */ + if (ret == 0) + capa->config.parser.layer = ODP_PKTIO_PARSER_LAYER_ALL; + + return ret; } unsigned odp_pktio_max_index(void) diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index cf7c2c415..9dba73413 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -445,7 +445,8 @@ static int check_params(odp_pool_param_t *params) { odp_pool_capability_t capa; - odp_pool_capability(&capa); + if (odp_pool_capability(&capa) < 0) + return -1; switch (params->type) { case ODP_POOL_BUFFER: @@ -768,6 +769,8 @@ odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl) pool_t *pool; int ret; + ODP_ASSERT(ODP_POOL_INVALID != pool_hdl); + pool = pool_entry_from_hdl(pool_hdl); ret = buffer_alloc_multi(pool, &buf, NULL, 1); @@ -781,6 +784,8 @@ int odp_buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int num) { pool_t *pool; + ODP_ASSERT(ODP_POOL_INVALID != pool_hdl); + pool = pool_entry_from_hdl(pool_hdl); return buffer_alloc_multi(pool, buf, NULL, num); diff --git a/platform/linux-generic/odp_queue.c b/platform/linux-generic/odp_queue.c index d4267c72f..450613394 100644 --- a/platform/linux-generic/odp_queue.c +++ b/platform/linux-generic/odp_queue.c @@ -95,6 +95,7 @@ static int queue_init(queue_entry_t *queue, const char *name, queue->s.dequeue_multi = queue_deq_multi; queue->s.pktin = PKTIN_INVALID; + queue->s.pktout = PKTOUT_INVALID; queue->s.head = NULL; queue->s.tail = NULL; @@ -175,6 +176,8 @@ int odp_queue_capability(odp_queue_capability_t *capa) capa->max_ordered_locks = sched_fn->max_ordered_locks(); capa->max_sched_groups = sched_fn->num_grps(); capa->sched_prios = odp_schedule_num_prio(); + capa->plain.max_num = capa->max_queues; + capa->sched.max_num = capa->max_queues; return 0; } diff --git a/platform/linux-generic/odp_rwlock.c b/platform/linux-generic/odp_rwlock.c index 13c17a2c7..5bef13a45 100644 --- a/platform/linux-generic/odp_rwlock.c +++ b/platform/linux-generic/odp_rwlock.c @@ -33,9 +33,14 @@ void odp_rwlock_read_lock(odp_rwlock_t *rwlock) int odp_rwlock_read_trylock(odp_rwlock_t *rwlock) { - uint32_t zero = 0; + uint32_t cnt = odp_atomic_load_u32(&rwlock->cnt); + + while (cnt != (uint32_t)-1) { + if (odp_atomic_cas_acq_u32(&rwlock->cnt, &cnt, cnt + 1)) + return 1; + } - return odp_atomic_cas_acq_u32(&rwlock->cnt, &zero, (uint32_t)1); + return 0; } void odp_rwlock_read_unlock(odp_rwlock_t *rwlock) diff --git a/platform/linux-generic/odp_schedule.c b/platform/linux-generic/odp_schedule.c index cd5bf21bd..c4567d810 100644 --- a/platform/linux-generic/odp_schedule.c +++ b/platform/linux-generic/odp_schedule.c @@ -20,6 +20,7 @@ #include <odp_config_internal.h> #include <odp_align_internal.h> #include <odp/api/sync.h> +#include <odp/api/packet_io.h> #include <odp_ring_internal.h> #include <odp_queue_internal.h> @@ -34,11 +35,18 @@ ODP_STATIC_ASSERT((ODP_SCHED_PRIO_NORMAL > 0) && "normal_prio_is_not_between_highest_and_lowest"); /* Number of scheduling groups */ -#define NUM_SCHED_GRPS 256 +#define NUM_SCHED_GRPS 32 /* Priority queues per priority */ #define QUEUES_PER_PRIO 4 +/* A thread polls a non preferred sched queue every this many polls + * of the prefer queue. */ +#define PREFER_RATIO 64 + +/* Size of poll weight table */ +#define WEIGHT_TBL_SIZE ((QUEUES_PER_PRIO - 1) * PREFER_RATIO) + /* Packet input poll cmd queues */ #define PKTIO_CMD_QUEUES 4 @@ -142,7 +150,6 @@ typedef struct { int index; int pause; uint16_t round; - uint16_t prefer_offset; uint16_t pktin_polls; uint32_t queue_index; odp_queue_t queue; @@ -157,6 +164,12 @@ typedef struct { ordered_stash_t stash[MAX_ORDERED_STASH]; } ordered; + uint32_t grp_epoch; + int num_grp; + uint8_t grp[NUM_SCHED_GRPS]; + uint8_t weight_tbl[WEIGHT_TBL_SIZE]; + uint8_t grp_weight[WEIGHT_TBL_SIZE]; + } sched_local_t; /* Priority queue */ @@ -191,7 +204,7 @@ typedef struct { pri_mask_t pri_mask[NUM_PRIO]; odp_spinlock_t mask_lock; - prio_queue_t prio_q[NUM_PRIO][QUEUES_PER_PRIO]; + prio_queue_t prio_q[NUM_SCHED_GRPS][NUM_PRIO][QUEUES_PER_PRIO]; odp_spinlock_t poll_cmd_lock; /* Number of commands in a command queue */ @@ -206,8 +219,10 @@ typedef struct { odp_shm_t shm; uint32_t pri_count[NUM_PRIO][QUEUES_PER_PRIO]; - odp_spinlock_t grp_lock; - odp_thrmask_t mask_all; + odp_thrmask_t mask_all; + odp_spinlock_t grp_lock; + odp_atomic_u32_t grp_epoch; + struct { char name[ODP_SCHED_GROUP_NAME_LEN]; odp_thrmask_t mask; @@ -215,6 +230,7 @@ typedef struct { } sched_grp[NUM_SCHED_GRPS]; struct { + int grp; int prio; int queue_per_prio; } queue[ODP_CONFIG_QUEUES]; @@ -237,17 +253,35 @@ static inline void schedule_release_context(void); static void sched_local_init(void) { + int i; + uint8_t id; + uint8_t offset = 0; + memset(&sched_local, 0, sizeof(sched_local_t)); sched_local.thr = odp_thread_id(); sched_local.queue = ODP_QUEUE_INVALID; sched_local.queue_index = PRIO_QUEUE_EMPTY; + + id = sched_local.thr & (QUEUES_PER_PRIO - 1); + + for (i = 0; i < WEIGHT_TBL_SIZE; i++) { + sched_local.weight_tbl[i] = id; + + if (i % PREFER_RATIO == 0) { + offset++; + sched_local.weight_tbl[i] = (id + offset) & + (QUEUES_PER_PRIO - 1); + if (offset == QUEUES_PER_PRIO - 1) + offset = 0; + } + } } static int schedule_init_global(void) { odp_shm_t shm; - int i, j; + int i, j, grp; ODP_DBG("Schedule init ... "); @@ -267,15 +301,20 @@ static int schedule_init_global(void) sched->shm = shm; odp_spinlock_init(&sched->mask_lock); - for (i = 0; i < NUM_PRIO; i++) { - for (j = 0; j < QUEUES_PER_PRIO; j++) { - int k; + for (grp = 0; grp < NUM_SCHED_GRPS; grp++) { + for (i = 0; i < NUM_PRIO; i++) { + for (j = 0; j < QUEUES_PER_PRIO; j++) { + prio_queue_t *prio_q; + int k; - ring_init(&sched->prio_q[i][j].ring); + prio_q = &sched->prio_q[grp][i][j]; + ring_init(&prio_q->ring); - for (k = 0; k < PRIO_QUEUE_RING_SIZE; k++) - sched->prio_q[i][j].queue_index[k] = - PRIO_QUEUE_EMPTY; + for (k = 0; k < PRIO_QUEUE_RING_SIZE; k++) { + prio_q->queue_index[k] = + PRIO_QUEUE_EMPTY; + } + } } } @@ -291,12 +330,17 @@ static int schedule_init_global(void) sched->pktio_cmd[i].cmd_index = PKTIO_CMD_FREE; odp_spinlock_init(&sched->grp_lock); + odp_atomic_init_u32(&sched->grp_epoch, 0); for (i = 0; i < NUM_SCHED_GRPS; i++) { memset(sched->sched_grp[i].name, 0, ODP_SCHED_GROUP_NAME_LEN); odp_thrmask_zero(&sched->sched_grp[i].mask); } + sched->sched_grp[ODP_SCHED_GROUP_ALL].allocated = 1; + sched->sched_grp[ODP_SCHED_GROUP_WORKER].allocated = 1; + sched->sched_grp[ODP_SCHED_GROUP_CONTROL].allocated = 1; + odp_thrmask_setall(&sched->mask_all); ODP_DBG("done\n"); @@ -304,29 +348,38 @@ static int schedule_init_global(void) return 0; } +static inline void queue_destroy_finalize(uint32_t qi) +{ + sched_cb_queue_destroy_finalize(qi); +} + static int schedule_term_global(void) { int ret = 0; int rc = 0; - int i, j; + int i, j, grp; - for (i = 0; i < NUM_PRIO; i++) { - for (j = 0; j < QUEUES_PER_PRIO; j++) { - ring_t *ring = &sched->prio_q[i][j].ring; - uint32_t qi; + for (grp = 0; grp < NUM_SCHED_GRPS; grp++) { + for (i = 0; i < NUM_PRIO; i++) { + for (j = 0; j < QUEUES_PER_PRIO; j++) { + ring_t *ring = &sched->prio_q[grp][i][j].ring; + uint32_t qi; - while ((qi = ring_deq(ring, PRIO_QUEUE_MASK)) != - RING_EMPTY) { - odp_event_t events[1]; - int num; + while ((qi = ring_deq(ring, PRIO_QUEUE_MASK)) != + RING_EMPTY) { + odp_event_t events[1]; + int num; - num = sched_cb_queue_deq_multi(qi, events, 1); + num = sched_cb_queue_deq_multi(qi, + events, + 1); - if (num < 0) - sched_cb_queue_destroy_finalize(qi); + if (num < 0) + queue_destroy_finalize(qi); - if (num > 0) - ODP_ERR("Queue not empty\n"); + if (num > 0) + ODP_ERR("Queue not empty\n"); + } } } } @@ -357,6 +410,40 @@ static int schedule_term_local(void) return 0; } +static inline void grp_update_mask(int grp, const odp_thrmask_t *new_mask) +{ + odp_thrmask_copy(&sched->sched_grp[grp].mask, new_mask); + odp_atomic_add_rel_u32(&sched->grp_epoch, 1); +} + +static inline int grp_update_tbl(void) +{ + int i; + int num = 0; + int thr = sched_local.thr; + + odp_spinlock_lock(&sched->grp_lock); + + for (i = 0; i < NUM_SCHED_GRPS; i++) { + if (sched->sched_grp[i].allocated == 0) + continue; + + if (odp_thrmask_isset(&sched->sched_grp[i].mask, thr)) { + sched_local.grp[num] = i; + num++; + } + } + + odp_spinlock_unlock(&sched->grp_lock); + + /* Update group weights. Round robin over all thread's groups. */ + for (i = 0; i < WEIGHT_TBL_SIZE; i++) + sched_local.grp_weight[i] = i % num; + + sched_local.num_grp = num; + return num; +} + static unsigned schedule_max_ordered_locks(void) { return MAX_ORDERED_LOCKS_PER_QUEUE; @@ -407,6 +494,7 @@ static int schedule_init_queue(uint32_t queue_index, int prio = sched_param->prio; pri_set_queue(queue_index, prio); + sched->queue[queue_index].grp = sched_param->group; sched->queue[queue_index].prio = prio; sched->queue[queue_index].queue_per_prio = queue_per_prio(queue_index); @@ -418,6 +506,7 @@ static void schedule_destroy_queue(uint32_t queue_index) int prio = sched->queue[queue_index].prio; pri_clr_queue(queue_index, prio); + sched->queue[queue_index].grp = 0; sched->queue[queue_index].prio = 0; sched->queue[queue_index].queue_per_prio = 0; } @@ -509,9 +598,10 @@ static void schedule_release_atomic(void) uint32_t qi = sched_local.queue_index; if (qi != PRIO_QUEUE_EMPTY && sched_local.num == 0) { - int prio = sched->queue[qi].prio; + int grp = sched->queue[qi].grp; + int prio = sched->queue[qi].prio; int queue_per_prio = sched->queue[qi].queue_per_prio; - ring_t *ring = &sched->prio_q[prio][queue_per_prio].ring; + ring_t *ring = &sched->prio_q[grp][prio][queue_per_prio].ring; /* Release current atomic queue */ ring_enq(ring, PRIO_QUEUE_MASK, qi); @@ -640,7 +730,9 @@ static int schedule_ord_enq_multi(uint32_t queue_index, void *buf_hdr[], return 0; } - if (odp_unlikely(stash_num >= MAX_ORDERED_STASH)) { + /* Pktout may drop packets, so the operation cannot be stashed. */ + if (dst_queue->s.pktout.pktio != ODP_PKTIO_INVALID || + odp_unlikely(stash_num >= MAX_ORDERED_STASH)) { /* If the local stash is full, wait until it is our turn and * then release the stash and do enqueue directly. */ wait_for_order(src_queue); @@ -662,54 +754,26 @@ static int schedule_ord_enq_multi(uint32_t queue_index, void *buf_hdr[], return 1; } -/* - * Schedule queues - */ -static int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[], - unsigned int max_num) +static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], + unsigned int max_num, int grp, int first) { int prio, i; int ret; int id; - int offset = 0; unsigned int max_deq = MAX_DEQ; uint32_t qi; - if (sched_local.num) { - ret = copy_events(out_ev, max_num); - - if (out_queue) - *out_queue = sched_local.queue; - - return ret; - } - - schedule_release_context(); - - if (odp_unlikely(sched_local.pause)) - return 0; - - /* Each thread prefers a priority queue. This offset avoids starvation - * of other priority queues on low thread counts. */ - if (odp_unlikely((sched_local.round & 0x3f) == 0)) { - offset = sched_local.prefer_offset; - sched_local.prefer_offset = (offset + 1) & - (QUEUES_PER_PRIO - 1); - } - - sched_local.round++; - /* Schedule events */ for (prio = 0; prio < NUM_PRIO; prio++) { if (sched->pri_mask[prio] == 0) continue; - id = (sched_local.thr + offset) & (QUEUES_PER_PRIO - 1); + /* Select the first ring based on weights */ + id = first; for (i = 0; i < QUEUES_PER_PRIO;) { int num; - int grp; int ordered; odp_queue_t handle; ring_t *ring; @@ -726,7 +790,7 @@ static int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[], } /* Get queue index from the priority queue */ - ring = &sched->prio_q[prio][id].ring; + ring = &sched->prio_q[grp][prio][id].ring; qi = ring_deq(ring, PRIO_QUEUE_MASK); /* Priority queue empty */ @@ -736,24 +800,10 @@ static int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[], continue; } - grp = sched_cb_queue_grp(qi); - - if (grp > ODP_SCHED_GROUP_ALL && - !odp_thrmask_isset(&sched->sched_grp[grp].mask, - sched_local.thr)) { - /* This thread is not eligible for work from - * this queue, so continue scheduling it. - */ - ring_enq(ring, PRIO_QUEUE_MASK, qi); - - i++; - id++; - continue; - } - /* Low priorities have smaller batch size to limit * head of line blocking latency. */ - if (odp_unlikely(prio > ODP_SCHED_PRIO_DEFAULT)) + if (odp_unlikely(MAX_DEQ > 1 && + prio > ODP_SCHED_PRIO_DEFAULT)) max_deq = MAX_DEQ / 2; ordered = sched_cb_queue_is_ordered(qi); @@ -818,6 +868,70 @@ static int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[], } } + return 0; +} + +/* + * Schedule queues + */ +static inline int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[], + unsigned int max_num) +{ + int i, num_grp; + int ret; + int id, first, grp_id; + uint16_t round; + uint32_t epoch; + + if (sched_local.num) { + ret = copy_events(out_ev, max_num); + + if (out_queue) + *out_queue = sched_local.queue; + + return ret; + } + + schedule_release_context(); + + if (odp_unlikely(sched_local.pause)) + return 0; + + /* Each thread prefers a priority queue. Poll weight table avoids + * starvation of other priority queues on low thread counts. */ + round = sched_local.round + 1; + + if (odp_unlikely(round == WEIGHT_TBL_SIZE)) + round = 0; + + sched_local.round = round; + first = sched_local.weight_tbl[round]; + + epoch = odp_atomic_load_acq_u32(&sched->grp_epoch); + num_grp = sched_local.num_grp; + + if (odp_unlikely(sched_local.grp_epoch != epoch)) { + num_grp = grp_update_tbl(); + sched_local.grp_epoch = epoch; + } + + grp_id = sched_local.grp_weight[round]; + + /* Schedule queues per group and priority */ + for (i = 0; i < num_grp; i++) { + int grp; + + grp = sched_local.grp[grp_id]; + ret = do_schedule_grp(out_queue, out_ev, max_num, grp, first); + + if (odp_likely(ret)) + return ret; + + grp_id++; + if (odp_unlikely(grp_id >= num_grp)) + grp_id = 0; + } + /* * Poll packet input when there are no events * * Each thread starts the search for a poll command from its @@ -1023,7 +1137,8 @@ static odp_schedule_group_t schedule_group_create(const char *name, ODP_SCHED_GROUP_NAME_LEN - 1); grp_name[ODP_SCHED_GROUP_NAME_LEN - 1] = 0; } - odp_thrmask_copy(&sched->sched_grp[i].mask, mask); + + grp_update_mask(i, mask); group = (odp_schedule_group_t)i; sched->sched_grp[i].allocated = 1; break; @@ -1036,13 +1151,16 @@ static odp_schedule_group_t schedule_group_create(const char *name, static int schedule_group_destroy(odp_schedule_group_t group) { + odp_thrmask_t zero; int ret; + odp_thrmask_zero(&zero); + odp_spinlock_lock(&sched->grp_lock); if (group < NUM_SCHED_GRPS && group >= SCHED_GROUP_NAMED && sched->sched_grp[group].allocated) { - odp_thrmask_zero(&sched->sched_grp[group].mask); + grp_update_mask(group, &zero); memset(sched->sched_grp[group].name, 0, ODP_SCHED_GROUP_NAME_LEN); sched->sched_grp[group].allocated = 0; @@ -1082,9 +1200,11 @@ static int schedule_group_join(odp_schedule_group_t group, if (group < NUM_SCHED_GRPS && group >= SCHED_GROUP_NAMED && sched->sched_grp[group].allocated) { - odp_thrmask_or(&sched->sched_grp[group].mask, - &sched->sched_grp[group].mask, - mask); + odp_thrmask_t new_mask; + + odp_thrmask_or(&new_mask, &sched->sched_grp[group].mask, mask); + grp_update_mask(group, &new_mask); + ret = 0; } else { ret = -1; @@ -1097,18 +1217,19 @@ static int schedule_group_join(odp_schedule_group_t group, static int schedule_group_leave(odp_schedule_group_t group, const odp_thrmask_t *mask) { + odp_thrmask_t new_mask; int ret; + odp_thrmask_xor(&new_mask, mask, &sched->mask_all); + odp_spinlock_lock(&sched->grp_lock); if (group < NUM_SCHED_GRPS && group >= SCHED_GROUP_NAMED && sched->sched_grp[group].allocated) { - odp_thrmask_t leavemask; + odp_thrmask_and(&new_mask, &sched->sched_grp[group].mask, + &new_mask); + grp_update_mask(group, &new_mask); - odp_thrmask_xor(&leavemask, mask, &sched->mask_all); - odp_thrmask_and(&sched->sched_grp[group].mask, - &sched->sched_grp[group].mask, - &leavemask); ret = 0; } else { ret = -1; @@ -1159,12 +1280,19 @@ static int schedule_group_info(odp_schedule_group_t group, static int schedule_thr_add(odp_schedule_group_t group, int thr) { + odp_thrmask_t mask; + odp_thrmask_t new_mask; + if (group < 0 || group >= SCHED_GROUP_NAMED) return -1; + odp_thrmask_zero(&mask); + odp_thrmask_set(&mask, thr); + odp_spinlock_lock(&sched->grp_lock); - odp_thrmask_set(&sched->sched_grp[group].mask, thr); + odp_thrmask_or(&new_mask, &sched->sched_grp[group].mask, &mask); + grp_update_mask(group, &new_mask); odp_spinlock_unlock(&sched->grp_lock); @@ -1173,12 +1301,20 @@ static int schedule_thr_add(odp_schedule_group_t group, int thr) static int schedule_thr_rem(odp_schedule_group_t group, int thr) { + odp_thrmask_t mask; + odp_thrmask_t new_mask; + if (group < 0 || group >= SCHED_GROUP_NAMED) return -1; + odp_thrmask_zero(&mask); + odp_thrmask_set(&mask, thr); + odp_thrmask_xor(&new_mask, &mask, &sched->mask_all); + odp_spinlock_lock(&sched->grp_lock); - odp_thrmask_clr(&sched->sched_grp[group].mask, thr); + odp_thrmask_and(&new_mask, &sched->sched_grp[group].mask, &new_mask); + grp_update_mask(group, &new_mask); odp_spinlock_unlock(&sched->grp_lock); @@ -1192,9 +1328,10 @@ static void schedule_prefetch(int num ODP_UNUSED) static int schedule_sched_queue(uint32_t queue_index) { + int grp = sched->queue[queue_index].grp; int prio = sched->queue[queue_index].prio; int queue_per_prio = sched->queue[queue_index].queue_per_prio; - ring_t *ring = &sched->prio_q[prio][queue_per_prio].ring; + ring_t *ring = &sched->prio_q[grp][prio][queue_per_prio].ring; ring_enq(ring, PRIO_QUEUE_MASK, queue_index); return 0; diff --git a/platform/linux-generic/odp_schedule_iquery.c b/platform/linux-generic/odp_schedule_iquery.c index b69245788..75470aff8 100644 --- a/platform/linux-generic/odp_schedule_iquery.c +++ b/platform/linux-generic/odp_schedule_iquery.c @@ -21,6 +21,7 @@ #include <odp/api/hints.h> #include <odp/api/cpu.h> #include <odp/api/thrmask.h> +#include <odp/api/packet_io.h> #include <odp_config_internal.h> /* Number of priority levels */ @@ -1176,7 +1177,9 @@ static int schedule_ord_enq_multi(uint32_t queue_index, void *buf_hdr[], return 0; } - if (odp_unlikely(stash_num >= MAX_ORDERED_STASH)) { + /* Pktout may drop packets, so the operation cannot be stashed. */ + if (dst_queue->s.pktout.pktio != ODP_PKTIO_INVALID || + odp_unlikely(stash_num >= MAX_ORDERED_STASH)) { /* If the local stash is full, wait until it is our turn and * then release the stash and do enqueue directly. */ wait_for_order(src_queue); @@ -1457,7 +1460,7 @@ static inline int consume_queue(int prio, unsigned int queue_index) /* Low priorities have smaller batch size to limit * head of line blocking latency. */ - if (odp_unlikely(prio > ODP_SCHED_PRIO_DEFAULT)) + if (odp_unlikely(MAX_DEQ > 1 && prio > ODP_SCHED_PRIO_DEFAULT)) max = MAX_DEQ / 2; /* For ordered queues we want consecutive events to diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index 18c61dbe7..40ffca078 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -14,6 +14,7 @@ #include <odp_posix_extensions.h> #include <odp/api/system_info.h> +#include <odp/api/version.h> #include <odp_internal.h> #include <odp_debug_internal.h> #include <odp/api/align.h> @@ -405,3 +406,32 @@ int odp_cpu_count(void) { return odp_global_data.system_info.cpu_count; } + +void odp_sys_info_print(void) +{ + int len; + int max_len = 512; + char str[max_len]; + + len = snprintf(str, max_len, "\n" + "ODP system info\n" + "---------------\n" + "ODP API version: %s\n" + "ODP impl name: %s\n" + "CPU model: %s\n" + "CPU freq (hz): %" PRIu64 "\n" + "Cache line size: %i\n" + "CPU count: %i\n" + "\n", + odp_version_api_str(), + odp_version_impl_name(), + odp_cpu_model_str(), + odp_cpu_hz_max(), + odp_sys_cache_line_size(), + odp_cpu_count()); + + str[len] = '\0'; + ODP_PRINT("%s", str); + + sys_info_print_arch(); +} diff --git a/platform/linux-generic/odp_time.c b/platform/linux-generic/odp_time.c index 81e05224c..2bbe56664 100644 --- a/platform/linux-generic/odp_time.c +++ b/platform/linux-generic/odp_time.c @@ -10,36 +10,43 @@ #include <odp/api/time.h> #include <odp/api/hints.h> #include <odp_debug_internal.h> +#include <odp_time_internal.h> +#include <string.h> +#include <inttypes.h> -static odp_time_t start_time; +typedef struct time_global_t { + struct timespec spec_start; + int use_hw; + uint64_t hw_start; + uint64_t hw_freq_hz; +} time_global_t; -static inline -uint64_t time_to_ns(odp_time_t time) -{ - uint64_t ns; - - ns = time.tv_sec * ODP_TIME_SEC_IN_NS; - ns += time.tv_nsec; +static time_global_t global; - return ns; -} +/* + * Posix timespec based functions + */ -static inline odp_time_t time_diff(odp_time_t t2, odp_time_t t1) +static inline uint64_t time_spec_diff_nsec(struct timespec *t2, + struct timespec *t1) { - odp_time_t time; + struct timespec diff; + uint64_t nsec; - time.tv_sec = t2.tv_sec - t1.tv_sec; - time.tv_nsec = t2.tv_nsec - t1.tv_nsec; + diff.tv_sec = t2->tv_sec - t1->tv_sec; + diff.tv_nsec = t2->tv_nsec - t1->tv_nsec; - if (time.tv_nsec < 0) { - time.tv_nsec += ODP_TIME_SEC_IN_NS; - --time.tv_sec; + if (diff.tv_nsec < 0) { + diff.tv_nsec += ODP_TIME_SEC_IN_NS; + diff.tv_sec -= 1; } - return time; + nsec = (diff.tv_sec * ODP_TIME_SEC_IN_NS) + diff.tv_nsec; + + return nsec; } -static inline odp_time_t time_local(void) +static inline odp_time_t time_spec_cur(void) { int ret; odp_time_t time; @@ -49,82 +56,176 @@ static inline odp_time_t time_local(void) if (odp_unlikely(ret != 0)) ODP_ABORT("clock_gettime failed\n"); - time.tv_sec = sys_time.tv_sec; - time.tv_nsec = sys_time.tv_nsec; + time.nsec = time_spec_diff_nsec(&sys_time, &global.spec_start); - return time_diff(time, start_time); + return time; } -static inline int time_cmp(odp_time_t t2, odp_time_t t1) +static inline uint64_t time_spec_res(void) { - if (t2.tv_sec < t1.tv_sec) - return -1; + int ret; + struct timespec tres; - if (t2.tv_sec > t1.tv_sec) - return 1; + ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); + if (odp_unlikely(ret != 0)) + ODP_ABORT("clock_getres failed\n"); - return t2.tv_nsec - t1.tv_nsec; + return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec; } -static inline odp_time_t time_sum(odp_time_t t1, odp_time_t t2) +static inline uint64_t time_spec_to_ns(odp_time_t time) +{ + return time.nsec; +} + +static inline odp_time_t time_spec_from_ns(uint64_t ns) { odp_time_t time; - time.tv_sec = t2.tv_sec + t1.tv_sec; - time.tv_nsec = t2.tv_nsec + t1.tv_nsec; + time.nsec = ns; + + return time; +} + +/* + * HW time counter based functions + */ - if (time.tv_nsec >= (long)ODP_TIME_SEC_IN_NS) { - time.tv_nsec -= ODP_TIME_SEC_IN_NS; - ++time.tv_sec; +static inline odp_time_t time_hw_cur(void) +{ + odp_time_t time; + + time.count = cpu_global_time() - global.hw_start; + + return time; +} + +static inline uint64_t time_hw_res(void) +{ + /* Promise a bit lower resolution than average cycle counter + * frequency */ + return global.hw_freq_hz / 10; +} + +static inline uint64_t time_hw_to_ns(odp_time_t time) +{ + uint64_t nsec; + uint64_t freq_hz = global.hw_freq_hz; + uint64_t count = time.count; + uint64_t sec = 0; + + if (count >= freq_hz) { + sec = count / freq_hz; + count = count - sec * freq_hz; } + nsec = (ODP_TIME_SEC_IN_NS * count) / freq_hz; + + return (sec * ODP_TIME_SEC_IN_NS) + nsec; +} + +static inline odp_time_t time_hw_from_ns(uint64_t ns) +{ + odp_time_t time; + uint64_t count; + uint64_t freq_hz = global.hw_freq_hz; + uint64_t sec = 0; + + if (ns >= ODP_TIME_SEC_IN_NS) { + sec = ns / ODP_TIME_SEC_IN_NS; + ns = ns - sec * ODP_TIME_SEC_IN_NS; + } + + count = sec * freq_hz; + count += (ns * freq_hz) / ODP_TIME_SEC_IN_NS; + + time.count = count; + return time; } -static inline odp_time_t time_local_from_ns(uint64_t ns) +/* + * Common functions + */ + +static inline odp_time_t time_cur(void) +{ + if (global.use_hw) + return time_hw_cur(); + + return time_spec_cur(); +} + +static inline uint64_t time_res(void) +{ + if (global.use_hw) + return time_hw_res(); + + return time_spec_res(); +} + +static inline int time_cmp(odp_time_t t2, odp_time_t t1) +{ + if (odp_likely(t2.u64 > t1.u64)) + return 1; + + if (t2.u64 < t1.u64) + return -1; + + return 0; +} + +static inline odp_time_t time_sum(odp_time_t t1, odp_time_t t2) { odp_time_t time; - time.tv_sec = ns / ODP_TIME_SEC_IN_NS; - time.tv_nsec = ns - time.tv_sec * ODP_TIME_SEC_IN_NS; + time.u64 = t1.u64 + t2.u64; return time; } -static inline void time_wait_until(odp_time_t time) +static inline uint64_t time_to_ns(odp_time_t time) { - odp_time_t cur; + if (global.use_hw) + return time_hw_to_ns(time); - do { - cur = time_local(); - } while (time_cmp(time, cur) > 0); + return time_spec_to_ns(time); } -static inline uint64_t time_local_res(void) +static inline odp_time_t time_from_ns(uint64_t ns) { - int ret; - struct timespec tres; + if (global.use_hw) + return time_hw_from_ns(ns); - ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); - if (odp_unlikely(ret != 0)) - ODP_ABORT("clock_getres failed\n"); + return time_spec_from_ns(ns); +} - return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec; +static inline void time_wait_until(odp_time_t time) +{ + odp_time_t cur; + + do { + cur = time_cur(); + } while (time_cmp(time, cur) > 0); } odp_time_t odp_time_local(void) { - return time_local(); + return time_cur(); } odp_time_t odp_time_global(void) { - return time_local(); + return time_cur(); } odp_time_t odp_time_diff(odp_time_t t2, odp_time_t t1) { - return time_diff(t2, t1); + odp_time_t time; + + time.u64 = t2.u64 - t1.u64; + + return time; } uint64_t odp_time_to_ns(odp_time_t time) @@ -134,12 +235,12 @@ uint64_t odp_time_to_ns(odp_time_t time) odp_time_t odp_time_local_from_ns(uint64_t ns) { - return time_local_from_ns(ns); + return time_from_ns(ns); } odp_time_t odp_time_global_from_ns(uint64_t ns) { - return time_local_from_ns(ns); + return time_from_ns(ns); } int odp_time_cmp(odp_time_t t2, odp_time_t t1) @@ -154,18 +255,18 @@ odp_time_t odp_time_sum(odp_time_t t1, odp_time_t t2) uint64_t odp_time_local_res(void) { - return time_local_res(); + return time_res(); } uint64_t odp_time_global_res(void) { - return time_local_res(); + return time_res(); } void odp_time_wait_ns(uint64_t ns) { - odp_time_t cur = time_local(); - odp_time_t wait = time_local_from_ns(ns); + odp_time_t cur = time_cur(); + odp_time_t wait = time_from_ns(ns); odp_time_t end_time = time_sum(cur, wait); time_wait_until(end_time); @@ -176,34 +277,31 @@ void odp_time_wait_until(odp_time_t time) return time_wait_until(time); } -uint64_t odp_time_to_u64(odp_time_t time) +int odp_time_init_global(void) { - int ret; - struct timespec tres; - uint64_t resolution; + int ret = 0; - ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres); - if (odp_unlikely(ret != 0)) - ODP_ABORT("clock_getres failed\n"); + memset(&global, 0, sizeof(time_global_t)); - resolution = (uint64_t)tres.tv_nsec; + if (cpu_has_global_time()) { + global.use_hw = 1; + global.hw_freq_hz = cpu_global_time_freq(); - return time_to_ns(time) / resolution; -} + if (global.hw_freq_hz == 0) + return -1; -int odp_time_init_global(void) -{ - int ret; - struct timespec time; - - ret = clock_gettime(CLOCK_MONOTONIC_RAW, &time); - if (ret) { - start_time = ODP_TIME_NULL; - } else { - start_time.tv_sec = time.tv_sec; - start_time.tv_nsec = time.tv_nsec; + printf("HW time counter freq: %" PRIu64 " hz\n\n", + global.hw_freq_hz); + + global.hw_start = cpu_global_time(); + return 0; } + global.spec_start.tv_sec = 0; + global.spec_start.tv_nsec = 0; + + ret = clock_gettime(CLOCK_MONOTONIC_RAW, &global.spec_start); + return ret; } diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index 4e9358b96..a93b3ba90 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -3238,7 +3238,7 @@ static void tm_sched_params_cvt_to(odp_tm_sched_params_t *odp_sched_params, if (weight == 0) inv_weight = 0; else - inv_weight = 0x10000 / weight; + inv_weight = 0xFFFF / weight; tm_sched_params->sched_modes[priority] = sched_mode; tm_sched_params->inverted_weights[priority] = inv_weight; @@ -3254,7 +3254,7 @@ static void tm_sched_params_cvt_from(tm_sched_params_t *tm_sched_params, for (priority = 0; priority < ODP_TM_MAX_PRIORITIES; priority++) { sched_mode = tm_sched_params->sched_modes[priority]; inv_weight = tm_sched_params->inverted_weights[priority]; - weight = 0x10000 / inv_weight; + weight = 0xFFFF / inv_weight; odp_sched_params->sched_modes[priority] = sched_mode; odp_sched_params->sched_weights[priority] = weight; diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 1922109a7..c52cd09d1 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -34,49 +34,6 @@ static int disable_pktio; /** !0 this pktio disabled, 0 enabled */ /* Has dpdk_pktio_init() been called */ static odp_bool_t dpdk_initialized; -#define PMD_EXT(drv) \ -extern void devinitfn_##drv(void) - -PMD_EXT(aesni_gcm_pmd_drv); -PMD_EXT(cryptodev_aesni_mb_pmd_drv); -PMD_EXT(cryptodev_kasumi_pmd_drv); -PMD_EXT(cryptodev_null_pmd_drv); -PMD_EXT(cryptodev_snow3g_pmd_drv); -PMD_EXT(pmd_qat_drv); -PMD_EXT(pmd_af_packet_drv); -PMD_EXT(rte_bnx2x_driver); -PMD_EXT(rte_bnx2xvf_driver); -PMD_EXT(bnxt_pmd_drv); -PMD_EXT(bond_drv); -PMD_EXT(rte_cxgbe_driver); -PMD_EXT(em_pmd_drv); -PMD_EXT(pmd_igb_drv); -PMD_EXT(pmd_igbvf_drv); -PMD_EXT(ena_pmd_drv); -PMD_EXT(rte_enic_driver); -PMD_EXT(rte_fm10k_driver); -PMD_EXT(rte_i40e_driver); -PMD_EXT(rte_i40evf_driver); -PMD_EXT(rte_ixgbe_driver); -PMD_EXT(rte_ixgbevf_driver); -PMD_EXT(rte_mlx4_driver); -PMD_EXT(rte_mlx5_driver); -PMD_EXT(pmd_mpipe_xgbe_drv); -PMD_EXT(pmd_mpipe_gbe_drv); -PMD_EXT(rte_nfp_net_driver); -PMD_EXT(pmd_null_drv); -PMD_EXT(pmd_pcap_drv); -PMD_EXT(rte_qede_driver); -PMD_EXT(rte_qedevf_driver); -PMD_EXT(pmd_ring_drv); -PMD_EXT(pmd_szedata2_drv); -PMD_EXT(rte_nicvf_driver); -PMD_EXT(pmd_vhost_drv); -PMD_EXT(rte_virtio_driver); -PMD_EXT(virtio_user_driver); -PMD_EXT(rte_vmxnet3_driver); -PMD_EXT(pmd_xenvirt_drv); - #define MEMPOOL_OPS(hdl) \ extern void mp_hdlr_init_##hdl(void) @@ -89,116 +46,11 @@ MEMPOOL_OPS(ops_stack); /* * This function is not called from anywhere, it's only purpose is to make sure * that if ODP and DPDK are statically linked to an application, the GCC - * constructors of the PMDs are linked as well. Otherwise the linker would omit - * them. It's not an issue with dynamic linking. */ + * constructors of mempool handlers are linked as well. Otherwise the linker + * would omit them. It's not an issue with dynamic linking. */ void refer_constructors(void); void refer_constructors(void) { -#ifdef RTE_LIBRTE_PMD_AESNI_GCM - devinitfn_aesni_gcm_pmd_drv(); -#endif -#ifdef RTE_LIBRTE_PMD_AESNI_MB - devinitfn_cryptodev_aesni_mb_pmd_drv(); -#endif -#ifdef RTE_LIBRTE_PMD_KASUMI - devinitfn_cryptodev_kasumi_pmd_drv(); -#endif -#ifdef RTE_LIBRTE_PMD_NULL_CRYPTO - devinitfn_cryptodev_null_pmd_drv(); -#endif -#ifdef RTE_LIBRTE_PMD_SNOW3G - devinitfn_cryptodev_snow3g_pmd_drv(); -#endif -#ifdef RTE_LIBRTE_PMD_QAT - devinitfn_pmd_qat_drv(); -#endif -#ifdef RTE_LIBRTE_PMD_AF_PACKET - devinitfn_pmd_af_packet_drv(); -#endif -#ifdef RTE_LIBRTE_BNX2X_PMD - devinitfn_rte_bnx2x_driver(); - devinitfn_rte_bnx2xvf_driver(); -#endif -#ifdef RTE_LIBRTE_BNXT_PMD - devinitfn_bnxt_pmd_drv(); -#endif -#ifdef RTE_LIBRTE_PMD_BOND - devinitfn_bond_drv(); -#endif -#ifdef RTE_LIBRTE_CXGBE_PMD - devinitfn_rte_cxgbe_driver(); -#endif -#ifdef RTE_LIBRTE_EM_PMD - devinitfn_em_pmd_drv(); -#endif -#ifdef RTE_LIBRTE_IGB_PMD - devinitfn_pmd_igb_drv(); - devinitfn_pmd_igbvf_drv(); -#endif -#ifdef RTE_LIBRTE_ENA_PMD - devinitfn_ena_pmd_drv(); -#endif -#ifdef RTE_LIBRTE_ENIC_PMD - devinitfn_rte_enic_driver(); -#endif -#ifdef RTE_LIBRTE_FM10K_PMD - devinitfn_rte_fm10k_driver(); -#endif -#ifdef RTE_LIBRTE_I40E_PMD - devinitfn_rte_i40e_driver(); - devinitfn_rte_i40evf_driver(); -#endif -#ifdef RTE_LIBRTE_IXGBE_PMD - devinitfn_rte_ixgbe_driver(); - devinitfn_rte_ixgbevf_driver(); -#endif -#ifdef RTE_LIBRTE_MLX4_PMD - devinitfn_rte_mlx4_driver(); -#endif -#ifdef RTE_LIBRTE_MLX5_PMD - devinitfn_rte_mlx5_driver(); -#endif -#ifdef RTE_LIBRTE_MPIPE_PMD - devinitfn_pmd_mpipe_xgbe_drv() - devinitfn_pmd_mpipe_gbe_drv() -#endif -#ifdef RTE_LIBRTE_NFP_PMD - devinitfn_rte_nfp_net_driver(); -#endif -#ifdef RTE_LIBRTE_PMD_NULL - devinitfn_pmd_null_drv(); -#endif -#ifdef RTE_LIBRTE_PMD_PCAP - devinitfn_pmd_pcap_drv(); -#endif -#ifdef RTE_LIBRTE_QEDE_PMD - devinitfn_rte_qede_driver(); - devinitfn_rte_qedevf_driver(); -#endif -#ifdef RTE_LIBRTE_PMD_RING - devinitfn_pmd_ring_drv(); -#endif -#ifdef RTE_LIBRTE_PMD_SZEDATA2 - devinitfn_pmd_szedata2_drv(); -#endif -#ifdef RTE_LIBRTE_THUNDERX_NICVF_PMD - devinitfn_rte_nicvf_driver(); -#endif -#ifdef RTE_LIBRTE_PMD_VHOST - devinitfn_pmd_vhost_drv(); -#endif -#ifdef RTE_LIBRTE_VIRTIO_PMD - devinitfn_rte_virtio_driver(); -#endif -#ifdef RTE_VIRTIO_USER - devinitfn_rte_virtio_driver(); -#endif -#ifdef RTE_LIBRTE_VMXNET3_PMD - devinitfn_rte_vmxnet3_driver(); -#endif -#ifdef RTE_LIBRTE_PMD_XENVIRT - devinitfn_pmd_xenvirt_drv(); -#endif mp_hdlr_init_ops_mp_mc(); mp_hdlr_init_ops_sp_sc(); mp_hdlr_init_ops_mp_sc(); @@ -801,7 +653,8 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, if (pktio_cls_enabled(pktio_entry)) copy_packet_cls_metadata(&parsed_hdr, pkt_hdr); else - packet_parse_l2(&pkt_hdr->p, pkt_len); + packet_parse_layer(pkt_hdr, + pktio_entry->s.config.parser.layer); if (mbuf->ol_flags & PKT_RX_RSS_HASH) odp_packet_flow_hash_set(pkt, mbuf->hash.rss); diff --git a/platform/linux-generic/pktio/ethtool.c b/platform/linux-generic/pktio/ethtool.c index 1b0f25b2c..d8f9e12cb 100644 --- a/platform/linux-generic/pktio/ethtool.c +++ b/platform/linux-generic/pktio/ethtool.c @@ -158,6 +158,7 @@ int ethtool_stats_get_fd(int fd, const char *name, odp_pktio_stats_t *stats) { struct ifreq ifr; + memset(&ifr, 0, sizeof(ifr)); snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", name); return ethtool_stats(fd, &ifr, stats); diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c index 70962839f..e9ad22ba2 100644 --- a/platform/linux-generic/pktio/loop.c +++ b/platform/linux-generic/pktio/loop.c @@ -132,7 +132,8 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, if (pktio_cls_enabled(pktio_entry)) copy_packet_cls_metadata(&parsed_hdr, pkt_hdr); else - packet_parse_l2(&pkt_hdr->p, pkt_len); + packet_parse_layer(pkt_hdr, + pktio_entry->s.config.parser.layer); packet_set_ts(pkt_hdr, ts); @@ -176,7 +177,7 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pktio_entry->s.stats.out_octets += bytes; } else { ODP_DBG("queue enqueue failed %i\n", ret); - return -1; + ret = -1; } odp_ticketlock_unlock(&pktio_entry->s.txl); diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c index ae3db34d9..928bb00af 100644 --- a/platform/linux-generic/pktio/netmap.c +++ b/platform/linux-generic/pktio/netmap.c @@ -663,7 +663,8 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, if (pktio_cls_enabled(pktio_entry)) copy_packet_cls_metadata(&parsed_hdr, pkt_hdr); else - packet_parse_l2(&pkt_hdr->p, len); + packet_parse_layer(pkt_hdr, + pktio_entry->s.config.parser.layer); packet_set_ts(pkt_hdr, ts); } diff --git a/platform/linux-generic/pktio/pcap.c b/platform/linux-generic/pktio/pcap.c index e54a56f5f..a467b6402 100644 --- a/platform/linux-generic/pktio/pcap.c +++ b/platform/linux-generic/pktio/pcap.c @@ -252,7 +252,8 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, break; } - packet_parse_l2(&pkt_hdr->p, pkt_len); + packet_parse_layer(pkt_hdr, + pktio_entry->s.config.parser.layer); pktio_entry->s.stats.in_octets += pkt_hdr->frame_len; packet_set_ts(pkt_hdr, ts); diff --git a/platform/linux-generic/pktio/ring.c b/platform/linux-generic/pktio/ring.c index aeda04b26..e3c73d1cb 100644 --- a/platform/linux-generic/pktio/ring.c +++ b/platform/linux-generic/pktio/ring.c @@ -263,8 +263,8 @@ int ___ring_mp_do_enqueue(_ring_t *r, void * const *obj_table, /* Reset n to the initial burst count */ n = max; - prod_head = r->prod.head; - cons_tail = r->cons.tail; + prod_head = __atomic_load_n(&r->prod.head, __ATOMIC_RELAXED); + cons_tail = __atomic_load_n(&r->cons.tail, __ATOMIC_ACQUIRE); /* The subtraction is done between two unsigned 32bits value * (the result is always modulo 32 bits even if we have * prod_head > cons_tail). So 'free_entries' is always between 0 @@ -306,12 +306,12 @@ int ___ring_mp_do_enqueue(_ring_t *r, void * const *obj_table, * If there are other enqueues in progress that preceded us, * we need to wait for them to complete */ - while (odp_unlikely(r->prod.tail != prod_head)) + while (odp_unlikely(__atomic_load_n(&r->prod.tail, __ATOMIC_RELAXED) != + prod_head)) odp_cpu_pause(); /* Release our entries and the memory they refer to */ - __atomic_thread_fence(__ATOMIC_RELEASE); - r->prod.tail = prod_next; + __atomic_store_n(&r->prod.tail, prod_next, __ATOMIC_RELEASE); return ret; } @@ -328,7 +328,7 @@ int ___ring_sp_do_enqueue(_ring_t *r, void * const *obj_table, int ret; prod_head = r->prod.head; - cons_tail = r->cons.tail; + cons_tail = __atomic_load_n(&r->cons.tail, __ATOMIC_ACQUIRE); /* The subtraction is done between two unsigned 32bits value * (the result is always modulo 32 bits even if we have * prod_head > cons_tail). So 'free_entries' is always between 0 @@ -361,8 +361,7 @@ int ___ring_sp_do_enqueue(_ring_t *r, void * const *obj_table, } /* Release our entries and the memory they refer to */ - __atomic_thread_fence(__ATOMIC_RELEASE); - r->prod.tail = prod_next; + __atomic_store_n(&r->prod.tail, prod_next, __ATOMIC_RELEASE); return ret; } @@ -385,8 +384,8 @@ int ___ring_mc_do_dequeue(_ring_t *r, void **obj_table, /* Restore n as it may change every loop */ n = max; - cons_head = r->cons.head; - prod_tail = r->prod.tail; + cons_head = __atomic_load_n(&r->cons.head, __ATOMIC_RELAXED); + prod_tail = __atomic_load_n(&r->prod.tail, __ATOMIC_ACQUIRE); /* The subtraction is done between two unsigned 32bits value * (the result is always modulo 32 bits even if we have * cons_head > prod_tail). So 'entries' is always between 0 @@ -419,12 +418,12 @@ int ___ring_mc_do_dequeue(_ring_t *r, void **obj_table, * If there are other dequeues in progress that preceded us, * we need to wait for them to complete */ - while (odp_unlikely(r->cons.tail != cons_head)) + while (odp_unlikely(__atomic_load_n(&r->cons.tail, __ATOMIC_RELAXED) != + cons_head)) odp_cpu_pause(); /* Release our entries and the memory they refer to */ - __atomic_thread_fence(__ATOMIC_RELEASE); - r->cons.tail = cons_next; + __atomic_store_n(&r->cons.tail, cons_next, __ATOMIC_RELEASE); return behavior == _RING_QUEUE_FIXED ? 0 : n; } @@ -441,7 +440,7 @@ int ___ring_sc_do_dequeue(_ring_t *r, void **obj_table, uint32_t mask = r->prod.mask; cons_head = r->cons.head; - prod_tail = r->prod.tail; + prod_tail = __atomic_load_n(&r->prod.tail, __ATOMIC_ACQUIRE); /* The subtraction is done between two unsigned 32bits value * (the result is always modulo 32 bits even if we have * cons_head > prod_tail). So 'entries' is always between 0 @@ -461,11 +460,10 @@ int ___ring_sc_do_dequeue(_ring_t *r, void **obj_table, r->cons.head = cons_next; /* Acquire the pointers and the memory they refer to */ - __atomic_thread_fence(__ATOMIC_ACQUIRE); /* copy in table */ DEQUEUE_PTRS(); - r->cons.tail = cons_next; + __atomic_store_n(&r->cons.tail, cons_next, __ATOMIC_RELEASE); return behavior == _RING_QUEUE_FIXED ? 0 : n; } diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c index 7d2396866..a383adc6a 100644 --- a/platform/linux-generic/pktio/socket.c +++ b/platform/linux-generic/pktio/socket.c @@ -234,6 +234,7 @@ static inline int get_rss_hash_options(int fd, const char *name, struct ifreq ifr; struct ethtool_rxnfc rsscmd; + memset(&ifr, 0, sizeof(ifr)); memset(&rsscmd, 0, sizeof(rsscmd)); *options = 0; @@ -461,8 +462,6 @@ static int sock_close(pktio_entry_t *pktio_entry) return -1; } - odp_shm_free(pkt_sock->shm); - return 0; } @@ -474,13 +473,11 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, { int sockfd; int err; - int i; unsigned int if_idx; struct ifreq ethreq; struct sockaddr_ll sa_ll; char shm_name[ODP_SHM_NAME_LEN]; pkt_sock_t *pkt_sock = &pktio_entry->s.pkt_sock; - uint8_t *addr; odp_pktio_stats_t cur_stats; /* Init pktio entry */ @@ -494,18 +491,6 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, snprintf(shm_name, ODP_SHM_NAME_LEN, "%s-%s", "pktio", netdev); shm_name[ODP_SHM_NAME_LEN - 1] = '\0'; - pkt_sock->shm = odp_shm_reserve(shm_name, PACKET_JUMBO_LEN, - PACKET_JUMBO_LEN * - ODP_PACKET_SOCKET_MAX_BURST_RX, 0); - if (pkt_sock->shm == ODP_SHM_INVALID) - return -1; - - addr = odp_shm_addr(pkt_sock->shm); - for (i = 0; i < ODP_PACKET_SOCKET_MAX_BURST_RX; i++) { - pkt_sock->cache_ptr[i] = addr; - addr += PACKET_JUMBO_LEN; - } - sockfd = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL)); if (sockfd == -1) { __odp_errno = errno; @@ -617,19 +602,17 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_packet_t pkt_table[], int len) { pkt_sock_t *pkt_sock = &pktio_entry->s.pkt_sock; + odp_pool_t pool = pkt_sock->pool; odp_time_t ts_val; odp_time_t *ts = NULL; const int sockfd = pkt_sock->sockfd; - int msgvec_len; - struct mmsghdr msgvec[ODP_PACKET_SOCKET_MAX_BURST_RX]; + struct mmsghdr msgvec[len]; + struct iovec iovecs[len][MAX_SEGS]; int nb_rx = 0; + int nb_pkts; int recv_msgs; - uint8_t **recv_cache; int i; - if (odp_unlikely(len > ODP_PACKET_SOCKET_MAX_BURST_RX)) - return -1; - odp_ticketlock_lock(&pktio_entry->s.rxl); if (pktio_entry->s.config.pktin.bit.ts_all || @@ -637,131 +620,72 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, ts = &ts_val; memset(msgvec, 0, sizeof(msgvec)); - recv_cache = pkt_sock->cache_ptr; - if (pktio_cls_enabled(pktio_entry)) { - struct iovec iovecs[ODP_PACKET_SOCKET_MAX_BURST_RX]; + nb_pkts = packet_alloc_multi(pool, pkt_sock->mtu, pkt_table, len); + for (i = 0; i < nb_pkts; i++) { + msgvec[i].msg_hdr.msg_iovlen = + _rx_pkt_to_iovec(pkt_table[i], iovecs[i]); + msgvec[i].msg_hdr.msg_iov = iovecs[i]; + } + + recv_msgs = recvmmsg(sockfd, msgvec, nb_pkts, MSG_DONTWAIT, NULL); - for (i = 0; i < (int)len; i++) { - msgvec[i].msg_hdr.msg_iovlen = 1; - iovecs[i].iov_base = recv_cache[i]; - iovecs[i].iov_len = PACKET_JUMBO_LEN; - msgvec[i].msg_hdr.msg_iov = &iovecs[i]; - } - msgvec_len = i; - - recv_msgs = recvmmsg(sockfd, msgvec, msgvec_len, - MSG_DONTWAIT, NULL); - - if (ts != NULL) - ts_val = odp_time_global(); - - for (i = 0; i < recv_msgs; i++) { - odp_packet_hdr_t *pkt_hdr; - odp_packet_t pkt; - odp_pool_t pool = pkt_sock->pool; - odp_packet_hdr_t parsed_hdr; - void *base = msgvec[i].msg_hdr.msg_iov->iov_base; - struct ethhdr *eth_hdr = base; - uint16_t pkt_len = msgvec[i].msg_len; - int num; - - /* Don't receive packets sent by ourselves */ - if (odp_unlikely(ethaddrs_equal(pkt_sock->if_mac, - eth_hdr->h_source))) - continue; + if (ts != NULL) + ts_val = odp_time_global(); - if (cls_classify_packet(pktio_entry, base, pkt_len, - pkt_len, &pool, &parsed_hdr)) - continue; + for (i = 0; i < recv_msgs; i++) { + void *base = msgvec[i].msg_hdr.msg_iov->iov_base; + struct ethhdr *eth_hdr = base; + odp_packet_t pkt = pkt_table[i]; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + uint16_t pkt_len = msgvec[i].msg_len; + int ret; - num = packet_alloc_multi(pool, pkt_len, &pkt, 1); - if (num != 1) - continue; + if (pktio_cls_enabled(pktio_entry)) { + uint16_t seg_len = pkt_len; - pkt_hdr = odp_packet_hdr(pkt); + if (msgvec[i].msg_hdr.msg_iov->iov_len < pkt_len) + seg_len = msgvec[i].msg_hdr.msg_iov->iov_len; - if (odp_packet_copy_from_mem(pkt, 0, pkt_len, - base) != 0) { + if (cls_classify_packet(pktio_entry, base, pkt_len, + seg_len, &pool, pkt_hdr)) { + ODP_ERR("cls_classify_packet failed"); odp_packet_free(pkt); continue; } - pkt_hdr->input = pktio_entry->s.handle; - copy_packet_cls_metadata(&parsed_hdr, pkt_hdr); - packet_set_ts(pkt_hdr, ts); - - pkt_table[nb_rx++] = pkt; } - } else { - struct iovec iovecs[ODP_PACKET_SOCKET_MAX_BURST_RX] - [MAX_SEGS]; - - for (i = 0; i < (int)len; i++) { - int num; - num = packet_alloc_multi(pkt_sock->pool, pkt_sock->mtu, - &pkt_table[i], 1); - - if (odp_unlikely(num != 1)) { - pkt_table[i] = ODP_PACKET_INVALID; - break; - } - - msgvec[i].msg_hdr.msg_iovlen = - _rx_pkt_to_iovec(pkt_table[i], iovecs[i]); - - msgvec[i].msg_hdr.msg_iov = iovecs[i]; + /* Don't receive packets sent by ourselves */ + if (odp_unlikely(ethaddrs_equal(pkt_sock->if_mac, + eth_hdr->h_source))) { + odp_packet_free(pkt); + continue; } - /* number of successfully allocated pkt buffers */ - msgvec_len = i; - - recv_msgs = recvmmsg(sockfd, msgvec, msgvec_len, - MSG_DONTWAIT, NULL); - - if (ts != NULL) - ts_val = odp_time_global(); - - for (i = 0; i < recv_msgs; i++) { - void *base = msgvec[i].msg_hdr.msg_iov->iov_base; - struct ethhdr *eth_hdr = base; - odp_packet_hdr_t *pkt_hdr; - odp_packet_t pkt; - int ret; - - pkt = pkt_table[i]; - - /* Don't receive packets sent by ourselves */ - if (odp_unlikely(ethaddrs_equal(pkt_sock->if_mac, - eth_hdr->h_source))) { - odp_packet_free(pkt); - continue; - } + ret = odp_packet_trunc_tail(&pkt, odp_packet_len(pkt) - pkt_len, + NULL, NULL); + if (ret < 0) { + ODP_ERR("trunk_tail failed"); + odp_packet_free(pkt); + continue; + } - /* Parse and set packet header data */ - ret = odp_packet_trunc_tail(&pkt, odp_packet_len(pkt) - - msgvec[i].msg_len, - NULL, NULL); - if (ret < 0) { - ODP_ERR("trunk_tail failed"); - odp_packet_free(pkt); - continue; - } + pkt_hdr->input = pktio_entry->s.handle; - pkt_hdr = odp_packet_hdr(pkt); - packet_parse_l2(&pkt_hdr->p, pkt_hdr->frame_len); - packet_set_ts(pkt_hdr, ts); - pkt_hdr->input = pktio_entry->s.handle; + if (!pktio_cls_enabled(pktio_entry)) + packet_parse_layer(pkt_hdr, + pktio_entry->s.config.parser.layer); - pkt_table[nb_rx] = pkt; - nb_rx++; - } + pkt_hdr->input = pktio_entry->s.handle; + packet_set_ts(pkt_hdr, ts); - /* Free unused pkt buffers */ - for (; i < msgvec_len; i++) - odp_packet_free(pkt_table[i]); + pkt_table[nb_rx++] = pkt; } + /* Free unused pkt buffers */ + for (; i < nb_pkts; i++) + odp_packet_free(pkt_table[i]); + odp_ticketlock_unlock(&pktio_entry->s.rxl); return nb_rx; @@ -793,15 +717,12 @@ static int sock_mmsg_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, const odp_packet_t pkt_table[], int len) { pkt_sock_t *pkt_sock = &pktio_entry->s.pkt_sock; - struct mmsghdr msgvec[ODP_PACKET_SOCKET_MAX_BURST_TX]; - struct iovec iovecs[ODP_PACKET_SOCKET_MAX_BURST_TX][MAX_SEGS]; + struct mmsghdr msgvec[len]; + struct iovec iovecs[len][MAX_SEGS]; int ret; int sockfd; int n, i; - if (odp_unlikely(len > ODP_PACKET_SOCKET_MAX_BURST_TX)) - return -1; - odp_ticketlock_lock(&pktio_entry->s.txl); sockfd = pkt_sock->sockfd; diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index 666aae6af..2dba7b08f 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -231,7 +231,8 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, if (pktio_cls_enabled(pktio_entry)) copy_packet_cls_metadata(&parsed_hdr, hdr); else - packet_parse_l2(&hdr->p, pkt_len); + packet_parse_layer(hdr, + pktio_entry->s.config.parser.layer); packet_set_ts(hdr, ts); @@ -454,11 +455,11 @@ static int mmap_sock(pkt_sock_mmap_t *pkt_sock) return 0; } -static void mmap_unmap_sock(pkt_sock_mmap_t *pkt_sock) +static int mmap_unmap_sock(pkt_sock_mmap_t *pkt_sock) { - munmap(pkt_sock->mmap_base, pkt_sock->mmap_len); free(pkt_sock->rx_ring.rd); free(pkt_sock->tx_ring.rd); + return munmap(pkt_sock->mmap_base, pkt_sock->mmap_len); } static int mmap_bind_sock(pkt_sock_mmap_t *pkt_sock, const char *netdev) @@ -486,8 +487,14 @@ static int mmap_bind_sock(pkt_sock_mmap_t *pkt_sock, const char *netdev) static int sock_mmap_close(pktio_entry_t *entry) { pkt_sock_mmap_t *const pkt_sock = &entry->s.pkt_sock_mmap; + int ret; + + ret = mmap_unmap_sock(pkt_sock); + if (ret != 0) { + ODP_ERR("mmap_unmap_sock() %s\n", strerror(errno)); + return -1; + } - mmap_unmap_sock(pkt_sock); if (pkt_sock->sockfd != -1 && close(pkt_sock->sockfd) != 0) { __odp_errno = errno; ODP_ERR("close(sockfd): %s\n", strerror(errno)); diff --git a/platform/linux-generic/pktio/tap.c b/platform/linux-generic/pktio/tap.c index ac2045606..650c12a77 100644 --- a/platform/linux-generic/pktio/tap.c +++ b/platform/linux-generic/pktio/tap.c @@ -213,7 +213,8 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data, if (pktio_cls_enabled(pktio_entry)) copy_packet_cls_metadata(&parsed_hdr, pkt_hdr); else - packet_parse_l2(&pkt_hdr->p, len); + packet_parse_layer(pkt_hdr, + pktio_entry->s.config.parser.layer); packet_set_ts(pkt_hdr, ts); pkt_hdr->input = pktio_entry->s.handle; diff --git a/scripts/build-pktio-dpdk b/scripts/build-pktio-dpdk index 36727dd7b..6c6830ac0 100755 --- a/scripts/build-pktio-dpdk +++ b/scripts/build-pktio-dpdk @@ -10,7 +10,7 @@ if [ "$?" != "0" ]; then echo "Error: pcap is not installed. You may need to install libpcap-dev" fi -git -c advice.detachedHead=false clone -q --depth=1 --single-branch --branch=v16.07 http://dpdk.org/git/dpdk dpdk +git -c advice.detachedHead=false clone -q --depth=1 --single-branch --branch=v17.02 http://dpdk.org/git/dpdk dpdk pushd dpdk git log --oneline --decorate diff --git a/scripts/ci-checkpatches.sh b/scripts/ci-checkpatches.sh new file mode 100755 index 000000000..cb1c4e65d --- /dev/null +++ b/scripts/ci-checkpatches.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +PATCHES=$1 +echo "Run checkpatch for ${PATCHES}" +# Generate patches provided with $1. +# In case of force push and range is broken +# validate only the latest commit if it's not merge commit. +git format-patch ${PATCHES} +if [ $? -ne 0 ]; then + git show --summary HEAD| grep -q '^Merge:'; + if [ $? -ne 0 ]; then + git format-patch HEAD^; + perl ./scripts/checkpatch.pl *.patch; + fi; +else + perl ./scripts/checkpatch.pl *.patch; +fi diff --git a/scripts/spelling.txt b/scripts/spelling.txt index bb8e4d0a1..eb38f49d4 100644 --- a/scripts/spelling.txt +++ b/scripts/spelling.txt @@ -16,6 +16,7 @@ absense||absence absolut||absolute absoulte||absolute acccess||access +acceess||access acceleratoin||acceleration accelleration||acceleration accesing||accessing @@ -32,19 +33,23 @@ accoring||according accout||account accquire||acquire accquired||acquired +accross||across acessable||accessible acess||access achitecture||architecture acient||ancient acitions||actions acitve||active -acknowldegement||acknowldegement +acknowldegement||acknowledgment acknowledgement||acknowledgment ackowledge||acknowledge ackowledged||acknowledged acording||according activete||activate +actived||activated +actualy||actually acumulating||accumulating +acumulator||accumulator adapater||adapter addional||additional additionaly||additionally @@ -59,16 +64,22 @@ adress||address adresses||addresses adviced||advised afecting||affecting +againt||against agaist||against albumns||albums alegorical||allegorical +algined||aligned algorith||algorithm algorithmical||algorithmically algoritm||algorithm algoritms||algorithms algorrithm||algorithm algorritm||algorithm +aligment||alignment +alignement||alignment allign||align +alligned||aligned +allocatote||allocate allocatrd||allocated allocte||allocate allpication||application @@ -83,6 +94,10 @@ alue||value ambigious||ambiguous amoung||among amout||amount +an union||a union +an user||a user +an userspace||a userspace +an one||a one analysator||analyzer ang||and anniversery||anniversary @@ -95,13 +110,16 @@ appearence||appearance applicaion||application appliction||application applictions||applications +applys||applies appplications||applications appropiate||appropriate appropriatly||appropriately approriate||appropriate approriately||appropriately +apropriate||appropriate aquainted||acquainted aquired||acquired +aquisition||acquisition arbitary||arbitrary architechture||architecture arguement||argument @@ -111,6 +129,8 @@ arne't||aren't arraival||arrival artifical||artificial artillary||artillery +asign||assign +assertation||assertion assiged||assigned assigment||assignment assigments||assignments @@ -125,6 +145,7 @@ asycronous||asynchronous asynchnous||asynchronous atomatically||automatically atomicly||atomically +atempt||attempt attachement||attachment attched||attached attemps||attempts @@ -136,6 +157,7 @@ automatize||automate automatized||automated automatizes||automates autonymous||autonomous +auxillary||auxiliary auxilliary||auxiliary avaiable||available avaible||available @@ -157,6 +179,7 @@ bakup||backup baloon||balloon baloons||balloons bandwith||bandwidth +banlance||balance batery||battery beacuse||because becasue||because @@ -178,6 +201,7 @@ cacluated||calculated caculation||calculation calender||calendar calle||called +callibration||calibration calucate||calculate calulate||calculate cancelation||cancellation @@ -187,6 +211,7 @@ capatibilities||capabilities carefuly||carefully cariage||carriage catagory||category +cehck||check challange||challenge challanges||challenges chanell||channel @@ -199,6 +224,8 @@ charactor||character charater||character charaters||characters charcter||character +chcek||check +chck||check checksuming||checksumming childern||children childs||children @@ -225,14 +252,20 @@ commited||committed commiting||committing committ||commit commoditiy||commodity +comsume||consume +comsumer||consumer +comsuming||consuming compability||compatibility compaibility||compatibility compatability||compatibility compatable||compatible compatibiliy||compatibility compatibilty||compatibility +compatiblity||compatibility +competion||completion compilant||compliant compleatly||completely +completition||completion completly||completely complient||compliant componnents||components @@ -243,9 +276,12 @@ comunication||communication conbination||combination conditionaly||conditionally conected||connected +connecetd||connected +configuartion||configuration configuratoin||configuration configuraton||configuration configuretion||configuration +configutation||configuration conider||consider conjuction||conjunction connectinos||connections @@ -262,11 +298,14 @@ continous||continuous continously||continuously continueing||continuing contraints||constraints +contol||control +contoller||controller controled||controlled controler||controller controll||control contruction||construction contry||country +conuntry||country convertion||conversion convertor||converter convienient||convenient @@ -281,6 +320,7 @@ coutner||counter cryptocraphic||cryptographic cunter||counter curently||currently +cylic||cyclic dafault||default deafult||default deamon||daemon @@ -291,8 +331,12 @@ defferred||deferred definate||definite definately||definitely defintion||definition +defintions||definitions defualt||default defult||default +deintializing||deinitializing +deintialize||deinitialize +deintialized||deinitialized deivce||device delared||declared delare||declare @@ -305,7 +349,9 @@ dependant||dependent depreacted||deprecated depreacte||deprecate desactivate||deactivate +desciptor||descriptor desciptors||descriptors +descripton||description descrition||description descritptor||descriptor desctiptor||descriptor @@ -327,14 +373,20 @@ devided||divided deviece||device diable||disable dictionnary||dictionary +didnt||didn't diferent||different differrence||difference +diffrent||different +diffrentiate||differentiate difinition||definition diplay||display direectly||directly +disassocation||disassociation disapear||disappear disapeared||disappeared disappared||disappeared +disble||disable +disbled||disabled disconnet||disconnect discontinous||discontinuous dispertion||dispersion @@ -344,6 +396,7 @@ docuentation||documentation documantation||documentation documentaion||documentation documment||document +doesnt||doesn't dorp||drop dosen||doesn downlad||download @@ -354,10 +407,13 @@ easilly||easily ecspecially||especially edditable||editable editting||editing +efective||effective efficently||efficiently ehther||ether eigth||eight +elementry||elementary eletronic||electronic +embeded||embedded enabledi||enabled enchanced||enhanced encorporating||incorporating @@ -393,6 +449,7 @@ expecially||especially explicite||explicit explicitely||explicitly explict||explicit +explictely||explicitly explictly||explicitly expresion||expression exprimental||experimental @@ -400,11 +457,15 @@ extened||extended extensability||extensibility extention||extension extracter||extractor +falied||failed faild||failed faill||fail +failied||failed +faillure||failure failue||failure failuer||failure faireness||fairness +falied||failed faliure||failure familar||familiar fatser||faster @@ -413,6 +474,7 @@ feautures||features fetaure||feature fetaures||features fileystem||filesystem +fimware||firmware finanize||finalize findn||find finilizes||finalizes @@ -420,11 +482,13 @@ finsih||finish flusing||flushing folloing||following followign||following +followings||following follwing||following forseeable||foreseeable forse||force fortan||fortran forwardig||forwarding +framming||framing framwork||framework frequncy||frequency frome||from @@ -443,6 +507,7 @@ futhermore||furthermore futrue||future gaurenteed||guaranteed generiously||generously +genereate||generate genric||generic globel||global grabing||grabbing @@ -450,11 +515,13 @@ grahical||graphical grahpical||graphical grapic||graphic guage||gauge +guarenteed||guaranteed guarentee||guarantee halfs||halves hander||handler handfull||handful hanled||handled +happend||happened harware||hardware heirarchically||hierarchically helpfull||helpful @@ -462,8 +529,11 @@ hierachy||hierarchy hierarchie||hierarchy howver||however hsould||should +hypervior||hypervisor hypter||hyper identidier||identifier +iligal||illegal +illigal||illegal imblance||imbalance immeadiately||immediately immedaite||immediate @@ -502,17 +572,21 @@ informtion||information infromation||information ingore||ignore inital||initial +initalized||initialized initalised||initialized initalise||initialize initalize||initialize initation||initiation initators||initiators +initialiazation||initialization initializiation||initialization initialzed||initialized initilization||initialization initilize||initialize inofficial||unofficial +insititute||institute instal||install +instanciated||instantiated inteface||interface integreated||integrated integrety||integrity @@ -534,19 +608,28 @@ interruptted||interrupted interupted||interrupted interupt||interrupt intial||initial +intialisation||initialisation +intialised||initialised +intialise||initialise +intialization||initialization intialized||initialized intialize||initialize intregral||integral intrrupt||interrupt +intterrupt||interrupt intuative||intuitive invaid||invalid -invalde||invald +invalde||invalid invalide||invalid +invalud||invalid invididual||individual invokation||invocation invokations||invocations irrelevent||irrelevant +isnt||isn't isssue||issue +iternations||iterations +itertation||iteration itslef||itself jave||java jeffies||jiffies @@ -558,6 +641,7 @@ langauage||language langauge||language langugage||language lauch||launch +layed||laid leightweight||lightweight lengh||length lenght||length @@ -600,16 +684,20 @@ messsage||message messsages||messages microprocesspr||microprocessor milliseonds||milliseconds +minium||minimum +minimam||minimum minumum||minimum +misalinged||misaligned miscelleneous||miscellaneous misformed||malformed mispelled||misspelled mispelt||misspelt +mising||missing miximum||maximum mmnemonic||mnemonic mnay||many -modeled||modelled modulues||modules +momery||memory monochorome||monochrome monochromo||monochrome monocrome||monochrome @@ -629,6 +717,7 @@ neccecary||necessary neccesary||necessary neccessary||necessary necesary||necessary +neded||needed negaive||negative negoitation||negotiation negotation||negotiation @@ -648,8 +737,11 @@ occurances||occurrences occured||occurred occurence||occurrence occure||occurred +occured||occurred occuring||occurring offet||offset +omited||omitted +omiting||omitting omitt||omit ommiting||omitting ommitted||omitted @@ -661,13 +753,19 @@ optionnal||optional optmizations||optimizations orientatied||orientated orientied||oriented +orignal||original otherise||otherwise ouput||output +oustanding||outstanding overaall||overall overhread||overhead overlaping||overlapping +overide||override +overrided||overridden overriden||overridden overun||overrun +overwritting||overwriting +overwriten||overwritten pacakge||package pachage||package packacge||package @@ -678,6 +776,7 @@ pakage||package pallette||palette paln||plan paramameters||parameters +paramaters||parameters paramater||parameter parametes||parameters parametised||parametrised @@ -685,6 +784,7 @@ paramter||parameter paramters||parameters particuarly||particularly particularily||particularly +partiton||partition pased||passed passin||passing pathes||paths @@ -704,6 +804,7 @@ pleaes||please ploting||plotting plugable||pluggable poinnter||pointer +pointeur||pointer poiter||pointer posible||possible positon||position @@ -714,6 +815,7 @@ preceeding||preceding preceed||precede precendence||precedence precission||precision +preemptable||preemptible prefered||preferred prefferably||preferably premption||preemption @@ -731,6 +833,7 @@ procceed||proceed proccesors||processors procesed||processed proces||process +procesing||processing processessing||processing processess||processes processpr||processor @@ -744,6 +847,7 @@ programers||programmers programm||program programms||programs progresss||progress +promiscous||promiscuous promps||prompts pronnounced||pronounced prononciation||pronunciation @@ -758,6 +862,7 @@ protable||portable protcol||protocol protecion||protection protocoll||protocol +promixity||proximity psudo||pseudo psuedo||pseudo psychadelic||psychedelic @@ -779,6 +884,7 @@ recommanded||recommended recyle||recycle redircet||redirect redirectrion||redirection +reename||rename refcounf||refcount refence||reference refered||referred @@ -791,6 +897,7 @@ registerd||registered registeresd||registered registes||registers registraration||registration +regsiter||register regster||register regualar||regular reguator||regulator @@ -808,6 +915,7 @@ replys||replies reponse||response representaion||representation reqeust||request +requestied||requested requiere||require requirment||requirement requred||required @@ -817,6 +925,7 @@ reseting||resetting resizeable||resizable resouces||resources resoures||resources +responce||response ressizes||resizes ressource||resource ressources||resources @@ -869,6 +978,7 @@ setts||sets settting||setting shotdown||shutdown shoud||should +shouldnt||shouldn't shoule||should shrinked||shrunk siginificantly||significantly @@ -899,6 +1009,7 @@ spinlcok||spinlock spinock||spinlock splitted||split spreaded||spread +spurrious||spurious sructure||structure stablilization||stabilization staically||statically @@ -913,12 +1024,17 @@ straming||streaming struc||struct structres||structures stuct||struct +strucuture||structure +stucture||structure sturcture||structure subdirectoires||subdirectories suble||subtle +substract||subtract succesfully||successfully succesful||successful +successed||succeeded successfull||successful +successfuly||successfully sucessfully||successfully sucess||success superflous||superfluous @@ -926,6 +1042,7 @@ superseeded||superseded suplied||supplied suported||supported suport||support +supportet||supported suppored||supported supportin||supporting suppoted||supported @@ -934,13 +1051,22 @@ suppport||support supress||suppress surpresses||suppresses susbsystem||subsystem +suspeneded||suspended suspicously||suspiciously swaping||swapping switchs||switches +swith||switch +swithable||switchable +swithc||switch +swithced||switched +swithcing||switching +swithed||switched +swithing||switching symetric||symmetric synax||syntax synchonized||synchronized syncronize||synchronize +syncronized||synchronized syncronizing||synchronizing syncronus||synchronous syste||system @@ -952,6 +1078,7 @@ targetting||targeting teh||the temorary||temporary temproarily||temporarily +therfore||therefore thier||their threds||threads threshhold||threshold @@ -959,13 +1086,14 @@ throught||through thses||these tiggered||triggered tipically||typically +timout||timeout tmis||this torerable||tolerable tramsmitted||transmitted tramsmit||transmit tranfer||transfer transciever||transceiver -transferd||transferrd +transferd||transferred transfered||transferred transfering||transferring transision||transition @@ -979,21 +1107,33 @@ ture||true tyep||type udpate||update uesd||used +uncommited||uncommitted unconditionaly||unconditionally underun||underrun unecessary||unnecessary unexecpted||unexpected +unexepected||unexpected +unexpcted||unexpected unexpectd||unexpected unexpeted||unexpected +unexpexted||unexpected unfortunatelly||unfortunately unifiy||unify +unintialized||uninitialized +unkmown||unknown unknonw||unknown unknow||unknown unkown||unknown +unneded||unneeded unneedingly||unnecessarily +unnsupported||unsupported +unmached||unmatched +unregester||unregister unresgister||unregister +unrgesiter||unregister unsinged||unsigned unstabel||unstable +unsolicitied||unsolicited unsuccessfull||unsuccessful unsuported||unsupported untill||until @@ -1014,6 +1154,7 @@ vaid||valid vaild||valid valide||valid variantions||variations +varible||variable varient||variant vaule||value verbse||verbose @@ -1027,7 +1168,9 @@ virtiual||virtual visiters||visitors vitual||virtual wating||waiting +wether||whether whataver||whatever +whcih||which whenver||whenever wheter||whether whe||when diff --git a/test/common_plat/miscellaneous/odp_api_from_cpp.cpp b/test/common_plat/miscellaneous/odp_api_from_cpp.cpp index 2b3078642..4578ae4be 100644 --- a/test/common_plat/miscellaneous/odp_api_from_cpp.cpp +++ b/test/common_plat/miscellaneous/odp_api_from_cpp.cpp @@ -1,10 +1,9 @@ #include <cstdio> #include <odp_api.h> -#include <odp/helper/threads.h> +#include <odp/helper/odph_api.h> int main(int argc ODP_UNUSED, const char *argv[] ODP_UNUSED) { - printf("\tODP API version: %s\n", odp_version_api_str()); printf("\tODP implementation version: %s\n", odp_version_impl_str()); diff --git a/test/common_plat/performance/Makefile.am b/test/common_plat/performance/Makefile.am index 9111c0c2d..3299f03f8 100644 --- a/test/common_plat/performance/Makefile.am +++ b/test/common_plat/performance/Makefile.am @@ -51,3 +51,5 @@ dist_odp_scheduling_SOURCES = odp_scheduling.c dist_odp_pktio_perf_SOURCES = odp_pktio_perf.c EXTRA_DIST = $(TESTSCRIPTS) + +dist_check_SCRIPTS = udp64.pcap diff --git a/test/common_plat/performance/odp_bench_packet.c b/test/common_plat/performance/odp_bench_packet.c index 8a1333276..c4cd6139b 100644 --- a/test/common_plat/performance/odp_bench_packet.c +++ b/test/common_plat/performance/odp_bench_packet.c @@ -338,6 +338,21 @@ static void alloc_concat_packets(void) TEST_REPEAT_COUNT); } +static void alloc_ref_packets(void) +{ + int i; + odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; + odp_packet_t *ref_tbl = gbl_args->pkt2_tbl; + + allocate_test_packets(gbl_args->pkt.len, pkt_tbl, TEST_REPEAT_COUNT); + + for (i = 0; i < TEST_REPEAT_COUNT; i++) { + ref_tbl[i] = odp_packet_ref(pkt_tbl[i], TEST_MIN_PKT_SIZE / 2); + if (ref_tbl[i] == ODP_PACKET_INVALID) + LOG_ABORT("Allocating packet reference failed\n"); + } +} + static void alloc_packets_twice(void) { allocate_test_packets(gbl_args->pkt.len, gbl_args->pkt_tbl, @@ -1212,6 +1227,67 @@ static int bench_packet_ts_set(void) return i; } +static int bench_packet_ref_static(void) +{ + int i; + odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; + odp_packet_t *ref_tbl = gbl_args->pkt2_tbl; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + ref_tbl[i] = odp_packet_ref_static(pkt_tbl[i]); + + return i; +} + +static int bench_packet_ref(void) +{ + int i; + uint32_t offset = TEST_MIN_PKT_SIZE / 2; + odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; + odp_packet_t *ref_tbl = gbl_args->pkt2_tbl; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + ref_tbl[i] = odp_packet_ref(pkt_tbl[i], offset); + + return i; +} + +static int bench_packet_ref_pkt(void) +{ + int i; + uint32_t offset = TEST_MIN_PKT_SIZE / 2; + odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; + odp_packet_t *hdr_tbl = gbl_args->pkt2_tbl; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + hdr_tbl[i] = odp_packet_ref_pkt(pkt_tbl[i], offset, hdr_tbl[i]); + + return i; +} + +static int bench_packet_unshared_len(void) +{ + int i; + uint32_t ret = 0; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + ret += odp_packet_unshared_len(gbl_args->pkt_tbl[i]); + + return ret; +} + +static int bench_packet_has_ref(void) +{ + int i; + uint32_t ret = 0; + odp_packet_t *pkt_tbl = gbl_args->pkt_tbl; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + ret += odp_packet_has_ref(pkt_tbl[i]); + + return i; +} + /** * Prinf usage information */ @@ -1445,6 +1521,16 @@ bench_info_t test_suite[] = { BENCH_INFO(bench_packet_ts, create_packets, free_packets, NULL), BENCH_INFO(bench_packet_ts_set, create_packets, free_packets, NULL), + BENCH_INFO(bench_packet_ref_static, create_packets, + free_packets_twice, NULL), + BENCH_INFO(bench_packet_ref, create_packets, + free_packets_twice, NULL), + BENCH_INFO(bench_packet_ref_pkt, alloc_packets_twice, + free_packets_twice, NULL), + BENCH_INFO(bench_packet_unshared_len, alloc_ref_packets, + free_packets_twice, NULL), + BENCH_INFO(bench_packet_has_ref, alloc_ref_packets, + free_packets_twice, NULL), }; /** diff --git a/test/common_plat/performance/odp_crypto.c b/test/common_plat/performance/odp_crypto.c index 954bdb794..c3dd6d38f 100644 --- a/test/common_plat/performance/odp_crypto.c +++ b/test/common_plat/performance/odp_crypto.c @@ -50,7 +50,6 @@ static uint8_t test_key24[24] = { 0x01, 0x02, 0x03, 0x04, 0x05, typedef struct { const char *name; /**< Algorithm name */ odp_crypto_session_param_t session; /**< Prefilled crypto session params */ - unsigned int hash_adjust; /**< Size of hash */ } crypto_alg_config_t; /** @@ -205,25 +204,25 @@ static crypto_alg_config_t algs_config[] = { .data = test_iv, .length = 8, }, - .auth_alg = ODP_AUTH_ALG_MD5_96, + .auth_alg = ODP_AUTH_ALG_MD5_HMAC, .auth_key = { .data = test_key16, .length = sizeof(test_key16) - } + }, + .auth_digest_len = 12, }, - .hash_adjust = 12 }, { .name = "null-hmac-md5-96", .session = { .cipher_alg = ODP_CIPHER_ALG_NULL, - .auth_alg = ODP_AUTH_ALG_MD5_96, + .auth_alg = ODP_AUTH_ALG_MD5_HMAC, .auth_key = { .data = test_key16, .length = sizeof(test_key16) - } + }, + .auth_digest_len = 12, }, - .hash_adjust = 12 }, }; @@ -578,7 +577,7 @@ run_measure_one(crypto_args_t *cargs, mem = odp_packet_data(params.out_pkt); print_mem("Immediately encrypted packet", mem, payload_length + - config->hash_adjust); + config->session.auth_digest_len); } if (!cargs->in_place) { if (cargs->reuse_packet) { @@ -611,7 +610,8 @@ run_measure_one(crypto_args_t *cargs, print_mem("Receieved encrypted packet", mem, payload_length + - config->hash_adjust); + config-> + session.auth_digest_len); } if (cargs->reuse_packet) { params.pkt = out_pkt; @@ -665,34 +665,32 @@ run_measure_one_config(crypto_args_t *cargs, int rc = 0; if (create_session_from_config(&session, config, cargs)) - rc = -1; - - if (!rc) { - if (cargs->payload_length) { - rc = run_measure_one(cargs, config, &session, - cargs->payload_length, &result); - if (!rc) { - print_result_header(); - print_result(cargs, cargs->payload_length, - config, &result); - } - } else { - unsigned i; + return -1; + if (cargs->payload_length) { + rc = run_measure_one(cargs, config, &session, + cargs->payload_length, &result); + if (!rc) { print_result_header(); - for (i = 0; i < num_payloads; i++) { - rc = run_measure_one(cargs, config, &session, - payloads[i], &result); - if (rc) - break; - print_result(cargs, payloads[i], - config, &result); - } + print_result(cargs, cargs->payload_length, + config, &result); + } + } else { + unsigned i; + + print_result_header(); + for (i = 0; i < num_payloads; i++) { + rc = run_measure_one(cargs, config, &session, + payloads[i], &result); + if (rc) + break; + print_result(cargs, payloads[i], + config, &result); } } - if (session != ODP_CRYPTO_SESSION_INVALID) - odp_crypto_session_destroy(session); + odp_crypto_session_destroy(session); + return rc; } diff --git a/test/common_plat/performance/odp_l2fwd.c b/test/common_plat/performance/odp_l2fwd.c index 8f5c5e152..78b3633bd 100644 --- a/test/common_plat/performance/odp_l2fwd.c +++ b/test/common_plat/performance/odp_l2fwd.c @@ -104,6 +104,7 @@ typedef struct { int src_change; /**< Change source eth addresses */ int error_check; /**< Check packet errors */ int sched_mode; /**< Scheduler mode */ + int num_groups; /**< Number of scheduling groups */ } appl_args_t; static int exit_threads; /**< Break workers loop if set to 1 */ @@ -130,6 +131,7 @@ typedef union { typedef struct thread_args_t { int thr_idx; int num_pktio; + int num_groups; struct { odp_pktin_queue_t pktin; @@ -142,7 +144,12 @@ typedef struct thread_args_t { int tx_queue_idx; } pktio[MAX_PKTIOS]; - stats_t *stats; /**< Pointer to per thread stats */ + /* Groups to join */ + odp_schedule_group_t group[MAX_PKTIOS]; + + /* Pointer to per thread stats */ + stats_t *stats; + } thread_args_t; /** @@ -297,6 +304,22 @@ static int run_worker_sched_mode(void *arg) thr = odp_thread_id(); + if (gbl_args->appl.num_groups) { + odp_thrmask_t mask; + + odp_thrmask_zero(&mask); + odp_thrmask_set(&mask, thr); + + /* Join non-default groups */ + for (i = 0; i < thr_args->num_groups; i++) { + if (odp_schedule_group_join(thr_args->group[i], + &mask)) { + LOG_ERR("Join failed\n"); + return -1; + } + } + } + num_pktio = thr_args->num_pktio; if (num_pktio > MAX_PKTIOS) { @@ -590,12 +613,13 @@ static int run_worker_direct_mode(void *arg) * @retval -1 on failure */ static int create_pktio(const char *dev, int idx, int num_rx, int num_tx, - odp_pool_t pool) + odp_pool_t pool, odp_schedule_group_t group) { odp_pktio_t pktio; odp_pktio_param_t pktio_param; odp_schedule_sync_t sync_mode; odp_pktio_capability_t capa; + odp_pktio_config_t config; odp_pktin_queue_param_t pktin_param; odp_pktout_queue_param_t pktout_param; odp_pktio_op_mode_t mode_rx; @@ -632,6 +656,12 @@ static int create_pktio(const char *dev, int idx, int num_rx, int num_tx, return -1; } + odp_pktio_config_init(&config); + config.parser.layer = gbl_args->appl.error_check ? + ODP_PKTIO_PARSER_LAYER_ALL : + ODP_PKTIO_PARSER_LAYER_NONE; + odp_pktio_config(pktio, &config); + odp_pktin_queue_param_init(&pktin_param); odp_pktout_queue_param_init(&pktout_param); @@ -650,7 +680,7 @@ static int create_pktio(const char *dev, int idx, int num_rx, int num_tx, pktin_param.queue_param.sched.prio = ODP_SCHED_PRIO_DEFAULT; pktin_param.queue_param.sched.sync = sync_mode; - pktin_param.queue_param.sched.group = ODP_SCHED_GROUP_ALL; + pktin_param.queue_param.sched.group = group; } if (num_rx > (int)capa.max_input_queues) { @@ -1016,39 +1046,46 @@ static void usage(char *progname) printf("\n" "OpenDataPlane L2 forwarding application.\n" "\n" - "Usage: %s OPTIONS\n" + "Usage: %s [options]\n" + "\n" " E.g. %s -i eth0,eth1,eth2,eth3 -m 0 -t 1\n" - " In the above example,\n" - " eth0 will send pkts to eth1 and vice versa\n" - " eth2 will send pkts to eth3 and vice versa\n" + " In the above example,\n" + " eth0 will send pkts to eth1 and vice versa\n" + " eth2 will send pkts to eth3 and vice versa\n" "\n" "Mandatory OPTIONS:\n" - " -i, --interface Eth interfaces (comma-separated, no spaces)\n" - " Interface count min 1, max %i\n" + " -i, --interface <name> Eth interfaces (comma-separated, no spaces)\n" + " Interface count min 1, max %i\n" "\n" "Optional OPTIONS:\n" - " -m, --mode Packet input mode\n" - " 0: Direct mode: PKTIN_MODE_DIRECT (default)\n" - " 1: Scheduler mode with parallel queues: PKTIN_MODE_SCHED + SCHED_SYNC_PARALLEL\n" - " 2: Scheduler mode with atomic queues: PKTIN_MODE_SCHED + SCHED_SYNC_ATOMIC\n" - " 3: Scheduler mode with ordered queues: PKTIN_MODE_SCHED + SCHED_SYNC_ORDERED\n" - " 4: Plain queue mode: ODP_PKTIN_MODE_QUEUE\n" - " -o, --out_mode Packet output mode\n" - " 0: Direct mode: PKTOUT_MODE_DIRECT (default)\n" - " 1: Queue mode: PKTOUT_MODE_QUEUE\n" - " -c, --count <number> CPU count.\n" - " -t, --time <number> Time in seconds to run.\n" - " -a, --accuracy <number> Time in seconds get print statistics\n" + " -m, --mode <arg> Packet input mode\n" + " 0: Direct mode: PKTIN_MODE_DIRECT (default)\n" + " 1: Scheduler mode with parallel queues:\n" + " PKTIN_MODE_SCHED + SCHED_SYNC_PARALLEL\n" + " 2: Scheduler mode with atomic queues:\n" + " PKTIN_MODE_SCHED + SCHED_SYNC_ATOMIC\n" + " 3: Scheduler mode with ordered queues:\n" + " PKTIN_MODE_SCHED + SCHED_SYNC_ORDERED\n" + " 4: Plain queue mode: PKTIN_MODE_QUEUE\n" + " -o, --out_mode <arg> Packet output mode\n" + " 0: Direct mode: PKTOUT_MODE_DIRECT (default)\n" + " 1: Queue mode: PKTOUT_MODE_QUEUE\n" + " -c, --count <num> CPU count.\n" + " -t, --time <sec> Time in seconds to run.\n" + " -a, --accuracy <sec> Time in seconds get print statistics\n" " (default is 1 second).\n" - " -d, --dst_change 0: Don't change packets' dst eth addresses\n" - " 1: Change packets' dst eth addresses (default)\n" - " -s, --src_change 0: Don't change packets' src eth addresses\n" - " 1: Change packets' src eth addresses (default)\n" - " -r, --dst_addr Destination addresses (comma-separated, no spaces)\n" - " Requires also the -d flag to be set\n" - " -e, --error_check 0: Don't check packet errors (default)\n" - " 1: Check packet errors\n" - " -h, --help Display help and exit.\n\n" + " -d, --dst_change <arg> 0: Don't change packets' dst eth addresses\n" + " 1: Change packets' dst eth addresses (default)\n" + " -s, --src_change <arg> 0: Don't change packets' src eth addresses\n" + " 1: Change packets' src eth addresses (default)\n" + " -r, --dst_addr <addr> Destination addresses (comma-separated, no spaces)\n" + " Requires also the -d flag to be set\n" + " -e, --error_check <arg> 0: Don't check packet errors (default)\n" + " 1: Check packet errors\n" + " -g, --groups <num> Number of groups to use: 0 ... num\n" + " 0: SCHED_GROUP_ALL (default)\n" + " num: must not exceed number of interfaces or workers\n" + " -h, --help Display help and exit.\n\n" "\n", NO_PATH(progname), NO_PATH(progname), MAX_PKTIOS ); } @@ -1079,11 +1116,12 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) {"dst_change", required_argument, NULL, 'd'}, {"src_change", required_argument, NULL, 's'}, {"error_check", required_argument, NULL, 'e'}, + {"groups", required_argument, NULL, 'g'}, {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} }; - static const char *shortopts = "+c:+t:+a:i:m:o:r:d:s:e:h"; + static const char *shortopts = "+c:+t:+a:i:m:o:r:d:s:e:g:h"; /* let helper collect its own arguments (e.g. --odph_proc) */ odph_parse_options(argc, argv, shortopts, longopts); @@ -1092,6 +1130,7 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) appl_args->accuracy = 1; /* get and print pps stats second */ appl_args->dst_change = 1; /* change eth dst address by default */ appl_args->src_change = 1; /* change eth src address by default */ + appl_args->num_groups = 0; /* use default group */ appl_args->error_check = 0; /* don't check packet errors by default */ opterr = 0; /* do not issue errors on helper options */ @@ -1217,6 +1256,9 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) case 'e': appl_args->error_check = atoi(optarg); break; + case 'g': + appl_args->num_groups = atoi(optarg); + break; case 'h': usage(argv[0]); exit(EXIT_SUCCESS); @@ -1305,6 +1347,24 @@ static void gbl_args_init(args_t *args) } } +static void create_groups(int num, odp_schedule_group_t *group) +{ + int i; + odp_thrmask_t zero; + + odp_thrmask_zero(&zero); + + /* Create groups */ + for (i = 0; i < num; i++) { + group[i] = odp_schedule_group_create(NULL, &zero); + + if (group[i] == ODP_SCHED_GROUP_INVALID) { + LOG_ERR("Group create failed\n"); + exit(EXIT_FAILURE); + } + } +} + /** * ODP L2 forwarding main function */ @@ -1325,6 +1385,8 @@ int main(int argc, char *argv[]) int if_count; int (*thr_run_func)(void *); odp_instance_t instance; + int num_groups; + odp_schedule_group_t group[MAX_PKTIOS]; /* Init ODP before calling anything else */ if (odp_init_global(&instance, NULL, NULL)) { @@ -1374,10 +1436,23 @@ int main(int argc, char *argv[]) if_count = gbl_args->appl.if_count; + num_groups = gbl_args->appl.num_groups; + printf("num worker threads: %i\n", num_workers); printf("first CPU: %i\n", odp_cpumask_first(&cpumask)); printf("cpu mask: %s\n", cpumaskstr); + if (num_groups) + printf("num groups: %i\n", num_groups); + + printf("\n"); + + if (num_groups > if_count || num_groups > num_workers) { + LOG_ERR("Too many groups. Number of groups may not exceed " + "number of interfaces or workers.\n"); + exit(EXIT_FAILURE); + } + /* Create packet pool */ odp_pool_param_init(¶ms); params.pkt.seg_len = SHM_PKT_POOL_BUF_SIZE; @@ -1399,9 +1474,18 @@ int main(int argc, char *argv[]) bind_workers(); + /* Default */ + if (num_groups == 0) { + group[0] = ODP_SCHED_GROUP_ALL; + num_groups = 1; + } else { + create_groups(num_groups, group); + } + for (i = 0; i < if_count; ++i) { const char *dev = gbl_args->appl.if_names[i]; int num_rx, num_tx; + odp_schedule_group_t grp; /* A queue per worker in scheduled mode */ num_rx = num_workers; @@ -1413,7 +1497,10 @@ int main(int argc, char *argv[]) num_tx = gbl_args->pktios[i].num_tx_thr; } - if (create_pktio(dev, i, num_rx, num_tx, pool)) + /* Round robin pktios to groups */ + grp = group[i % num_groups]; + + if (create_pktio(dev, i, num_rx, num_tx, pool, grp)) exit(EXIT_FAILURE); /* Save interface ethernet address */ @@ -1473,6 +1560,10 @@ int main(int argc, char *argv[]) thr_params.thr_type = ODP_THREAD_WORKER; thr_params.instance = instance; + /* Round robin threads to groups */ + gbl_args->thread[i].num_groups = 1; + gbl_args->thread[i].group[0] = group[i % num_groups]; + gbl_args->thread[i].stats = &stats[i]; odp_cpumask_zero(&thd_mask); diff --git a/test/common_plat/performance/odp_l2fwd_run.sh b/test/common_plat/performance/odp_l2fwd_run.sh index dd42ede97..6871e4b07 100755 --- a/test/common_plat/performance/odp_l2fwd_run.sh +++ b/test/common_plat/performance/odp_l2fwd_run.sh @@ -96,6 +96,9 @@ run_l2fwd() ret=1 elif [ $ret -eq 0 ]; then PASS_PPS=5000 + if [ "${TEST}" = "coverage" ]; then + PASS_PPS=10 + fi MAX_PPS=$(awk '/TEST RESULT/ {print $3}' $LOG) if [ "$MAX_PPS" -lt "$PASS_PPS" ]; then echo "FAIL: pps below threshold $MAX_PPS < $PASS_PPS" diff --git a/test/common_plat/performance/odp_pktio_ordered.c b/test/common_plat/performance/odp_pktio_ordered.c index bff4586e5..4bb0bef93 100644 --- a/test/common_plat/performance/odp_pktio_ordered.c +++ b/test/common_plat/performance/odp_pktio_ordered.c @@ -586,6 +586,7 @@ static int create_pktio(const char *dev, int idx, int num_rx, int num_tx, odp_pktio_t pktio; odp_pktio_param_t pktio_param; odp_pktio_capability_t capa; + odp_pktio_config_t config; odp_pktin_queue_param_t pktin_param; odp_pktout_queue_param_t pktout_param; odp_pktio_op_mode_t mode_rx; @@ -611,6 +612,10 @@ static int create_pktio(const char *dev, int idx, int num_rx, int num_tx, return -1; } + odp_pktio_config_init(&config); + config.parser.layer = ODP_PKTIO_PARSER_LAYER_L2; + odp_pktio_config(pktio, &config); + odp_pktin_queue_param_init(&pktin_param); odp_pktout_queue_param_init(&pktout_param); diff --git a/test/common_plat/performance/odp_pktio_ordered_run.sh b/test/common_plat/performance/odp_pktio_ordered_run.sh index d91211c0c..d6c2be526 100755 --- a/test/common_plat/performance/odp_pktio_ordered_run.sh +++ b/test/common_plat/performance/odp_pktio_ordered_run.sh @@ -5,14 +5,21 @@ # # SPDX-License-Identifier: BSD-3-Clause # +TEST_SRC_DIR=$(dirname $0) +TEST_DIR="${TEST_DIR:-$(dirname $0)}" DURATION=5 LOG=odp_pktio_ordered.log LOOPS=100000000 PASS_PPS=5000 -PCAP_IN=`find . ${TEST_DIR} $(dirname $0) -name udp64.pcap -print -quit` +PCAP_IN=`find . ${TEST_SRC_DIR} $(dirname $0) -name udp64.pcap -print -quit` PCAP_OUT=/dev/null +if [ ! -f ${PCAP_IN} ]; then + echo "FAIL: no udp64.pcap" + exit 1 +fi + # This just turns off output buffering so that you still get periodic # output while piping to tee, as long as stdbuf is available. if [ "$(which stdbuf)" != "" ]; then @@ -21,20 +28,29 @@ else STDBUF= fi -$STDBUF ./odp_pktio_ordered${EXEEXT} -i pcap:in=${PCAP_IN}:loops=$LOOPS,\ -pcap:out=${PCAP_OUT} -t $DURATION | tee $LOG +$STDBUF ${TEST_DIR}/odp_pktio_ordered${EXEEXT} \ + -i pcap:in=${PCAP_IN}:loops=$LOOPS,pcap:out=${PCAP_OUT} \ + -t $DURATION | tee $LOG -ret=$? +ret=${PIPESTATUS[0]} + +if [ $ret -ne 0 ]; then + echo "FAIL: no odp_pktio_ordered${EXEEXT}" + rm -f $LOG + exit $ret +fi if [ ! -f $LOG ]; then echo "FAIL: $LOG not found" ret=1 -elif [ $ret -eq 0 ]; then - MAX_PPS=$(awk '/TEST RESULT/ {print $3}' $LOG) - if [ "$MAX_PPS" -lt "$PASS_PPS" ]; then - echo "FAIL: pps below threshold $MAX_PPS < $PASS_PPS" - ret=1 - fi + exit $ret +fi + +MAX_PPS=$(awk '/TEST RESULT/ {print $3}' $LOG) +echo "MAX_PPS=$MAX_PPS" +if [ $MAX_PPS -lt $PASS_PPS ]; then + echo "FAIL: pps below threshold $MAX_PPS < $PASS_PPS" + ret=1 fi rm -f $LOG diff --git a/test/common_plat/performance/odp_sched_latency.c b/test/common_plat/performance/odp_sched_latency.c index 2b28cd7bc..026f2f6c7 100644 --- a/test/common_plat/performance/odp_sched_latency.c +++ b/test/common_plat/performance/odp_sched_latency.c @@ -57,9 +57,10 @@ ODP_STATIC_ASSERT(LO_PRIO_QUEUES <= MAX_QUEUES, "Too many LO priority queues"); /** Test event types */ typedef enum { - WARM_UP, /**< Warm up event */ - TRAFFIC, /**< Event used only as traffic load */ - SAMPLE /**< Event used to measure latency */ + WARM_UP, /**< Warm up event */ + COOL_DOWN,/**< Last event on queue */ + TRAFFIC, /**< Event used only as traffic load */ + SAMPLE /**< Event used to measure latency */ } event_type_t; /** Test event */ @@ -112,20 +113,52 @@ typedef struct { /** * Clear all scheduled queues. * - * Retry to be sure that all buffers have been scheduled. + * Use special cool_down event to guarantee that queue is drained. */ -static void clear_sched_queues(void) +static void clear_sched_queues(test_globals_t *globals) { odp_event_t ev; + odp_buffer_t buf; + test_event_t *event; + int i, j; + odp_queue_t fromq; - while (1) { - ev = odp_schedule(NULL, ODP_SCHED_NO_WAIT); + /* Allocate the cool_down event. */ + buf = odp_buffer_alloc(globals->pool); + if (buf == ODP_BUFFER_INVALID) + LOG_ABORT("Buffer alloc failed.\n"); - if (ev == ODP_EVENT_INVALID) - break; + event = odp_buffer_addr(buf); + event->type = COOL_DOWN; + ev = odp_buffer_to_event(buf); - odp_event_free(ev); + for (i = 0; i < NUM_PRIOS; i++) { + for (j = 0; j < globals->args.prio[i].queues; j++) { + /* Enqueue cool_down event on each queue. */ + if (odp_queue_enq(globals->queue[i][j], ev)) + LOG_ABORT("Queue enqueue failed.\n"); + + /* Invoke scheduler until cool_down event has been + * received. */ + while (1) { + ev = odp_schedule(NULL, ODP_SCHED_WAIT); + buf = odp_buffer_from_event(ev); + event = odp_buffer_addr(buf); + if (event->type == COOL_DOWN) + break; + odp_event_free(ev); + } + } } + + /* Free the cool_down event. */ + odp_event_free(ev); + + /* Call odp_schedule() to trigger a release of any scheduler context. */ + ev = odp_schedule(&fromq, ODP_SCHED_NO_WAIT); + if (ev != ODP_EVENT_INVALID) + LOG_ABORT("Queue %" PRIu64 " not empty.\n", + odp_queue_to_u64(fromq)); } /** @@ -394,10 +427,10 @@ static int test_schedule(int thr, test_globals_t *globals) odp_barrier_wait(&globals->barrier); - clear_sched_queues(); - - if (thr == MAIN_THREAD) + if (thr == MAIN_THREAD) { + clear_sched_queues(globals); print_results(globals); + } return 0; } @@ -634,16 +667,7 @@ int main(int argc, char *argv[]) return -1; } - printf("\n"); - printf("ODP system info\n"); - printf("---------------\n"); - printf("ODP API version: %s\n", odp_version_api_str()); - printf("ODP impl name: %s\n", odp_version_impl_name()); - printf("ODP impl details: %s\n", odp_version_impl_str()); - printf("CPU model: %s\n", odp_cpu_model_str()); - printf("CPU freq (hz): %" PRIu64 "\n", odp_cpu_hz_max()); - printf("Cache line size: %i\n", odp_sys_cache_line_size()); - printf("Max CPU count: %i\n", odp_cpu_count()); + odp_sys_info_print(); /* Get default worker cpumask */ if (args.cpu_count) @@ -654,9 +678,10 @@ int main(int argc, char *argv[]) (void)odp_cpumask_to_str(&cpumask, cpumaskstr, sizeof(cpumaskstr)); - printf("Worker threads: %i\n", num_workers); - printf("First CPU: %i\n", odp_cpumask_first(&cpumask)); - printf("CPU mask: %s\n\n", cpumaskstr); + printf("CPU mask info:\n"); + printf(" Worker threads: %i\n", num_workers); + printf(" First CPU: %i\n", odp_cpumask_first(&cpumask)); + printf(" CPU mask: %s\n", cpumaskstr); thread_tbl = calloc(sizeof(odph_odpthread_t), num_workers); if (!thread_tbl) { diff --git a/test/common_plat/performance/odp_scheduling.c b/test/common_plat/performance/odp_scheduling.c index c74a07133..38e76257b 100644 --- a/test/common_plat/performance/odp_scheduling.c +++ b/test/common_plat/performance/odp_scheduling.c @@ -273,7 +273,7 @@ static int test_plain_queue(int thr, test_globals_t *globals) test_message_t *t_msg; odp_queue_t queue; uint64_t c1, c2, cycles; - int i; + int i, j; /* Alloc test message */ buf = odp_buffer_alloc(globals->pool); @@ -307,7 +307,15 @@ static int test_plain_queue(int thr, test_globals_t *globals) return -1; } - ev = odp_queue_deq(queue); + /* When enqueue and dequeue are decoupled (e.g. not using a + * common lock), an enqueued event may not be immediately + * visible to dequeue. So we just try again for a while. */ + for (j = 0; j < 100; j++) { + ev = odp_queue_deq(queue); + if (ev != ODP_EVENT_INVALID) + break; + odp_cpu_pause(); + } buf = odp_buffer_from_event(ev); diff --git a/test/common_plat/validation/api/classification/odp_classification_common.c b/test/common_plat/validation/api/classification/odp_classification_common.c index 3b379c14c..eca30b87f 100644 --- a/test/common_plat/validation/api/classification/odp_classification_common.c +++ b/test/common_plat/validation/api/classification/odp_classification_common.c @@ -278,14 +278,14 @@ odp_packet_t create_packet(cls_packet_info_t pkt_info) ethhdr = (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, NULL); memcpy(ethhdr->src.addr, &src_mac, ODPH_ETHADDR_LEN); memcpy(ethhdr->dst.addr, &dst_mac_be, ODPH_ETHADDR_LEN); - vlan_type = (odp_u16be_t *)ðhdr->type; + vlan_type = (odp_u16be_t *)(void *)ðhdr->type; vlan_hdr = (odph_vlanhdr_t *)(ethhdr + 1); if (pkt_info.vlan_qinq) { odp_packet_has_vlan_qinq_set(pkt, 1); *vlan_type = odp_cpu_to_be_16(ODPH_ETHTYPE_VLAN_OUTER); vlan_hdr->tci = odp_cpu_to_be_16(0); - vlan_type = (uint16_t *)&vlan_hdr->type; + vlan_type = (uint16_t *)(void *)&vlan_hdr->type; vlan_hdr++; } if (pkt_info.vlan) { diff --git a/test/common_plat/validation/api/crypto/crypto.h b/test/common_plat/validation/api/crypto/crypto.h index 9b909aa04..dd15b448f 100644 --- a/test/common_plat/validation/api/crypto/crypto.h +++ b/test/common_plat/validation/api/crypto/crypto.h @@ -10,6 +10,8 @@ #include "odp_cunit_common.h" /* test functions: */ +void crypto_test_enc_alg_null(void); +void crypto_test_dec_alg_null(void); void crypto_test_enc_alg_3des_cbc(void); void crypto_test_enc_alg_3des_cbc_ovr_iv(void); void crypto_test_dec_alg_3des_cbc(void); @@ -22,8 +24,14 @@ void crypto_test_enc_alg_aes128_gcm(void); void crypto_test_enc_alg_aes128_gcm_ovr_iv(void); void crypto_test_dec_alg_aes128_gcm(void); void crypto_test_dec_alg_aes128_gcm_ovr_iv(void); -void crypto_test_alg_hmac_md5(void); -void crypto_test_alg_hmac_sha256(void); +void crypto_test_gen_alg_hmac_md5(void); +void crypto_test_check_alg_hmac_md5(void); +void crypto_test_gen_alg_hmac_sha1(void); +void crypto_test_check_alg_hmac_sha1(void); +void crypto_test_gen_alg_hmac_sha256(void); +void crypto_test_check_alg_hmac_sha256(void); +void crypto_test_gen_alg_hmac_sha512(void); +void crypto_test_check_alg_hmac_sha512(void); /* test arrays: */ extern odp_testinfo_t crypto_suite[]; diff --git a/test/common_plat/validation/api/crypto/odp_crypto_test_inp.c b/test/common_plat/validation/api/crypto/odp_crypto_test_inp.c index 43ddb2ffd..ae600e230 100644 --- a/test/common_plat/validation/api/crypto/odp_crypto_test_inp.c +++ b/test/common_plat/validation/api/crypto/odp_crypto_test_inp.c @@ -28,8 +28,12 @@ static const char *auth_alg_name(odp_auth_alg_t auth) return "ODP_AUTH_ALG_NULL"; case ODP_AUTH_ALG_MD5_HMAC: return "ODP_AUTH_ALG_MD5_HMAC"; + case ODP_AUTH_ALG_SHA1_HMAC: + return "ODP_AUTH_ALG_SHA1_HMAC"; case ODP_AUTH_ALG_SHA256_HMAC: return "ODP_AUTH_ALG_SHA256_HMAC"; + case ODP_AUTH_ALG_SHA512_HMAC: + return "ODP_AUTH_ALG_SHA512_HMAC"; case ODP_AUTH_ALG_AES_GCM: return "ODP_AUTH_ALG_AES_GCM"; default: @@ -65,14 +69,17 @@ static const char *cipher_alg_name(odp_cipher_alg_t cipher) * buffer can be used. * */ static void alg_test(odp_crypto_op_t op, + odp_bool_t should_fail, odp_cipher_alg_t cipher_alg, odp_crypto_iv_t ses_iv, uint8_t *op_iv_ptr, odp_crypto_key_t cipher_key, odp_auth_alg_t auth_alg, odp_crypto_key_t auth_key, - odp_crypto_data_range_t *cipher_range, - odp_crypto_data_range_t *auth_range, + odp_packet_data_range_t *cipher_range, + odp_packet_data_range_t *auth_range, + uint8_t *aad, + uint32_t aad_len, const uint8_t *plaintext, unsigned int plaintext_len, const uint8_t *ciphertext, @@ -128,9 +135,15 @@ static void alg_test(odp_crypto_op_t op, if (auth_alg == ODP_AUTH_ALG_NULL && !(capa.auths.bit.null)) rc = -1; + if (auth_alg == ODP_AUTH_ALG_SHA1_HMAC && + !(capa.auths.bit.sha1_hmac)) + rc = -1; if (auth_alg == ODP_AUTH_ALG_SHA256_HMAC && !(capa.auths.bit.sha256_hmac)) rc = -1; + if (auth_alg == ODP_AUTH_ALG_SHA512_HMAC && + !(capa.auths.bit.sha512_hmac)) + rc = -1; CU_ASSERT(!rc); CU_ASSERT((~capa.auths.all_bits & capa.hw_auths.all_bits) == 0); @@ -138,13 +151,8 @@ static void alg_test(odp_crypto_op_t op, num = odp_crypto_cipher_capability(cipher_alg, cipher_capa, MAX_ALG_CAPA); - if (cipher_alg != ODP_CIPHER_ALG_NULL) { - CU_ASSERT(num > 0); - found = 0; - } else { - CU_ASSERT(num == 0); - found = 1; - } + CU_ASSERT(num > 0); + found = 0; CU_ASSERT(num <= MAX_ALG_CAPA); @@ -164,13 +172,8 @@ static void alg_test(odp_crypto_op_t op, num = odp_crypto_auth_capability(auth_alg, auth_capa, MAX_ALG_CAPA); - if (auth_alg != ODP_AUTH_ALG_NULL) { - CU_ASSERT(num > 0); - found = 0; - } else { - CU_ASSERT(num == 0); - found = 1; - } + CU_ASSERT(num > 0); + found = 0; CU_ASSERT(num <= MAX_ALG_CAPA); @@ -200,6 +203,7 @@ static void alg_test(odp_crypto_op_t op, ses_params.cipher_key = cipher_key; ses_params.iv = ses_iv; ses_params.auth_key = auth_key; + ses_params.auth_digest_len = digest_len; rc = odp_crypto_session_create(&ses_params, &session, &status); CU_ASSERT_FATAL(!rc); @@ -238,7 +242,14 @@ static void alg_test(odp_crypto_op_t op, if (op_iv_ptr) op_params.override_iv_ptr = op_iv_ptr; + op_params.aad.ptr = aad; + op_params.aad.length = aad_len; + op_params.hash_result_offset = plaintext_len; + if (0 != digest_len) { + memcpy(data_addr + op_params.hash_result_offset, + digest, digest_len); + } rc = odp_crypto_operation(&op_params, &posted, &result); if (rc < 0) { @@ -259,8 +270,15 @@ static void alg_test(odp_crypto_op_t op, odp_crypto_compl_free(compl_event); } - CU_ASSERT(result.ok); CU_ASSERT(result.pkt == pkt); + CU_ASSERT(result.ctx == (void *)0xdeadbeef); + + if (should_fail) { + CU_ASSERT(!result.ok); + goto cleanup; + } + + CU_ASSERT(result.ok); if (cipher_alg != ODP_CIPHER_ALG_NULL) CU_ASSERT(!memcmp(data_addr, ciphertext, ciphertext_len)); @@ -268,8 +286,6 @@ static void alg_test(odp_crypto_op_t op, if (op == ODP_CRYPTO_OP_ENCODE && auth_alg != ODP_AUTH_ALG_NULL) CU_ASSERT(!memcmp(data_addr + op_params.hash_result_offset, digest, digest_len)); - - CU_ASSERT(result.ctx == (void *)0xdeadbeef); cleanup: rc = odp_crypto_session_destroy(session); CU_ASSERT(!rc); @@ -330,10 +346,18 @@ static int check_alg_support(odp_cipher_alg_t cipher, odp_auth_alg_t auth) if (!capability.auths.bit.md5_hmac) return ODP_TEST_INACTIVE; break; + case ODP_AUTH_ALG_SHA1_HMAC: + if (!capability.auths.bit.sha1_hmac) + return ODP_TEST_INACTIVE; + break; case ODP_AUTH_ALG_SHA256_HMAC: if (!capability.auths.bit.sha256_hmac) return ODP_TEST_INACTIVE; break; + case ODP_AUTH_ALG_SHA512_HMAC: + if (!capability.auths.bit.sha512_hmac) + return ODP_TEST_INACTIVE; + break; case ODP_AUTH_ALG_AES_GCM: if (!capability.auths.bit.aes_gcm) return ODP_TEST_INACTIVE; @@ -416,6 +440,73 @@ static int check_auth_options(odp_auth_alg_t auth, uint32_t key_len, return 1; } +static int check_alg_null(void) +{ + return check_alg_support(ODP_CIPHER_ALG_NULL, ODP_AUTH_ALG_NULL); +} + +void crypto_test_enc_alg_null(void) +{ + odp_crypto_key_t cipher_key = { .data = NULL, .length = 0 }, + auth_key = { .data = NULL, .length = 0 }; + odp_crypto_iv_t iv = { .data = NULL, .length = 0}; + unsigned int test_vec_num = (sizeof(null_reference_length) / + sizeof(null_reference_length[0])); + unsigned int i; + + for (i = 0; i < test_vec_num; i++) { + if (!check_cipher_options(ODP_CIPHER_ALG_NULL, + cipher_key.length, iv.length)) + continue; + + alg_test(ODP_CRYPTO_OP_ENCODE, + 0, + ODP_CIPHER_ALG_NULL, + iv, + NULL, + cipher_key, + ODP_AUTH_ALG_NULL, + auth_key, + NULL, NULL, + NULL, 0, + null_reference_plaintext[i], + null_reference_length[i], + null_reference_plaintext[i], + null_reference_length[i], NULL, 0); + } +} + +void crypto_test_dec_alg_null(void) +{ + odp_crypto_key_t cipher_key = { .data = NULL, .length = 0 }, + auth_key = { .data = NULL, .length = 0 }; + odp_crypto_iv_t iv = { .data = NULL, .length = 0 }; + unsigned int test_vec_num = (sizeof(null_reference_length) / + sizeof(null_reference_length[0])); + unsigned int i; + + for (i = 0; i < test_vec_num; i++) { + if (!check_cipher_options(ODP_CIPHER_ALG_NULL, + cipher_key.length, iv.length)) + continue; + + alg_test(ODP_CRYPTO_OP_DECODE, + 0, + ODP_CIPHER_ALG_NULL, + iv, + NULL, + cipher_key, + ODP_AUTH_ALG_NULL, + auth_key, + NULL, NULL, + NULL, 0, + null_reference_plaintext[i], + null_reference_length[i], + null_reference_plaintext[i], + null_reference_length[i], NULL, 0); + } +} + static int check_alg_3des_cbc(void) { return check_alg_support(ODP_CIPHER_ALG_3DES_CBC, ODP_AUTH_ALG_NULL); @@ -445,6 +536,7 @@ void crypto_test_enc_alg_3des_cbc(void) continue; alg_test(ODP_CRYPTO_OP_ENCODE, + 0, ODP_CIPHER_ALG_3DES_CBC, iv, NULL, @@ -452,6 +544,7 @@ void crypto_test_enc_alg_3des_cbc(void) ODP_AUTH_ALG_NULL, auth_key, NULL, NULL, + NULL, 0, tdes_cbc_reference_plaintext[i], tdes_cbc_reference_length[i], tdes_cbc_reference_ciphertext[i], @@ -480,6 +573,7 @@ void crypto_test_enc_alg_3des_cbc_ovr_iv(void) continue; alg_test(ODP_CRYPTO_OP_ENCODE, + 0, ODP_CIPHER_ALG_3DES_CBC, iv, tdes_cbc_reference_iv[i], @@ -487,6 +581,7 @@ void crypto_test_enc_alg_3des_cbc_ovr_iv(void) ODP_AUTH_ALG_NULL, auth_key, NULL, NULL, + NULL, 0, tdes_cbc_reference_plaintext[i], tdes_cbc_reference_length[i], tdes_cbc_reference_ciphertext[i], @@ -519,6 +614,7 @@ void crypto_test_dec_alg_3des_cbc(void) continue; alg_test(ODP_CRYPTO_OP_DECODE, + 0, ODP_CIPHER_ALG_3DES_CBC, iv, NULL, @@ -526,6 +622,7 @@ void crypto_test_dec_alg_3des_cbc(void) ODP_AUTH_ALG_NULL, auth_key, NULL, NULL, + NULL, 0, tdes_cbc_reference_ciphertext[i], tdes_cbc_reference_length[i], tdes_cbc_reference_plaintext[i], @@ -556,6 +653,7 @@ void crypto_test_dec_alg_3des_cbc_ovr_iv(void) continue; alg_test(ODP_CRYPTO_OP_DECODE, + 0, ODP_CIPHER_ALG_3DES_CBC, iv, tdes_cbc_reference_iv[i], @@ -563,6 +661,7 @@ void crypto_test_dec_alg_3des_cbc_ovr_iv(void) ODP_AUTH_ALG_NULL, auth_key, NULL, NULL, + NULL, 0, tdes_cbc_reference_ciphertext[i], tdes_cbc_reference_length[i], tdes_cbc_reference_plaintext[i], @@ -598,10 +697,12 @@ void crypto_test_enc_alg_aes128_gcm(void) cipher_key.length, iv.length)) continue; if (!check_auth_options(ODP_AUTH_ALG_AES_GCM, - auth_key.length, AES128_GCM_CHECK_LEN)) + auth_key.length, + aes128_gcm_reference_tag_length[i])) continue; alg_test(ODP_CRYPTO_OP_ENCODE, + 0, ODP_CIPHER_ALG_AES_GCM, iv, NULL, @@ -609,14 +710,16 @@ void crypto_test_enc_alg_aes128_gcm(void) ODP_AUTH_ALG_AES_GCM, auth_key, &aes128_gcm_cipher_range[i], - &aes128_gcm_auth_range[i], + &aes128_gcm_cipher_range[i], + aes128_gcm_reference_aad[i], + aes128_gcm_reference_aad_length[i], aes128_gcm_reference_plaintext[i], aes128_gcm_reference_length[i], aes128_gcm_reference_ciphertext[i], aes128_gcm_reference_length[i], aes128_gcm_reference_ciphertext[i] + aes128_gcm_reference_length[i], - AES128_GCM_CHECK_LEN); + aes128_gcm_reference_tag_length[i]); } } @@ -641,10 +744,12 @@ void crypto_test_enc_alg_aes128_gcm_ovr_iv(void) cipher_key.length, iv.length)) continue; if (!check_auth_options(ODP_AUTH_ALG_AES_GCM, - auth_key.length, AES128_GCM_CHECK_LEN)) + auth_key.length, + aes128_gcm_reference_tag_length[i])) continue; alg_test(ODP_CRYPTO_OP_ENCODE, + 0, ODP_CIPHER_ALG_AES_GCM, iv, aes128_gcm_reference_iv[i], @@ -652,14 +757,16 @@ void crypto_test_enc_alg_aes128_gcm_ovr_iv(void) ODP_AUTH_ALG_AES_GCM, auth_key, &aes128_gcm_cipher_range[i], - &aes128_gcm_auth_range[i], + &aes128_gcm_cipher_range[i], + aes128_gcm_reference_aad[i], + aes128_gcm_reference_aad_length[i], aes128_gcm_reference_plaintext[i], aes128_gcm_reference_length[i], aes128_gcm_reference_ciphertext[i], aes128_gcm_reference_length[i], aes128_gcm_reference_ciphertext[i] + aes128_gcm_reference_length[i], - AES128_GCM_CHECK_LEN); + aes128_gcm_reference_tag_length[i]); } } @@ -673,10 +780,13 @@ void crypto_test_dec_alg_aes128_gcm(void) odp_crypto_key_t cipher_key = { .data = NULL, .length = 0 }, auth_key = { .data = NULL, .length = 0 }; odp_crypto_iv_t iv = { .data = NULL, .length = AES128_GCM_IV_LEN }; + uint8_t wrong_digest[AES128_GCM_DIGEST_LEN]; unsigned int test_vec_num = (sizeof(aes128_gcm_reference_length) / sizeof(aes128_gcm_reference_length[0])); unsigned int i; + memset(wrong_digest, 0xa5, sizeof(wrong_digest)); + for (i = 0; i < test_vec_num; i++) { cipher_key.data = aes128_gcm_reference_key[i]; cipher_key.length = sizeof(aes128_gcm_reference_key[i]); @@ -687,10 +797,12 @@ void crypto_test_dec_alg_aes128_gcm(void) cipher_key.length, iv.length)) continue; if (!check_auth_options(ODP_AUTH_ALG_AES_GCM, - auth_key.length, AES128_GCM_CHECK_LEN)) + auth_key.length, + aes128_gcm_reference_tag_length[i])) continue; alg_test(ODP_CRYPTO_OP_DECODE, + 0, ODP_CIPHER_ALG_AES_GCM, iv, NULL, @@ -698,14 +810,35 @@ void crypto_test_dec_alg_aes128_gcm(void) ODP_AUTH_ALG_AES_GCM, auth_key, &aes128_gcm_cipher_range[i], - &aes128_gcm_auth_range[i], + &aes128_gcm_cipher_range[i], + aes128_gcm_reference_aad[i], + aes128_gcm_reference_aad_length[i], aes128_gcm_reference_ciphertext[i], - aes128_gcm_reference_length[i] + AES128_GCM_CHECK_LEN, + aes128_gcm_reference_length[i], aes128_gcm_reference_plaintext[i], aes128_gcm_reference_length[i], aes128_gcm_reference_ciphertext[i] + aes128_gcm_reference_length[i], - AES128_GCM_CHECK_LEN); + aes128_gcm_reference_tag_length[i]); + + alg_test(ODP_CRYPTO_OP_DECODE, + 1, + ODP_CIPHER_ALG_AES_GCM, + iv, + NULL, + cipher_key, + ODP_AUTH_ALG_AES_GCM, + auth_key, + &aes128_gcm_cipher_range[i], + &aes128_gcm_cipher_range[i], + aes128_gcm_reference_aad[i], + aes128_gcm_reference_aad_length[i], + aes128_gcm_reference_ciphertext[i], + aes128_gcm_reference_length[i], + aes128_gcm_reference_plaintext[i], + aes128_gcm_reference_length[i], + wrong_digest, + aes128_gcm_reference_tag_length[i]); } } @@ -719,10 +852,13 @@ void crypto_test_dec_alg_aes128_gcm_ovr_iv(void) odp_crypto_key_t cipher_key = { .data = NULL, .length = 0 }, auth_key = { .data = NULL, .length = 0 }; odp_crypto_iv_t iv = { .data = NULL, .length = AES128_GCM_IV_LEN }; + uint8_t wrong_digest[AES128_GCM_DIGEST_LEN]; unsigned int test_vec_num = (sizeof(aes128_gcm_reference_length) / sizeof(aes128_gcm_reference_length[0])); unsigned int i; + memset(wrong_digest, 0xa5, sizeof(wrong_digest)); + for (i = 0; i < test_vec_num; i++) { cipher_key.data = aes128_gcm_reference_key[i]; cipher_key.length = sizeof(aes128_gcm_reference_key[i]); @@ -731,10 +867,12 @@ void crypto_test_dec_alg_aes128_gcm_ovr_iv(void) cipher_key.length, iv.length)) continue; if (!check_auth_options(ODP_AUTH_ALG_AES_GCM, - auth_key.length, AES128_GCM_CHECK_LEN)) + auth_key.length, + aes128_gcm_reference_tag_length[i])) continue; alg_test(ODP_CRYPTO_OP_DECODE, + 0, ODP_CIPHER_ALG_AES_GCM, iv, aes128_gcm_reference_iv[i], @@ -742,14 +880,35 @@ void crypto_test_dec_alg_aes128_gcm_ovr_iv(void) ODP_AUTH_ALG_AES_GCM, auth_key, &aes128_gcm_cipher_range[i], - &aes128_gcm_auth_range[i], + &aes128_gcm_cipher_range[i], + aes128_gcm_reference_aad[i], + aes128_gcm_reference_aad_length[i], aes128_gcm_reference_ciphertext[i], - aes128_gcm_reference_length[i] + AES128_GCM_CHECK_LEN, + aes128_gcm_reference_length[i], aes128_gcm_reference_plaintext[i], aes128_gcm_reference_length[i], aes128_gcm_reference_ciphertext[i] + aes128_gcm_reference_length[i], - AES128_GCM_CHECK_LEN); + aes128_gcm_reference_tag_length[i]); + + alg_test(ODP_CRYPTO_OP_DECODE, + 1, + ODP_CIPHER_ALG_AES_GCM, + iv, + aes128_gcm_reference_iv[i], + cipher_key, + ODP_AUTH_ALG_AES_GCM, + auth_key, + &aes128_gcm_cipher_range[i], + &aes128_gcm_cipher_range[i], + aes128_gcm_reference_aad[i], + aes128_gcm_reference_aad_length[i], + aes128_gcm_reference_ciphertext[i], + aes128_gcm_reference_length[i], + aes128_gcm_reference_plaintext[i], + aes128_gcm_reference_length[i], + wrong_digest, + aes128_gcm_reference_tag_length[i]); } } @@ -782,6 +941,7 @@ void crypto_test_enc_alg_aes128_cbc(void) continue; alg_test(ODP_CRYPTO_OP_ENCODE, + 0, ODP_CIPHER_ALG_AES_CBC, iv, NULL, @@ -789,6 +949,7 @@ void crypto_test_enc_alg_aes128_cbc(void) ODP_AUTH_ALG_NULL, auth_key, NULL, NULL, + NULL, 0, aes128_cbc_reference_plaintext[i], aes128_cbc_reference_length[i], aes128_cbc_reference_ciphertext[i], @@ -817,6 +978,7 @@ void crypto_test_enc_alg_aes128_cbc_ovr_iv(void) continue; alg_test(ODP_CRYPTO_OP_ENCODE, + 0, ODP_CIPHER_ALG_AES_CBC, iv, aes128_cbc_reference_iv[i], @@ -824,6 +986,7 @@ void crypto_test_enc_alg_aes128_cbc_ovr_iv(void) ODP_AUTH_ALG_NULL, auth_key, NULL, NULL, + NULL, 0, aes128_cbc_reference_plaintext[i], aes128_cbc_reference_length[i], aes128_cbc_reference_ciphertext[i], @@ -856,6 +1019,7 @@ void crypto_test_dec_alg_aes128_cbc(void) continue; alg_test(ODP_CRYPTO_OP_DECODE, + 0, ODP_CIPHER_ALG_AES_CBC, iv, NULL, @@ -863,6 +1027,7 @@ void crypto_test_dec_alg_aes128_cbc(void) ODP_AUTH_ALG_NULL, auth_key, NULL, NULL, + NULL, 0, aes128_cbc_reference_ciphertext[i], aes128_cbc_reference_length[i], aes128_cbc_reference_plaintext[i], @@ -893,6 +1058,7 @@ void crypto_test_dec_alg_aes128_cbc_ovr_iv(void) continue; alg_test(ODP_CRYPTO_OP_DECODE, + 0, ODP_CIPHER_ALG_AES_CBC, iv, aes128_cbc_reference_iv[i], @@ -900,6 +1066,7 @@ void crypto_test_dec_alg_aes128_cbc_ovr_iv(void) ODP_AUTH_ALG_NULL, auth_key, NULL, NULL, + NULL, 0, aes128_cbc_reference_ciphertext[i], aes128_cbc_reference_length[i], aes128_cbc_reference_plaintext[i], @@ -919,7 +1086,7 @@ static int check_alg_hmac_md5(void) * In addition the test verifies if the implementation can use the * packet buffer as completion event buffer. * */ -void crypto_test_alg_hmac_md5(void) +void crypto_test_gen_alg_hmac_md5(void) { odp_crypto_key_t cipher_key = { .data = NULL, .length = 0 }, auth_key = { .data = NULL, .length = 0 }; @@ -934,10 +1101,50 @@ void crypto_test_alg_hmac_md5(void) auth_key.length = sizeof(hmac_md5_reference_key[i]); if (!check_auth_options(ODP_AUTH_ALG_MD5_HMAC, auth_key.length, - HMAC_MD5_96_CHECK_LEN)) + hmac_md5_reference_digest_length[i])) continue; alg_test(ODP_CRYPTO_OP_ENCODE, + 0, + ODP_CIPHER_ALG_NULL, + iv, + iv.data, + cipher_key, + ODP_AUTH_ALG_MD5_HMAC, + auth_key, + NULL, NULL, + NULL, 0, + hmac_md5_reference_plaintext[i], + hmac_md5_reference_length[i], + NULL, 0, + hmac_md5_reference_digest[i], + hmac_md5_reference_digest_length[i]); + } +} + +void crypto_test_check_alg_hmac_md5(void) +{ + odp_crypto_key_t cipher_key = { .data = NULL, .length = 0 }, + auth_key = { .data = NULL, .length = 0 }; + odp_crypto_iv_t iv = { .data = NULL, .length = 0 }; + uint8_t wrong_digest[HMAC_MD5_DIGEST_LEN]; + + unsigned int test_vec_num = (sizeof(hmac_md5_reference_length) / + sizeof(hmac_md5_reference_length[0])); + unsigned int i; + + memset(wrong_digest, 0xa5, sizeof(wrong_digest)); + + for (i = 0; i < test_vec_num; i++) { + auth_key.data = hmac_md5_reference_key[i]; + auth_key.length = sizeof(hmac_md5_reference_key[i]); + + if (!check_auth_options(ODP_AUTH_ALG_MD5_HMAC, auth_key.length, + hmac_md5_reference_digest_length[i])) + continue; + + alg_test(ODP_CRYPTO_OP_DECODE, + 0, ODP_CIPHER_ALG_NULL, iv, iv.data, @@ -945,11 +1152,135 @@ void crypto_test_alg_hmac_md5(void) ODP_AUTH_ALG_MD5_HMAC, auth_key, NULL, NULL, + NULL, 0, hmac_md5_reference_plaintext[i], hmac_md5_reference_length[i], NULL, 0, hmac_md5_reference_digest[i], - HMAC_MD5_96_CHECK_LEN); + hmac_md5_reference_digest_length[i]); + + alg_test(ODP_CRYPTO_OP_DECODE, + 1, + ODP_CIPHER_ALG_NULL, + iv, + iv.data, + cipher_key, + ODP_AUTH_ALG_MD5_HMAC, + auth_key, + NULL, NULL, + NULL, 0, + hmac_md5_reference_plaintext[i], + hmac_md5_reference_length[i], + NULL, 0, + wrong_digest, + hmac_md5_reference_digest_length[i]); + } +} + +static int check_alg_hmac_sha1(void) +{ + return check_alg_support(ODP_CIPHER_ALG_NULL, ODP_AUTH_ALG_SHA1_HMAC); +} + +/* This test verifies the correctness of HMAC_SHA1 digest operation. + * The output check length is truncated to 12 bytes (96 bits) as + * returned by the crypto operation API call. + * Note that hash digest is a one-way operation. + * In addition the test verifies if the implementation can use the + * packet buffer as completion event buffer. + * */ +void crypto_test_gen_alg_hmac_sha1(void) +{ + odp_crypto_key_t cipher_key = { .data = NULL, .length = 0 }, + auth_key = { .data = NULL, .length = 0 }; + odp_crypto_iv_t iv = { .data = NULL, .length = 0 }; + + unsigned int test_vec_num = (sizeof(hmac_sha1_reference_length) / + sizeof(hmac_sha1_reference_length[0])); + + unsigned int i; + + for (i = 0; i < test_vec_num; i++) { + auth_key.data = hmac_sha1_reference_key[i]; + auth_key.length = sizeof(hmac_sha1_reference_key[i]); + + if (!check_auth_options(ODP_AUTH_ALG_SHA1_HMAC, + auth_key.length, + HMAC_SHA1_96_CHECK_LEN)) + continue; + + alg_test(ODP_CRYPTO_OP_ENCODE, + 0, + ODP_CIPHER_ALG_NULL, + iv, + iv.data, + cipher_key, + ODP_AUTH_ALG_SHA1_HMAC, + auth_key, + NULL, NULL, + NULL, 0, + hmac_sha1_reference_plaintext[i], + hmac_sha1_reference_length[i], + NULL, 0, + hmac_sha1_reference_digest[i], + HMAC_SHA1_96_CHECK_LEN); + } +} + +void crypto_test_check_alg_hmac_sha1(void) +{ + odp_crypto_key_t cipher_key = { .data = NULL, .length = 0 }, + auth_key = { .data = NULL, .length = 0 }; + odp_crypto_iv_t iv = { .data = NULL, .length = 0 }; + uint8_t wrong_digest[HMAC_SHA1_DIGEST_LEN]; + + unsigned int test_vec_num = (sizeof(hmac_sha1_reference_length) / + sizeof(hmac_sha1_reference_length[0])); + + unsigned int i; + + memset(wrong_digest, 0xa5, sizeof(wrong_digest)); + + for (i = 0; i < test_vec_num; i++) { + auth_key.data = hmac_sha1_reference_key[i]; + auth_key.length = sizeof(hmac_sha1_reference_key[i]); + + if (!check_auth_options(ODP_AUTH_ALG_SHA1_HMAC, + auth_key.length, + HMAC_SHA1_96_CHECK_LEN)) + continue; + + alg_test(ODP_CRYPTO_OP_DECODE, + 0, + ODP_CIPHER_ALG_NULL, + iv, + iv.data, + cipher_key, + ODP_AUTH_ALG_SHA1_HMAC, + auth_key, + NULL, NULL, + NULL, 0, + hmac_sha1_reference_plaintext[i], + hmac_sha1_reference_length[i], + NULL, 0, + hmac_sha1_reference_digest[i], + HMAC_SHA1_96_CHECK_LEN); + + alg_test(ODP_CRYPTO_OP_DECODE, + 1, + ODP_CIPHER_ALG_NULL, + iv, + iv.data, + cipher_key, + ODP_AUTH_ALG_SHA1_HMAC, + auth_key, + NULL, NULL, + NULL, 0, + hmac_sha1_reference_plaintext[i], + hmac_sha1_reference_length[i], + NULL, 0, + wrong_digest, + HMAC_SHA1_96_CHECK_LEN); } } @@ -958,14 +1289,14 @@ static int check_alg_hmac_sha256(void) return check_alg_support(ODP_CIPHER_ALG_NULL, ODP_AUTH_ALG_SHA256_HMAC); } -/* This test verifies the correctness of HMAC_MD5 digest operation. - * The output check length is truncated to 12 bytes (96 bits) as +/* This test verifies the correctness of HMAC_SHA256 digest operation. + * The output check length is truncated to 16 bytes (128 bits) as * returned by the crypto operation API call. * Note that hash digest is a one-way operation. * In addition the test verifies if the implementation can use the * packet buffer as completion event buffer. * */ -void crypto_test_alg_hmac_sha256(void) +void crypto_test_gen_alg_hmac_sha256(void) { odp_crypto_key_t cipher_key = { .data = NULL, .length = 0 }, auth_key = { .data = NULL, .length = 0 }; @@ -982,10 +1313,11 @@ void crypto_test_alg_hmac_sha256(void) if (!check_auth_options(ODP_AUTH_ALG_SHA256_HMAC, auth_key.length, - HMAC_SHA256_128_CHECK_LEN)) + hmac_sha256_reference_digest_length[i])) continue; alg_test(ODP_CRYPTO_OP_ENCODE, + 0, ODP_CIPHER_ALG_NULL, iv, iv.data, @@ -993,11 +1325,176 @@ void crypto_test_alg_hmac_sha256(void) ODP_AUTH_ALG_SHA256_HMAC, auth_key, NULL, NULL, + NULL, 0, hmac_sha256_reference_plaintext[i], hmac_sha256_reference_length[i], NULL, 0, hmac_sha256_reference_digest[i], - HMAC_SHA256_128_CHECK_LEN); + hmac_sha256_reference_digest_length[i]); + } +} + +void crypto_test_check_alg_hmac_sha256(void) +{ + odp_crypto_key_t cipher_key = { .data = NULL, .length = 0 }, + auth_key = { .data = NULL, .length = 0 }; + odp_crypto_iv_t iv = { .data = NULL, .length = 0 }; + uint8_t wrong_digest[HMAC_SHA256_DIGEST_LEN]; + + unsigned int test_vec_num = (sizeof(hmac_sha256_reference_length) / + sizeof(hmac_sha256_reference_length[0])); + + unsigned int i; + + memset(wrong_digest, 0xa5, sizeof(wrong_digest)); + + for (i = 0; i < test_vec_num; i++) { + auth_key.data = hmac_sha256_reference_key[i]; + auth_key.length = sizeof(hmac_sha256_reference_key[i]); + + if (!check_auth_options(ODP_AUTH_ALG_SHA256_HMAC, + auth_key.length, + hmac_sha256_reference_digest_length[i])) + continue; + + alg_test(ODP_CRYPTO_OP_DECODE, + 0, + ODP_CIPHER_ALG_NULL, + iv, + iv.data, + cipher_key, + ODP_AUTH_ALG_SHA256_HMAC, + auth_key, + NULL, NULL, + NULL, 0, + hmac_sha256_reference_plaintext[i], + hmac_sha256_reference_length[i], + NULL, 0, + hmac_sha256_reference_digest[i], + hmac_sha256_reference_digest_length[i]); + + alg_test(ODP_CRYPTO_OP_DECODE, + 1, + ODP_CIPHER_ALG_NULL, + iv, + iv.data, + cipher_key, + ODP_AUTH_ALG_SHA256_HMAC, + auth_key, + NULL, NULL, + NULL, 0, + hmac_sha256_reference_plaintext[i], + hmac_sha256_reference_length[i], + NULL, 0, + wrong_digest, + hmac_sha256_reference_digest_length[i]); + } +} + +static int check_alg_hmac_sha512(void) +{ + return check_alg_support(ODP_CIPHER_ALG_NULL, ODP_AUTH_ALG_SHA512_HMAC); +} + +/* This test verifies the correctness of HMAC_SHA512 digest operation. + * The output check length is truncated to 32 bytes (256 bits) as + * returned by the crypto operation API call. + * Note that hash digest is a one-way operation. + * In addition the test verifies if the implementation can use the + * packet buffer as completion event buffer. + * */ +void crypto_test_gen_alg_hmac_sha512(void) +{ + odp_crypto_key_t cipher_key = { .data = NULL, .length = 0 }, + auth_key = { .data = NULL, .length = 0 }; + odp_crypto_iv_t iv = { .data = NULL, .length = 0 }; + + unsigned int test_vec_num = (sizeof(hmac_sha512_reference_length) / + sizeof(hmac_sha512_reference_length[0])); + + unsigned int i; + + for (i = 0; i < test_vec_num; i++) { + auth_key.data = hmac_sha512_reference_key[i]; + auth_key.length = sizeof(hmac_sha512_reference_key[i]); + + if (!check_auth_options(ODP_AUTH_ALG_SHA512_HMAC, + auth_key.length, + HMAC_SHA512_256_CHECK_LEN)) + continue; + + alg_test(ODP_CRYPTO_OP_ENCODE, + 0, + ODP_CIPHER_ALG_NULL, + iv, + iv.data, + cipher_key, + ODP_AUTH_ALG_SHA512_HMAC, + auth_key, + NULL, NULL, + NULL, 0, + hmac_sha512_reference_plaintext[i], + hmac_sha512_reference_length[i], + NULL, 0, + hmac_sha512_reference_digest[i], + HMAC_SHA512_256_CHECK_LEN); + } +} + +void crypto_test_check_alg_hmac_sha512(void) +{ + odp_crypto_key_t cipher_key = { .data = NULL, .length = 0 }, + auth_key = { .data = NULL, .length = 0 }; + odp_crypto_iv_t iv = { .data = NULL, .length = 0 }; + uint8_t wrong_digest[HMAC_SHA512_DIGEST_LEN]; + + unsigned int test_vec_num = (sizeof(hmac_sha512_reference_length) / + sizeof(hmac_sha512_reference_length[0])); + + unsigned int i; + + memset(wrong_digest, 0xa5, sizeof(wrong_digest)); + + for (i = 0; i < test_vec_num; i++) { + auth_key.data = hmac_sha512_reference_key[i]; + auth_key.length = sizeof(hmac_sha512_reference_key[i]); + + if (!check_auth_options(ODP_AUTH_ALG_SHA512_HMAC, + auth_key.length, + HMAC_SHA512_256_CHECK_LEN)) + continue; + + alg_test(ODP_CRYPTO_OP_DECODE, + 0, + ODP_CIPHER_ALG_NULL, + iv, + iv.data, + cipher_key, + ODP_AUTH_ALG_SHA512_HMAC, + auth_key, + NULL, NULL, + NULL, 0, + hmac_sha512_reference_plaintext[i], + hmac_sha512_reference_length[i], + NULL, 0, + hmac_sha512_reference_digest[i], + HMAC_SHA512_256_CHECK_LEN); + + alg_test(ODP_CRYPTO_OP_DECODE, + 1, + ODP_CIPHER_ALG_NULL, + iv, + iv.data, + cipher_key, + ODP_AUTH_ALG_SHA512_HMAC, + auth_key, + NULL, NULL, + NULL, 0, + hmac_sha512_reference_plaintext[i], + hmac_sha512_reference_length[i], + NULL, 0, + wrong_digest, + HMAC_SHA512_256_CHECK_LEN); } } @@ -1026,6 +1523,10 @@ int crypto_suite_async_init(void) } odp_testinfo_t crypto_suite[] = { + ODP_TEST_INFO_CONDITIONAL(crypto_test_enc_alg_null, + check_alg_null), + ODP_TEST_INFO_CONDITIONAL(crypto_test_dec_alg_null, + check_alg_null), ODP_TEST_INFO_CONDITIONAL(crypto_test_enc_alg_3des_cbc, check_alg_3des_cbc), ODP_TEST_INFO_CONDITIONAL(crypto_test_dec_alg_3des_cbc, @@ -1050,10 +1551,22 @@ odp_testinfo_t crypto_suite[] = { check_alg_aes_gcm), ODP_TEST_INFO_CONDITIONAL(crypto_test_dec_alg_aes128_gcm_ovr_iv, check_alg_aes_gcm), - ODP_TEST_INFO_CONDITIONAL(crypto_test_alg_hmac_md5, + ODP_TEST_INFO_CONDITIONAL(crypto_test_gen_alg_hmac_md5, check_alg_hmac_md5), - ODP_TEST_INFO_CONDITIONAL(crypto_test_alg_hmac_sha256, + ODP_TEST_INFO_CONDITIONAL(crypto_test_check_alg_hmac_md5, + check_alg_hmac_md5), + ODP_TEST_INFO_CONDITIONAL(crypto_test_gen_alg_hmac_sha1, + check_alg_hmac_sha1), + ODP_TEST_INFO_CONDITIONAL(crypto_test_check_alg_hmac_sha1, + check_alg_hmac_sha1), + ODP_TEST_INFO_CONDITIONAL(crypto_test_gen_alg_hmac_sha256, + check_alg_hmac_sha256), + ODP_TEST_INFO_CONDITIONAL(crypto_test_check_alg_hmac_sha256, check_alg_hmac_sha256), + ODP_TEST_INFO_CONDITIONAL(crypto_test_gen_alg_hmac_sha512, + check_alg_hmac_sha512), + ODP_TEST_INFO_CONDITIONAL(crypto_test_check_alg_hmac_sha512, + check_alg_hmac_sha512), ODP_TEST_INFO_NULL, }; diff --git a/test/common_plat/validation/api/crypto/test_vectors.h b/test/common_plat/validation/api/crypto/test_vectors.h index da4610f33..bd8bf347d 100644 --- a/test/common_plat/validation/api/crypto/test_vectors.h +++ b/test/common_plat/validation/api/crypto/test_vectors.h @@ -8,6 +8,18 @@ #define _ODP_TEST_CRYPTO_VECTORS_H_ #include "test_vectors_len.h" + +/** length in bytes */ +static uint32_t null_reference_length[] = { 8, 16 }; + +static uint8_t +null_reference_plaintext[][NULL_MAX_DATA_LEN] = { + {0x32, 0x6a, 0x49, 0x4c, 0xd3, 0x3f, 0xe7, 0x56}, + + {0x84, 0x40, 0x1f, 0x78, 0xfe, 0x6c, 0x10, 0x87, 0x6d, 0x8e, 0xa2, 0x30, + 0x94, 0xea, 0x53, 0x09} +}; + /* TDES-CBC reference vectors, according to * "http://csrc.nist.gov/groups/STM/cavp/documents/des/DESMMT.pdf" */ @@ -137,29 +149,31 @@ static uint8_t aes128_gcm_reference_iv[][AES128_GCM_IV_LEN] = { 0xa2, 0xfc, 0xa1, 0xa3 } }; -static uint32_t aes128_gcm_reference_length[] = { 84, 72, 72, 40}; +static uint32_t aes128_gcm_reference_length[] = { 72, 64, 64, 28}; -static odp_crypto_data_range_t aes128_gcm_cipher_range[] = { - { .offset = 12, .length = 72 }, - { .offset = 8, .length = 64 }, - { .offset = 8, .length = 64 }, - { .offset = 12, .length = 28 }, -}; +static uint32_t aes128_gcm_reference_tag_length[] = { 16, 16, 16, 16}; -static odp_crypto_data_range_t aes128_gcm_auth_range[] = { - { .offset = 0, .length = 84 }, - { .offset = 0, .length = 72 }, +static uint32_t aes128_gcm_reference_aad_length[] = { 12, 8, 8, 12}; + +static odp_packet_data_range_t aes128_gcm_cipher_range[] = { { .offset = 0, .length = 72 }, - { .offset = 0, .length = 40 }, + { .offset = 0, .length = 64 }, + { .offset = 0, .length = 64 }, + { .offset = 0, .length = 28 }, +}; + +static uint8_t aes128_gcm_reference_aad[][AES128_GCM_MAX_DATA_LEN] = { + { 0x00, 0x00, 0x43, 0x21, 0x87, 0x65, 0x43, 0x21, + 0x00, 0x00, 0x00, 0x00, }, + { 0x00, 0x00, 0xa5, 0xf8, 0x00, 0x00, 0x00, 0x0a, }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, }, + { 0x42, 0xf6, 0x7e, 0x3f, 0x10, 0x10, 0x10, 0x10, + 0x10, 0x10, 0x10, 0x10, }, }; static uint8_t aes128_gcm_reference_plaintext[][AES128_GCM_MAX_DATA_LEN] = { - { /* Aad */ - 0x00, 0x00, 0x43, 0x21, 0x87, 0x65, 0x43, 0x21, - 0x00, 0x00, 0x00, 0x00, - /* Plain */ - 0x45, 0x00, 0x00, 0x48, 0x69, 0x9a, 0x00, 0x00, + { 0x45, 0x00, 0x00, 0x48, 0x69, 0x9a, 0x00, 0x00, 0x80, 0x11, 0x4d, 0xb7, 0xc0, 0xa8, 0x01, 0x02, 0xc0, 0xa8, 0x01, 0x01, 0x0a, 0x9b, 0xf1, 0x56, 0x38, 0xd3, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, @@ -169,10 +183,7 @@ aes128_gcm_reference_plaintext[][AES128_GCM_MAX_DATA_LEN] = { 0x63, 0x69, 0x74, 0x79, 0x02, 0x64, 0x6b, 0x00, 0x00, 0x21, 0x00, 0x01, 0x01, 0x02, 0x02, 0x01 }, - { /* Aad */ - 0x00, 0x00, 0xa5, 0xf8, 0x00, 0x00, 0x00, 0x0a, - /* Plain */ - 0x45, 0x00, 0x00, 0x3e, 0x69, 0x8f, 0x00, 0x00, + { 0x45, 0x00, 0x00, 0x3e, 0x69, 0x8f, 0x00, 0x00, 0x80, 0x11, 0x4d, 0xcc, 0xc0, 0xa8, 0x01, 0x02, 0xc0, 0xa8, 0x01, 0x01, 0x0a, 0x98, 0x00, 0x35, 0x00, 0x2a, 0x23, 0x43, 0xb2, 0xd0, 0x01, 0x00, @@ -181,10 +192,7 @@ aes128_gcm_reference_plaintext[][AES128_GCM_MAX_DATA_LEN] = { 0x65, 0x72, 0x63, 0x69, 0x74, 0x79, 0x02, 0x64, 0x6b, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01 }, - { /* Aad */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - /* Plain */ - 0x45, 0x00, 0x00, 0x3c, 0x99, 0xc5, 0x00, 0x00, + { 0x45, 0x00, 0x00, 0x3c, 0x99, 0xc5, 0x00, 0x00, 0x80, 0x01, 0xcb, 0x7a, 0x40, 0x67, 0x93, 0x18, 0x01, 0x01, 0x01, 0x01, 0x08, 0x00, 0x07, 0x5c, 0x02, 0x00, 0x44, 0x00, 0x61, 0x62, 0x63, 0x64, @@ -193,11 +201,7 @@ aes128_gcm_reference_plaintext[][AES128_GCM_MAX_DATA_LEN] = { 0x75, 0x76, 0x77, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x01, 0x02, 0x02, 0x01 }, - { /* Aad */ - 0x42, 0xf6, 0x7e, 0x3f, 0x10, 0x10, 0x10, 0x10, - 0x10, 0x10, 0x10, 0x10, - /* Plain */ - 0x45, 0x00, 0x00, 0x1c, 0x42, 0xa2, 0x00, 0x00, + { 0x45, 0x00, 0x00, 0x1c, 0x42, 0xa2, 0x00, 0x00, 0x80, 0x01, 0x44, 0x1f, 0x40, 0x67, 0x93, 0xb6, 0xe0, 0x00, 0x00, 0x02, 0x0a, 0x00, 0xf5, 0xff, 0x01, 0x02, 0x02, 0x01 } @@ -205,10 +209,7 @@ aes128_gcm_reference_plaintext[][AES128_GCM_MAX_DATA_LEN] = { static uint8_t aes128_gcm_reference_ciphertext[][AES128_GCM_MAX_DATA_LEN] = { - { /* Aad */ - 0x00, 0x00, 0x43, 0x21, 0x87, 0x65, 0x43, 0x21, - 0x00, 0x00, 0x00, 0x00, - /* Plain */ + { /* Plain */ 0xfe, 0xcf, 0x53, 0x7e, 0x72, 0x9d, 0x5b, 0x07, 0xdc, 0x30, 0xdf, 0x52, 0x8d, 0xd2, 0x2b, 0x76, 0x8d, 0x1b, 0x98, 0x73, 0x66, 0x96, 0xa6, 0xfd, @@ -222,9 +223,7 @@ aes128_gcm_reference_ciphertext[][AES128_GCM_MAX_DATA_LEN] = { 0x45, 0x90, 0x18, 0x14, 0x8f, 0x6c, 0xbe, 0x72, 0x2f, 0xd0, 0x47, 0x96, 0x56, 0x2d, 0xfd, 0xb4 }, - { /* Aad */ - 0x00, 0x00, 0xa5, 0xf8, 0x00, 0x00, 0x00, 0x0a, - /* Plain */ + { /* Plain */ 0xde, 0xb2, 0x2c, 0xd9, 0xb0, 0x7c, 0x72, 0xc1, 0x6e, 0x3a, 0x65, 0xbe, 0xeb, 0x8d, 0xf3, 0x04, 0xa5, 0xa5, 0x89, 0x7d, 0x33, 0xae, 0x53, 0x0f, @@ -236,9 +235,8 @@ aes128_gcm_reference_ciphertext[][AES128_GCM_MAX_DATA_LEN] = { /* Digest */ 0x83, 0xb7, 0x0d, 0x3a, 0xa8, 0xbc, 0x6e, 0xe4, 0xc3, 0x09, 0xe9, 0xd8, 0x5a, 0x41, 0xad, 0x4a }, - { /* Aad */ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - /* Plain */ + + { /* Plain */ 0x46, 0x88, 0xda, 0xf2, 0xf9, 0x73, 0xa3, 0x92, 0x73, 0x29, 0x09, 0xc3, 0x31, 0xd5, 0x6d, 0x60, 0xf6, 0x94, 0xab, 0xaa, 0x41, 0x4b, 0x5e, 0x7f, @@ -251,10 +249,7 @@ aes128_gcm_reference_ciphertext[][AES128_GCM_MAX_DATA_LEN] = { 0xf8, 0x21, 0xd4, 0x96, 0xee, 0xb0, 0x96, 0xe9, 0x8a, 0xd2, 0xb6, 0x9e, 0x47, 0x99, 0xc7, 0x1d }, - { /* Aad */ - 0x42, 0xf6, 0x7e, 0x3f, 0x10, 0x10, 0x10, 0x10, - 0x10, 0x10, 0x10, 0x10, - /* Plain */ + { /* Plain */ 0xfb, 0xa2, 0xca, 0x84, 0x5e, 0x5d, 0xf9, 0xf0, 0xf2, 0x2c, 0x3e, 0x6e, 0x86, 0xdd, 0x83, 0x1e, 0x1f, 0xc6, 0x57, 0x92, 0xcd, 0x1a, 0xf9, 0x13, @@ -306,6 +301,10 @@ static uint8_t hmac_md5_reference_digest[][HMAC_MD5_DIGEST_LEN] = { 0xdb, 0xb8, 0xc7, 0x33, 0xf0, 0xe8, 0xb3, 0xf6 } }; +static uint32_t hmac_md5_reference_digest_length[] = { + 12, 12, 12 +}; + static uint8_t hmac_sha256_reference_key[][HMAC_SHA256_KEY_LEN] = { { 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, @@ -350,4 +349,102 @@ static uint8_t hmac_sha256_reference_digest[][HMAC_SHA256_DIGEST_LEN] = { 0x85, 0x4d, 0xb8, 0xeb, 0xd0, 0x91, 0x81, 0xa7 } }; +static uint32_t hmac_sha256_reference_digest_length[] = { + 16, 16, 16 +}; + +static uint8_t hmac_sha1_reference_key[][HMAC_SHA1_KEY_LEN] = { + { 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0b, 0x0b, 0x0b, 0x0b }, + + /* "Jefe" */ + { 0x4a, 0x65, 0x66, 0x65 }, + + { 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa } +}; + +static uint32_t hmac_sha1_reference_length[] = { 8, 28, 50 }; + +static uint8_t +hmac_sha1_reference_plaintext[][HMAC_SHA1_MAX_DATA_LEN] = { + /* "Hi There" */ + { 0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65}, + + /* what do ya want for nothing?*/ + { 0x77, 0x68, 0x61, 0x74, 0x20, 0x64, 0x6f, 0x20, + 0x79, 0x61, 0x20, 0x77, 0x61, 0x6e, 0x74, 0x20, + 0x66, 0x6f, 0x72, 0x20, 0x6e, 0x6f, 0x74, 0x68, + 0x69, 0x6e, 0x67, 0x3f }, + + { 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd } +}; + +static uint8_t hmac_sha1_reference_digest[][HMAC_SHA1_DIGEST_LEN] = { + { 0xb6, 0x17, 0x31, 0x86, 0x55, 0x05, + 0x72, 0x64, 0xe2, 0x8b, 0xc0, 0xb6 }, + + { 0xef, 0xfc, 0xdf, 0x6a, 0xe5, 0xeb, + 0x2f, 0xa2, 0xd2, 0x74, 0x16, 0xd5 }, + + { 0x12, 0x5d, 0x73, 0x42, 0xb9, 0xac, + 0x11, 0xcd, 0x91, 0xa3, 0x9a, 0xf4 }, +}; + +static uint8_t hmac_sha512_reference_key[][HMAC_SHA512_KEY_LEN] = { + { 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0b, 0x0b, 0x0b, 0x0b }, + + /* "Jefe" */ + { 0x4a, 0x65, 0x66, 0x65 }, + + { 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa } +}; + +static uint32_t hmac_sha512_reference_length[] = { 8, 28, 50 }; + +static uint8_t +hmac_sha512_reference_plaintext[][HMAC_SHA512_MAX_DATA_LEN] = { + /* "Hi There" */ + { 0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65}, + + /* what do ya want for nothing?*/ + { 0x77, 0x68, 0x61, 0x74, 0x20, 0x64, 0x6f, 0x20, + 0x79, 0x61, 0x20, 0x77, 0x61, 0x6e, 0x74, 0x20, + 0x66, 0x6f, 0x72, 0x20, 0x6e, 0x6f, 0x74, 0x68, + 0x69, 0x6e, 0x67, 0x3f }, + + { 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, + 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd, 0xdd } +}; + +static uint8_t hmac_sha512_reference_digest[][HMAC_SHA512_DIGEST_LEN] = { + { 0x87, 0xaa, 0x7c, 0xde, 0xa5, 0xef, 0x61, 0x9d, + 0x4f, 0xf0, 0xb4, 0x24, 0x1a, 0x1d, 0x6c, 0xb0, + 0x23, 0x79, 0xf4, 0xe2, 0xce, 0x4e, 0xc2, 0x78, + 0x7a, 0xd0, 0xb3, 0x05, 0x45, 0xe1, 0x7c, 0xde }, + + { 0x16, 0x4b, 0x7a, 0x7b, 0xfc, 0xf8, 0x19, 0xe2, + 0xe3, 0x95, 0xfb, 0xe7, 0x3b, 0x56, 0xe0, 0xa3, + 0x87, 0xbd, 0x64, 0x22, 0x2e, 0x83, 0x1f, 0xd6, + 0x10, 0x27, 0x0c, 0xd7, 0xea, 0x25, 0x05, 0x54 }, + + { 0xfa, 0x73, 0xb0, 0x08, 0x9d, 0x56, 0xa2, 0x84, + 0xef, 0xb0, 0xf0, 0x75, 0x6c, 0x89, 0x0b, 0xe9, + 0xb1, 0xb5, 0xdb, 0xdd, 0x8e, 0xe8, 0x1a, 0x36, + 0x55, 0xf8, 0x3e, 0x33, 0xb2, 0x27, 0x9d, 0x39 } +}; + #endif diff --git a/test/common_plat/validation/api/crypto/test_vectors_len.h b/test/common_plat/validation/api/crypto/test_vectors_len.h index 4fbb5cd70..20a7ddbad 100644 --- a/test/common_plat/validation/api/crypto/test_vectors_len.h +++ b/test/common_plat/validation/api/crypto/test_vectors_len.h @@ -6,6 +6,9 @@ #ifndef TEST_VECTORS_LEN_ #define TEST_VECTORS_LEN_ +/* NULL */ +#define NULL_MAX_DATA_LEN 16 + /* TDES-CBC */ #define TDES_CBC_KEY_LEN 24 #define TDES_CBC_IV_LEN 8 @@ -21,18 +24,27 @@ #define AES128_GCM_IV_LEN 12 #define AES128_GCM_MAX_DATA_LEN 106 #define AES128_GCM_DIGEST_LEN 16 -#define AES128_GCM_CHECK_LEN 16 /* HMAC-MD5 */ #define HMAC_MD5_KEY_LEN 16 #define HMAC_MD5_MAX_DATA_LEN 128 #define HMAC_MD5_DIGEST_LEN 16 -#define HMAC_MD5_96_CHECK_LEN 12 /* HMAC-SHA256 */ #define HMAC_SHA256_KEY_LEN 32 #define HMAC_SHA256_MAX_DATA_LEN 128 #define HMAC_SHA256_DIGEST_LEN 32 -#define HMAC_SHA256_128_CHECK_LEN 16 + +/* HMAC-SHA1 */ +#define HMAC_SHA1_KEY_LEN 20 +#define HMAC_SHA1_MAX_DATA_LEN 128 +#define HMAC_SHA1_DIGEST_LEN 20 +#define HMAC_SHA1_96_CHECK_LEN 12 + +/* HMAC-SHA512 */ +#define HMAC_SHA512_KEY_LEN 64 +#define HMAC_SHA512_MAX_DATA_LEN 128 +#define HMAC_SHA512_DIGEST_LEN 64 +#define HMAC_SHA512_256_CHECK_LEN 32 #endif diff --git a/test/common_plat/validation/api/lock/lock.c b/test/common_plat/validation/api/lock/lock.c index bd9a2aad2..f8a1d8c01 100644 --- a/test/common_plat/validation/api/lock/lock.c +++ b/test/common_plat/validation/api/lock/lock.c @@ -257,7 +257,7 @@ static int ticketlock_api_tests(void *arg UNUSED) static void rwlock_api_test(odp_rwlock_t *rw_lock) { - int rc; + int rc = 0; odp_rwlock_init(rw_lock); /* CU_ASSERT(odp_rwlock_is_locked(rw_lock) == 0); */ @@ -265,23 +265,40 @@ static void rwlock_api_test(odp_rwlock_t *rw_lock) odp_rwlock_read_lock(rw_lock); rc = odp_rwlock_read_trylock(rw_lock); - CU_ASSERT(rc == 0); + CU_ASSERT(rc != 0); + if (rc == 1) + odp_rwlock_read_unlock(rw_lock); + rc = odp_rwlock_write_trylock(rw_lock); CU_ASSERT(rc == 0); + if (rc == 1) + odp_rwlock_write_unlock(rw_lock); odp_rwlock_read_unlock(rw_lock); rc = odp_rwlock_read_trylock(rw_lock); + CU_ASSERT(rc != 0); if (rc == 1) odp_rwlock_read_unlock(rw_lock); odp_rwlock_write_lock(rw_lock); /* CU_ASSERT(odp_rwlock_is_locked(rw_lock) == 1); */ + rc = odp_rwlock_read_trylock(rw_lock); + CU_ASSERT(rc == 0); + if (rc == 1) + odp_rwlock_read_unlock(rw_lock); + + rc = odp_rwlock_write_trylock(rw_lock); + CU_ASSERT(rc == 0); + if (rc == 1) + odp_rwlock_write_unlock(rw_lock); + odp_rwlock_write_unlock(rw_lock); /* CU_ASSERT(odp_rwlock_is_locked(rw_lock) == 0); */ rc = odp_rwlock_write_trylock(rw_lock); + CU_ASSERT(rc != 0); if (rc == 1) odp_rwlock_write_unlock(rw_lock); } diff --git a/test/common_plat/validation/api/pktio/Makefile.am b/test/common_plat/validation/api/pktio/Makefile.am index 466d690dc..c6368fba3 100644 --- a/test/common_plat/validation/api/pktio/Makefile.am +++ b/test/common_plat/validation/api/pktio/Makefile.am @@ -4,7 +4,7 @@ noinst_LTLIBRARIES = libtestpktio.la libtestpktio_la_SOURCES = pktio.c test_PROGRAMS = pktio_main$(EXEEXT) -dist_pktio_main_SOURCES = pktio_main.c +dist_pktio_main_SOURCES = pktio_main.c parser.c pktio_main_LDADD = libtestpktio.la $(LIBCUNIT_COMMON) $(LIBODP) -EXTRA_DIST = pktio.h +EXTRA_DIST = pktio.h parser.h diff --git a/test/common_plat/validation/api/pktio/parser.c b/test/common_plat/validation/api/pktio/parser.c new file mode 100644 index 000000000..ad7101d08 --- /dev/null +++ b/test/common_plat/validation/api/pktio/parser.c @@ -0,0 +1,545 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +#include <odp_api.h> +#include <odp_cunit_common.h> + +#include <odp/helper/odph_api.h> + +#include <stdlib.h> +#include "parser.h" +#include "pktio.h" + +#define MAX_NUM_IFACES 2 +#define PKT_POOL_NUM 256 +#define PKT_POOL_BUF_LEN (2 * 1024) + +/** + * local container for pktio attributes + */ +typedef struct { + const char *name; + odp_pktio_t hdl; + odp_pktout_queue_t pktout; + odp_pktin_queue_t pktin; +} pktio_info_t; + +/** Interface names used for testing */ +static const char *iface_name[MAX_NUM_IFACES]; + +/** Test interfaces */ +pktio_info_t pktios[MAX_NUM_IFACES]; +pktio_info_t *pktio_a; +pktio_info_t *pktio_b; + +/** Number of interfaces being used (1=loopback, 2=pair) */ +static int num_ifaces; + +/** While testing real-world interfaces additional time may be needed for + * external network to enable link to pktio interface that just become up. + */ +static bool wait_for_network; + +/** Parser packet pool */ +odp_pool_t parser_pool = ODP_POOL_INVALID; + +static inline void wait_linkup(odp_pktio_t pktio) +{ + /* wait 1 second for link up */ + uint64_t wait_ns = (10 * ODP_TIME_MSEC_IN_NS); + int wait_num = 100; + int i; + int ret = -1; + + for (i = 0; i < wait_num; i++) { + ret = odp_pktio_link_status(pktio); + if (ret < 0 || ret == 1) + break; + /* link is down, call status again after delay */ + odp_time_wait_ns(wait_ns); + } +} + +static int pkt_pool_create(void) +{ + odp_pool_capability_t capa; + odp_pool_param_t params; + + if (odp_pool_capability(&capa) != 0) { + printf("Error: unable to query pool capability.\n"); + return -1; + } + + if (capa.pkt.max_num && capa.pkt.max_num < PKT_POOL_NUM) { + printf("Error: packet pool size not supported.\n"); + printf("MAX: %" PRIu32 "\n", capa.pkt.max_num); + return -1; + } else if (capa.pkt.max_len && capa.pkt.max_len < PKT_POOL_BUF_LEN) { + printf("Error: packet length not supported.\n"); + return -1; + } else if (capa.pkt.max_seg_len && + capa.pkt.max_seg_len < PKT_POOL_BUF_LEN) { + printf("Error: segment length not supported.\n"); + return -1; + } + + odp_pool_param_init(¶ms); + params.pkt.seg_len = PKT_POOL_BUF_LEN; + params.pkt.len = PKT_POOL_BUF_LEN; + params.pkt.num = PKT_POOL_NUM; + params.type = ODP_POOL_PACKET; + + parser_pool = odp_pool_create("pkt_pool_default", ¶ms); + if (parser_pool == ODP_POOL_INVALID) { + printf("Error: packet pool create failed.\n"); + return -1; + } + + return 0; +} + +static odp_pktio_t create_pktio(int iface_idx, odp_pool_t pool) +{ + odp_pktio_t pktio; + odp_pktio_config_t config; + odp_pktio_param_t pktio_param; + const char *iface = iface_name[iface_idx]; + + odp_pktio_param_init(&pktio_param); + pktio_param.in_mode = ODP_PKTIN_MODE_DIRECT; + pktio_param.out_mode = ODP_PKTOUT_MODE_DIRECT; + + pktio = odp_pktio_open(iface, pool, &pktio_param); + if (pktio == ODP_PKTIO_INVALID) { + printf("Error: failed to open %s\n", iface); + return ODP_PKTIO_INVALID; + } + + odp_pktio_config_init(&config); + config.parser.layer = ODP_PKTIO_PARSER_LAYER_ALL; + if (odp_pktio_config(pktio, &config)) { + printf("Error: failed to configure %s\n", iface); + return ODP_PKTIO_INVALID; + } + + /* By default, single input and output queue is used */ + if (odp_pktin_queue_config(pktio, NULL)) { + printf("Error: failed to config input queue for %s\n", iface); + return ODP_PKTIO_INVALID; + } + if (odp_pktout_queue_config(pktio, NULL)) { + printf("Error: failed to config output queue for %s\n", iface); + return ODP_PKTIO_INVALID; + } + + if (wait_for_network) + odp_time_wait_ns(ODP_TIME_SEC_IN_NS / 4); + + return pktio; +} + +static odp_packet_t create_packet(const uint8_t *data, uint32_t len) +{ + odp_packet_t pkt; + + pkt = odp_packet_alloc(parser_pool, len); + if (pkt == ODP_PACKET_INVALID) + return ODP_PACKET_INVALID; + + if (odp_packet_copy_from_mem(pkt, 0, len, data)) { + printf("Error: failed to copy test packet data\n"); + odp_packet_free(pkt); + return ODP_PACKET_INVALID; + } + + odp_packet_l2_offset_set(pkt, 0); + + return pkt; +} + +/** + * Receive incoming packets and compare them to the original. Function returns + * a valid packet handle only when the received packet matches to the original + * packet. + */ +static odp_packet_t recv_and_cmp_packet(odp_pktin_queue_t pktin, + odp_packet_t orig_pkt, uint64_t ns) +{ + odp_packet_t pkt = ODP_PACKET_INVALID; + odp_time_t wait_time, end; + uint32_t orig_len; + uint8_t *orig_data; + + orig_len = odp_packet_len(orig_pkt); + orig_data = odp_packet_data(orig_pkt); + wait_time = odp_time_local_from_ns(ns); + end = odp_time_sum(odp_time_local(), wait_time); + + do { + int ret; + odp_packet_t tmp_pkt; + + ret = odp_pktin_recv(pktin, &tmp_pkt, 1); + if (ret < 0) + break; + + if (ret == 1) { + uint32_t len; + uint8_t *data; + + len = odp_packet_len(tmp_pkt); + data = odp_packet_data(tmp_pkt); + + if (len == orig_len && + memcmp(data, orig_data, len) == 0) { + pkt = tmp_pkt; + break; + } + odp_packet_free(tmp_pkt); + } + } while (odp_time_cmp(end, odp_time_local()) > 0); + + return pkt; +} + +/** + * Creates a test packet from data array and loops it through the test pktio + * interfaces forcing packet parsing. + */ +static odp_packet_t loopback_packet(pktio_info_t *pktio_a, + pktio_info_t *pktio_b, const uint8_t *data, + uint32_t len) +{ + odp_packet_t pkt; + odp_packet_t sent_pkt; + + pkt = create_packet(data, len); + if (pkt == ODP_PACKET_INVALID) { + CU_FAIL("failed to generate test packet"); + return ODP_PACKET_INVALID; + } + + pktio_pkt_set_macs(pkt, pktio_a->hdl, pktio_b->hdl); + + sent_pkt = odp_packet_copy(pkt, parser_pool); + if (sent_pkt == ODP_PACKET_INVALID) { + CU_FAIL_FATAL("failed to copy test packet"); + odp_packet_free(pkt); + return ODP_PACKET_INVALID; + } + + while (1) { + int ret = odp_pktout_send(pktio_a->pktout, &pkt, 1); + + if (ret < 0) { + CU_FAIL_FATAL("failed to send test packet"); + odp_packet_free(pkt); + odp_packet_free(sent_pkt); + return ODP_PACKET_INVALID; + } + if (ret == 1) + break; + } + + /* and wait for them to arrive back */ + pkt = recv_and_cmp_packet(pktio_b->pktin, sent_pkt, ODP_TIME_SEC_IN_NS); + odp_packet_free(sent_pkt); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + CU_ASSERT(odp_packet_input(pkt) == pktio_b->hdl); + CU_ASSERT(odp_packet_has_error(pkt) == 0); + + return pkt; +} + +void parser_test_arp(void) +{ + odp_packet_t pkt; + + pkt = loopback_packet(pktio_a, pktio_b, test_packet_arp, + sizeof(test_packet_arp)); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + CU_ASSERT(odp_packet_has_eth(pkt)); + CU_ASSERT(odp_packet_has_arp(pkt)); + + CU_ASSERT(!odp_packet_has_ipv4(pkt)); + CU_ASSERT(!odp_packet_has_ipv6(pkt)); + + odp_packet_free(pkt); +} + +void parser_test_ipv4_icmp(void) +{ + odp_packet_t pkt; + + pkt = loopback_packet(pktio_a, pktio_b, test_packet_ipv4_icmp, + sizeof(test_packet_ipv4_icmp)); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + CU_ASSERT(odp_packet_has_eth(pkt)); + CU_ASSERT(odp_packet_has_ipv4(pkt)); + CU_ASSERT(odp_packet_has_icmp(pkt)); + + CU_ASSERT(!odp_packet_has_ipv6(pkt)); + CU_ASSERT(!odp_packet_has_tcp(pkt)); + CU_ASSERT(!odp_packet_has_udp(pkt)); + + odp_packet_free(pkt); +} + +void parser_test_ipv4_tcp(void) +{ + odp_packet_t pkt; + + pkt = loopback_packet(pktio_a, pktio_b, test_packet_ipv4_tcp, + sizeof(test_packet_ipv4_tcp)); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + CU_ASSERT(odp_packet_has_eth(pkt)); + CU_ASSERT(odp_packet_has_ipv4(pkt)); + CU_ASSERT(odp_packet_has_tcp(pkt)); + + CU_ASSERT(!odp_packet_has_ipv6(pkt)); + CU_ASSERT(!odp_packet_has_udp(pkt)); + + odp_packet_free(pkt); +} + +void parser_test_ipv4_udp(void) +{ + odp_packet_t pkt; + + pkt = loopback_packet(pktio_a, pktio_b, test_packet_ipv4_udp, + sizeof(test_packet_ipv4_udp)); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + CU_ASSERT(odp_packet_has_eth(pkt)); + CU_ASSERT(odp_packet_has_ipv4(pkt)); + CU_ASSERT(odp_packet_has_udp(pkt)); + + CU_ASSERT(!odp_packet_has_ipv6(pkt)); + CU_ASSERT(!odp_packet_has_tcp(pkt)); + + odp_packet_free(pkt); +} + +void parser_test_vlan_ipv4_udp(void) +{ + odp_packet_t pkt; + + pkt = loopback_packet(pktio_a, pktio_b, test_packet_vlan_ipv4_udp, + sizeof(test_packet_vlan_ipv4_udp)); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + CU_ASSERT(odp_packet_has_eth(pkt)); + CU_ASSERT(odp_packet_has_vlan(pkt)); + CU_ASSERT(odp_packet_has_ipv4(pkt)); + CU_ASSERT(odp_packet_has_udp(pkt)); + + CU_ASSERT(!odp_packet_has_ipv6(pkt)); + CU_ASSERT(!odp_packet_has_tcp(pkt)); + + odp_packet_free(pkt); +} + +void parser_test_vlan_qinq_ipv4_udp(void) +{ + odp_packet_t pkt; + + pkt = loopback_packet(pktio_a, pktio_b, test_packet_vlan_qinq_ipv4_udp, + sizeof(test_packet_vlan_qinq_ipv4_udp)); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + CU_ASSERT(odp_packet_has_eth(pkt)); + CU_ASSERT(odp_packet_has_vlan(pkt)); + CU_ASSERT(odp_packet_has_vlan_qinq(pkt)); + CU_ASSERT(odp_packet_has_ipv4(pkt)); + CU_ASSERT(odp_packet_has_udp(pkt)); + + CU_ASSERT(!odp_packet_has_ipv6(pkt)); + CU_ASSERT(!odp_packet_has_tcp(pkt)); + + odp_packet_free(pkt); +} + +void parser_test_ipv6_icmp(void) +{ + odp_packet_t pkt; + + pkt = loopback_packet(pktio_a, pktio_b, test_packet_ipv6_icmp, + sizeof(test_packet_ipv6_icmp)); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + CU_ASSERT(odp_packet_has_eth(pkt)); + CU_ASSERT(odp_packet_has_ipv6(pkt)); + CU_ASSERT(odp_packet_has_icmp(pkt)); + + CU_ASSERT(!odp_packet_has_ipv4(pkt)); + CU_ASSERT(!odp_packet_has_tcp(pkt)); + CU_ASSERT(!odp_packet_has_udp(pkt)); + + odp_packet_free(pkt); +} + +void parser_test_ipv6_tcp(void) +{ + odp_packet_t pkt; + + pkt = loopback_packet(pktio_a, pktio_b, test_packet_ipv6_tcp, + sizeof(test_packet_ipv6_tcp)); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + CU_ASSERT(odp_packet_has_eth(pkt)); + CU_ASSERT(odp_packet_has_ipv6(pkt)); + CU_ASSERT(odp_packet_has_tcp(pkt)); + + CU_ASSERT(!odp_packet_has_ipv4(pkt)); + CU_ASSERT(!odp_packet_has_udp(pkt)); + + odp_packet_free(pkt); +} + +void parser_test_ipv6_udp(void) +{ + odp_packet_t pkt; + + pkt = loopback_packet(pktio_a, pktio_b, test_packet_ipv6_udp, + sizeof(test_packet_ipv6_udp)); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + CU_ASSERT(odp_packet_has_eth(pkt)); + CU_ASSERT(odp_packet_has_ipv6(pkt)); + CU_ASSERT(odp_packet_has_udp(pkt)); + + CU_ASSERT(!odp_packet_has_ipv4(pkt)); + CU_ASSERT(!odp_packet_has_tcp(pkt)); + + odp_packet_free(pkt); +} + +void parser_test_vlan_ipv6_udp(void) +{ + odp_packet_t pkt; + + pkt = loopback_packet(pktio_a, pktio_b, test_packet_vlan_ipv6_udp, + sizeof(test_packet_vlan_ipv6_udp)); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID); + CU_ASSERT(odp_packet_has_eth(pkt)); + CU_ASSERT(odp_packet_has_vlan(pkt)); + CU_ASSERT(odp_packet_has_ipv6(pkt)); + CU_ASSERT(odp_packet_has_udp(pkt)); + + CU_ASSERT(!odp_packet_has_ipv4(pkt)); + CU_ASSERT(!odp_packet_has_tcp(pkt)); + + odp_packet_free(pkt); +} + +int parser_suite_init(void) +{ + int i; + + if (getenv("ODP_WAIT_FOR_NETWORK")) + wait_for_network = true; + + iface_name[0] = getenv("ODP_PKTIO_IF0"); + iface_name[1] = getenv("ODP_PKTIO_IF1"); + num_ifaces = 1; + + if (!iface_name[0]) { + printf("No interfaces specified, using default \"loop\".\n"); + iface_name[0] = "loop"; + } else if (!iface_name[1]) { + printf("Using loopback interface: %s\n", iface_name[0]); + } else { + num_ifaces = 2; + printf("Using paired interfaces: %s %s\n", + iface_name[0], iface_name[1]); + } + + if (pkt_pool_create() != 0) { + printf("Error: failed to create parser pool\n"); + return -1; + } + + /* Create pktios and associate input/output queues */ + for (i = 0; i < num_ifaces; ++i) { + pktio_info_t *io; + + io = &pktios[i]; + io->name = iface_name[i]; + io->hdl = create_pktio(i, parser_pool); + if (io->hdl == ODP_PKTIO_INVALID) { + printf("Error: failed to open iface"); + return -1; + } + + if (odp_pktout_queue(io->hdl, &io->pktout, 1) != 1) { + printf("Error: failed to start iface: %s\n", io->name); + return -1; + } + + if (odp_pktin_queue(io->hdl, &io->pktin, 1) != 1) { + printf("Error: failed to start iface: %s\n", io->name); + return -1; + } + + if (odp_pktio_start(io->hdl)) { + printf("Error: failed to start iface: %s\n", io->name); + return -1; + } + + wait_linkup(io->hdl); + } + + pktio_a = &pktios[0]; + pktio_b = &pktios[1]; + if (num_ifaces == 1) + pktio_b = pktio_a; + + return 0; +} + +int parser_suite_term(void) +{ + int i; + int ret = 0; + + for (i = 0; i < num_ifaces; ++i) { + if (odp_pktio_stop(pktios[i].hdl)) { + printf("Error: failed to stop pktio: %s\n", + pktios[i].name); + ret = -1; + } + if (odp_pktio_close(pktios[i].hdl)) { + printf("Error: failed to close pktio: %s\n", + pktios[i].name); + ret = -1; + } + } + + if (odp_pool_destroy(parser_pool) != 0) { + printf("Error: failed to destroy packet pool\n"); + ret = -1; + } + + return ret; +} + +/** + * Certain tests can only be run with 'loop' pktio. + */ +static int loop_pktio(void) +{ + if (strcmp(iface_name[0], "loop") == 0) + return ODP_TEST_ACTIVE; + else + return ODP_TEST_INACTIVE; +} + +odp_testinfo_t parser_suite[] = { + ODP_TEST_INFO(parser_test_arp), + ODP_TEST_INFO(parser_test_ipv4_icmp), + ODP_TEST_INFO(parser_test_ipv4_tcp), + ODP_TEST_INFO(parser_test_ipv4_udp), + ODP_TEST_INFO_CONDITIONAL(parser_test_vlan_ipv4_udp, loop_pktio), + ODP_TEST_INFO_CONDITIONAL(parser_test_vlan_qinq_ipv4_udp, loop_pktio), + ODP_TEST_INFO(parser_test_ipv6_icmp), + ODP_TEST_INFO(parser_test_ipv6_tcp), + ODP_TEST_INFO(parser_test_ipv6_udp), + ODP_TEST_INFO_CONDITIONAL(parser_test_vlan_ipv6_udp, loop_pktio), + ODP_TEST_INFO_NULL +}; diff --git a/test/common_plat/validation/api/pktio/parser.h b/test/common_plat/validation/api/pktio/parser.h new file mode 100644 index 000000000..5cc2b988c --- /dev/null +++ b/test/common_plat/validation/api/pktio/parser.h @@ -0,0 +1,180 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _ODP_TEST_PARSER_H_ +#define _ODP_TEST_PARSER_H_ + +#include <odp_cunit_common.h> + +/* test functions: */ +void parser_test_arp(void); +void parser_test_ipv4_icmp(void); +void parser_test_ipv4_tcp(void); +void parser_test_ipv4_udp(void); +void parser_test_vlan_ipv4_udp(void); +void parser_test_vlan_qinq_ipv4_udp(void); +void parser_test_ipv6_icmp(void); +void parser_test_ipv6_tcp(void); +void parser_test_ipv6_udp(void); +void parser_test_vlan_ipv6_udp(void); + +/* test array init/term functions: */ +int parser_suite_term(void); +int parser_suite_init(void); + +/* test arrays: */ +extern odp_testinfo_t parser_suite[]; + +/* Test packets without CRC */ + +/** + * ARP request + */ +static const uint8_t test_packet_arp[] = { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, + 0x09, 0x00, 0x04, 0x00, 0x08, 0x06, 0x00, 0x01, + 0x08, 0x00, 0x06, 0x04, 0x00, 0x01, 0x00, 0x00, + 0x09, 0x00, 0x04, 0x00, 0xC0, 0xA8, 0x01, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC0, 0xA8, + 0x01, 0x02, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, + 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, + 0x0E, 0x0F, 0x10, 0x11 +}; + +/** + * ICMPv4 echo reply + */ +static const uint8_t test_packet_ipv4_icmp[] = { + 0x00, 0x00, 0x09, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x09, 0x00, 0x04, 0x00, 0x08, 0x00, 0x45, 0x00, + 0x00, 0x2E, 0x00, 0x00, 0x00, 0x00, 0x40, 0x01, + 0xF3, 0x7B, 0xC0, 0xA8, 0x01, 0x01, 0xC4, 0xA8, + 0x01, 0x02, 0x00, 0x00, 0xB7, 0xAB, 0x00, 0x01, + 0x00, 0x02, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, + 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, + 0x0E, 0x0F, 0x10, 0x11 +}; + +/** + * IPv4 TCP + */ +static const uint8_t test_packet_ipv4_tcp[] = { + 0x00, 0x00, 0x09, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x09, 0x00, 0x04, 0x00, 0x08, 0x00, 0x45, 0x00, + 0x00, 0x2E, 0x00, 0x00, 0x00, 0x00, 0x40, 0x06, + 0xF3, 0x76, 0xC0, 0xA8, 0x01, 0x02, 0xC4, 0xA8, + 0x01, 0x01, 0x04, 0xD2, 0x10, 0xE1, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x50, 0x00, + 0x00, 0x00, 0x0C, 0xCC, 0x00, 0x00, 0x00, 0x01, + 0x02, 0x03, 0x04, 0x05 +}; + +/** + * IPv4 UDP + */ +static const uint8_t test_packet_ipv4_udp[] = { + 0x00, 0x00, 0x09, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x09, 0x00, 0x04, 0x00, 0x08, 0x00, 0x45, 0x00, + 0x00, 0x2E, 0x00, 0x00, 0x00, 0x00, 0x40, 0x11, + 0xF3, 0x6B, 0xC0, 0xA8, 0x01, 0x02, 0xC4, 0xA8, + 0x01, 0x01, 0x00, 0x3F, 0x00, 0x3F, 0x00, 0x1A, + 0x2F, 0x97, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, + 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, + 0x0E, 0x0F, 0x10, 0x11 +}; + +/** + * VLAN IPv4 UDP + * - ID: 23 + */ +static const uint8_t test_packet_vlan_ipv4_udp[] = { + 0x00, 0x00, 0x09, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x09, 0x00, 0x04, 0x00, 0x81, 0x00, 0x00, 0x17, + 0x08, 0x00, 0x45, 0x00, 0x00, 0x2A, 0x00, 0x00, + 0x00, 0x00, 0x40, 0x11, 0xF3, 0x6F, 0xC0, 0xA8, + 0x01, 0x02, 0xC4, 0xA8, 0x01, 0x01, 0x00, 0x3F, + 0x00, 0x3F, 0x00, 0x16, 0x4D, 0xBF, 0x00, 0x01, + 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, + 0x0A, 0x0B, 0x0C, 0x0D +}; + +/** + * VLAN Q-in-Q IPv4 UDP + * - Outer: Tag Protocol ID 0x88a8, VLAN ID 1 + * - Inner: Tag Protocol ID 0x8100, VLAN ID 2 + */ +static const uint8_t test_packet_vlan_qinq_ipv4_udp[] = { + 0x00, 0x00, 0x09, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x09, 0x00, 0x04, 0x00, 0x88, 0xA8, 0x00, 0x01, + 0x81, 0x00, 0x00, 0x02, 0x08, 0x00, 0x45, 0x00, + 0x00, 0x26, 0x00, 0x00, 0x00, 0x00, 0x40, 0x11, + 0xF3, 0x73, 0xC0, 0xA8, 0x01, 0x02, 0xC4, 0xA8, + 0x01, 0x01, 0x00, 0x3F, 0x00, 0x3F, 0x00, 0x12, + 0x63, 0xDF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, + 0x06, 0x07, 0x08, 0x09 +}; + +/** + * ICMPv6 echo request + */ +static const uint8_t test_packet_ipv6_icmp[] = { + 0x00, 0x00, 0x09, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x09, 0x00, 0x04, 0x00, 0x86, 0xDD, 0x60, 0x30, + 0x00, 0x00, 0x00, 0x08, 0x3A, 0xFF, 0xFE, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, + 0x09, 0xFF, 0xFE, 0x00, 0x04, 0x00, 0x35, 0x55, + 0x55, 0x55, 0x66, 0x66, 0x66, 0x66, 0x77, 0x77, + 0x77, 0x77, 0x88, 0x88, 0x88, 0x88, 0x80, 0x00, + 0x1B, 0xC2, 0x00, 0x01, 0x00, 0x02 +}; + +/** + * IPv6 TCP + */ +static const uint8_t test_packet_ipv6_tcp[] = { + 0x00, 0x00, 0x09, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x09, 0x00, 0x04, 0x00, 0x86, 0xDD, 0x60, 0x30, + 0x00, 0x00, 0x00, 0x14, 0x06, 0xFF, 0xFE, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, + 0x09, 0xFF, 0xFE, 0x00, 0x04, 0x00, 0x35, 0x55, + 0x55, 0x55, 0x66, 0x66, 0x66, 0x66, 0x77, 0x77, + 0x77, 0x77, 0x88, 0x88, 0x88, 0x88, 0x04, 0xD2, + 0x10, 0xE1, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x02, 0x50, 0x00, 0x00, 0x00, 0x36, 0x37, + 0x00, 0x00 +}; + +/** + * IPv6 UDP + */ +static const uint8_t test_packet_ipv6_udp[] = { + 0x00, 0x00, 0x09, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x09, 0x00, 0x04, 0x00, 0x86, 0xDD, 0x60, 0x30, + 0x00, 0x00, 0x00, 0x08, 0x11, 0xFF, 0xFE, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, + 0x09, 0xFF, 0xFE, 0x00, 0x04, 0x00, 0x35, 0x55, + 0x55, 0x55, 0x66, 0x66, 0x66, 0x66, 0x77, 0x77, + 0x77, 0x77, 0x88, 0x88, 0x88, 0x88, 0x00, 0x3F, + 0x00, 0x3F, 0x00, 0x08, 0x9B, 0x68 +}; + +/** + * VLAN IPv6 + * - ID: 23 + */ +static const uint8_t test_packet_vlan_ipv6_udp[] = { + 0x00, 0x00, 0x09, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x09, 0x00, 0x04, 0x00, 0x81, 0x00, 0x00, 0x17, + 0x86, 0xDD, 0x60, 0x30, 0x00, 0x00, 0x00, 0x08, + 0x11, 0xFF, 0xFE, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x02, 0x00, 0x09, 0xFF, 0xFE, 0x00, + 0x04, 0x00, 0x35, 0x55, 0x55, 0x55, 0x66, 0x66, + 0x66, 0x66, 0x77, 0x77, 0x77, 0x77, 0x88, 0x88, + 0x88, 0x88, 0x00, 0x3F, 0x00, 0x3F, 0x00, 0x08, + 0x9B, 0x68 +}; + +#endif diff --git a/test/common_plat/validation/api/pktio/pktio.c b/test/common_plat/validation/api/pktio/pktio.c index 54f206efc..c476a7126 100644 --- a/test/common_plat/validation/api/pktio/pktio.c +++ b/test/common_plat/validation/api/pktio/pktio.c @@ -10,6 +10,7 @@ #include <stdlib.h> #include "pktio.h" +#include "parser.h" #define PKT_BUF_NUM 32 #define PKT_BUF_SIZE (9 * 1024) @@ -122,26 +123,28 @@ static inline void _pktio_wait_linkup(odp_pktio_t pktio) static void set_pool_len(odp_pool_param_t *params, odp_pool_capability_t *capa) { + uint32_t len; uint32_t seg_len; + len = (capa->pkt.max_len && capa->pkt.max_len < PKT_BUF_SIZE) ? + capa->pkt.max_len : PKT_BUF_SIZE; seg_len = capa->pkt.max_seg_len ? capa->pkt.max_seg_len : PKT_BUF_SIZE; switch (pool_segmentation) { case PKT_POOL_SEGMENTED: /* Force segment to minimum size */ params->pkt.seg_len = 0; - params->pkt.len = PKT_BUF_SIZE; + params->pkt.len = len; break; case PKT_POOL_UNSEGMENTED: default: params->pkt.seg_len = seg_len; - params->pkt.len = PKT_BUF_SIZE; + params->pkt.len = len; break; } } -static void pktio_pkt_set_macs(odp_packet_t pkt, - odp_pktio_t src, odp_pktio_t dst) +void pktio_pkt_set_macs(odp_packet_t pkt, odp_pktio_t src, odp_pktio_t dst) { uint32_t len; odph_ethhdr_t *eth = (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, &len); @@ -1179,6 +1182,8 @@ void pktio_test_pktio_config(void) odp_pktio_config_init(&config); + CU_ASSERT(config.parser.layer == ODP_PKTIO_PARSER_LAYER_ALL); + CU_ASSERT(odp_pktio_config(pktio, NULL) == 0); CU_ASSERT(odp_pktio_config(pktio, &config) == 0); @@ -1426,7 +1431,9 @@ int pktio_check_statistics_counters(void) void pktio_test_statistics_counters(void) { odp_pktio_t pktio_rx, pktio_tx; - odp_pktio_t pktio[MAX_NUM_IFACES]; + odp_pktio_t pktio[MAX_NUM_IFACES] = { + ODP_PKTIO_INVALID, ODP_PKTIO_INVALID + }; odp_packet_t pkt; odp_packet_t tx_pkt[1000]; uint32_t pkt_seq[1000]; @@ -2185,6 +2192,7 @@ odp_suiteinfo_t pktio_suites[] = { pktio_suite_term, pktio_suite_unsegmented}, {"Packet I/O Segmented", pktio_suite_init_segmented, pktio_suite_term, pktio_suite_segmented}, + {"Packet parser", parser_suite_init, parser_suite_term, parser_suite}, ODP_SUITE_INFO_NULL }; diff --git a/test/common_plat/validation/api/pktio/pktio.h b/test/common_plat/validation/api/pktio/pktio.h index 8131d05fe..b8799d9eb 100644 --- a/test/common_plat/validation/api/pktio/pktio.h +++ b/test/common_plat/validation/api/pktio/pktio.h @@ -61,4 +61,7 @@ extern odp_suiteinfo_t pktio_suites[]; /* main test program: */ int pktio_main(int argc, char *argv[]); +/* functions shared by parser test suite */ +void pktio_pkt_set_macs(odp_packet_t pkt, odp_pktio_t src, odp_pktio_t dst); + #endif diff --git a/test/common_plat/validation/api/queue/queue.c b/test/common_plat/validation/api/queue/queue.c index 1f7913a12..6a13c0062 100644 --- a/test/common_plat/validation/api/queue/queue.c +++ b/test/common_plat/validation/api/queue/queue.c @@ -56,7 +56,7 @@ void queue_test_capa(void) odp_queue_param_t qparams; char name[ODP_QUEUE_NAME_LEN]; odp_queue_t queue[MAX_QUEUES]; - uint32_t num_queues, i; + uint32_t num_queues, min, i, j; memset(&capa, 0, sizeof(odp_queue_capability_t)); CU_ASSERT(odp_queue_capability(&capa) == 0); @@ -65,34 +65,49 @@ void queue_test_capa(void) CU_ASSERT(capa.max_ordered_locks != 0); CU_ASSERT(capa.max_sched_groups != 0); CU_ASSERT(capa.sched_prios != 0); + CU_ASSERT(capa.plain.max_num != 0); + CU_ASSERT(capa.sched.max_num != 0); + + min = capa.plain.max_num; + if (min > capa.sched.max_num) + min = capa.sched.max_num; + + CU_ASSERT(capa.max_queues >= min); for (i = 0; i < ODP_QUEUE_NAME_LEN; i++) name[i] = 'A' + (i % 26); name[ODP_QUEUE_NAME_LEN - 1] = 0; - if (capa.max_queues > MAX_QUEUES) - num_queues = MAX_QUEUES; - else - num_queues = capa.max_queues; - odp_queue_param_init(&qparams); - for (i = 0; i < num_queues; i++) { - generate_name(name, i); - queue[i] = odp_queue_create(name, &qparams); + for (j = 0; j < 2; j++) { + if (j == 0) { + num_queues = capa.plain.max_num; + } else { + num_queues = capa.sched.max_num; + qparams.type = ODP_QUEUE_TYPE_SCHED; + } + + if (num_queues > MAX_QUEUES) + num_queues = MAX_QUEUES; - if (queue[i] == ODP_QUEUE_INVALID) { - CU_FAIL("Queue create failed"); - num_queues = i; - break; + for (i = 0; i < num_queues; i++) { + generate_name(name, i); + queue[i] = odp_queue_create(name, &qparams); + + if (queue[i] == ODP_QUEUE_INVALID) { + CU_FAIL("Queue create failed"); + num_queues = i; + break; + } + + CU_ASSERT(odp_queue_lookup(name) != ODP_QUEUE_INVALID); } - CU_ASSERT(odp_queue_lookup(name) != ODP_QUEUE_INVALID); + for (i = 0; i < num_queues; i++) + CU_ASSERT(odp_queue_destroy(queue[i]) == 0); } - - for (i = 0; i < num_queues; i++) - CU_ASSERT(odp_queue_destroy(queue[i]) == 0); } void queue_test_mode(void) diff --git a/test/common_plat/validation/api/scheduler/scheduler.c b/test/common_plat/validation/api/scheduler/scheduler.c index 952561cd3..4f99435c9 100644 --- a/test/common_plat/validation/api/scheduler/scheduler.c +++ b/test/common_plat/validation/api/scheduler/scheduler.c @@ -129,6 +129,14 @@ static int exit_schedule_loop(void) return ret; } +static void release_context(odp_schedule_sync_t sync) +{ + if (sync == ODP_SCHED_SYNC_ATOMIC) + odp_schedule_release_atomic(); + else if (sync == ODP_SCHED_SYNC_ORDERED) + odp_schedule_release_ordered(); +} + void scheduler_test_wait_time(void) { int i; @@ -251,8 +259,7 @@ void scheduler_test_queue_destroy(void) CU_ASSERT_FATAL(u32[0] == MAGIC); odp_buffer_free(buf); - odp_schedule_release_ordered(); - + release_context(qp.sched.sync); CU_ASSERT_FATAL(odp_queue_destroy(queue) == 0); } @@ -820,12 +827,7 @@ static int schedule_common_(void *arg) } } - if (sync == ODP_SCHED_SYNC_ATOMIC) - odp_schedule_release_atomic(); - - if (sync == ODP_SCHED_SYNC_ORDERED) - odp_schedule_release_ordered(); - + release_context(sync); odp_ticketlock_lock(&globals->lock); globals->buf_count -= num; @@ -1388,6 +1390,7 @@ static int create_queues(void) snprintf(name, sizeof(name), "sched_%d_%d_a", i, j); p.sched.sync = ODP_SCHED_SYNC_ATOMIC; + p.size = BUFS_PER_QUEUE_EXCL; q = odp_queue_create(name, &p); if (q == ODP_QUEUE_INVALID) { @@ -1423,6 +1426,7 @@ static int create_queues(void) snprintf(name, sizeof(name), "sched_%d_%d_o", i, j); p.sched.sync = ODP_SCHED_SYNC_ORDERED; p.sched.lock_count = capa.max_ordered_locks; + p.size = 0; q = odp_queue_create(name, &p); if (q == ODP_QUEUE_INVALID) { diff --git a/test/common_plat/validation/api/system/system.c b/test/common_plat/validation/api/system/system.c index 57ff34eb9..5b7ca01ae 100644 --- a/test/common_plat/validation/api/system/system.c +++ b/test/common_plat/validation/api/system/system.c @@ -301,6 +301,13 @@ void system_test_odp_cpu_hz_max_id(void) } } +void system_test_info_print(void) +{ + printf("\n\nCalling system info print...\n"); + odp_sys_info_print(); + printf("...done. "); +} + odp_testinfo_t system_suite[] = { ODP_TEST_INFO(system_test_odp_version_numbers), ODP_TEST_INFO(system_test_odp_cpu_count), @@ -319,6 +326,7 @@ odp_testinfo_t system_suite[] = { ODP_TEST_INFO(system_test_odp_cpu_cycles_max), ODP_TEST_INFO(system_test_odp_cpu_cycles_resolution), ODP_TEST_INFO(system_test_odp_cpu_cycles_diff), + ODP_TEST_INFO(system_test_info_print), ODP_TEST_INFO_NULL, }; diff --git a/test/common_plat/validation/api/system/system.h b/test/common_plat/validation/api/system/system.h index cbb994eb0..c33729b94 100644 --- a/test/common_plat/validation/api/system/system.h +++ b/test/common_plat/validation/api/system/system.h @@ -30,6 +30,7 @@ void system_test_odp_cpu_cycles_max(void); void system_test_odp_cpu_cycles(void); void system_test_odp_cpu_cycles_diff(void); void system_test_odp_cpu_cycles_resolution(void); +void system_test_info_print(void); /* test arrays: */ extern odp_testinfo_t system_suite[]; diff --git a/test/common_plat/validation/api/time/Makefile.am b/test/common_plat/validation/api/time/Makefile.am index bf2d0268c..82682dfc3 100644 --- a/test/common_plat/validation/api/time/Makefile.am +++ b/test/common_plat/validation/api/time/Makefile.am @@ -1,5 +1,12 @@ include ../Makefile.inc +TESTS_ENVIRONMENT += TEST_DIR=${builddir} + +TESTSCRIPTS = time.sh +TEST_EXTENSIONS = .sh + +TESTS = $(TESTSCRIPTS) + noinst_LTLIBRARIES = libtesttime.la libtesttime_la_SOURCES = time.c @@ -7,4 +14,5 @@ test_PROGRAMS = time_main$(EXEEXT) dist_time_main_SOURCES = time_main.c time_main_LDADD = libtesttime.la $(LIBCUNIT_COMMON) $(LIBODP) -EXTRA_DIST = time.h +EXTRA_DIST = time_test.h $(TESTSCRIPTS) +dist_check_SCRIPTS = $(TESTSCRIPTS) diff --git a/test/common_plat/validation/api/time/time.c b/test/common_plat/validation/api/time/time.c index 530d5c07a..e2ca2e17c 100644 --- a/test/common_plat/validation/api/time/time.c +++ b/test/common_plat/validation/api/time/time.c @@ -6,7 +6,8 @@ #include <odp_api.h> #include "odp_cunit_common.h" -#include "time.h" +#include "time_test.h" +#include <time.h> #define BUSY_LOOP_CNT 30000000 /* used for t > min resolution */ #define BUSY_LOOP_CNT_LONG 6000000000 /* used for t > 4 sec */ @@ -140,25 +141,25 @@ void time_test_monotony(void) CU_ASSERT(ns3 > ns2); } -static void time_test_cmp(time_cb time, time_from_ns_cb time_from_ns) +static void time_test_cmp(time_cb time_cur, time_from_ns_cb time_from_ns) { /* volatile to stop optimization of busy loop */ volatile int count = 0; odp_time_t t1, t2, t3; - t1 = time(); + t1 = time_cur(); while (count < BUSY_LOOP_CNT) { count++; }; - t2 = time(); + t2 = time_cur(); while (count < BUSY_LOOP_CNT * 2) { count++; }; - t3 = time(); + t3 = time_cur(); CU_ASSERT(odp_time_cmp(t2, t1) > 0); CU_ASSERT(odp_time_cmp(t3, t2) > 0); @@ -191,7 +192,7 @@ void time_test_global_cmp(void) } /* check that a time difference gives a reasonable result */ -static void time_test_diff(time_cb time, +static void time_test_diff(time_cb time_cur, time_from_ns_cb time_from_ns, uint64_t res) { @@ -202,13 +203,13 @@ static void time_test_diff(time_cb time, uint64_t upper_limit, lower_limit; /* test timestamp diff */ - t1 = time(); + t1 = time_cur(); while (count < BUSY_LOOP_CNT) { count++; }; - t2 = time(); + t2 = time_cur(); CU_ASSERT(odp_time_cmp(t2, t1) > 0); diff = odp_time_diff(t2, t1); @@ -268,7 +269,7 @@ void time_test_global_diff(void) } /* check that a time sum gives a reasonable result */ -static void time_test_sum(time_cb time, +static void time_test_sum(time_cb time_cur, time_from_ns_cb time_from_ns, uint64_t res) { @@ -277,7 +278,7 @@ static void time_test_sum(time_cb time, uint64_t upper_limit, lower_limit; /* sum timestamp and interval */ - t1 = time(); + t1 = time_cur(); ns2 = 103; t2 = time_from_ns(ns2); ns1 = odp_time_to_ns(t1); @@ -319,20 +320,20 @@ void time_test_global_sum(void) time_test_sum(odp_time_global, odp_time_global_from_ns, global_res); } -static void time_test_wait_until(time_cb time, time_from_ns_cb time_from_ns) +static void time_test_wait_until(time_cb time_cur, time_from_ns_cb time_from_ns) { int i; odp_time_t lower_limit, upper_limit; odp_time_t start_time, end_time, wait; odp_time_t second = time_from_ns(ODP_TIME_SEC_IN_NS); - start_time = time(); + start_time = time_cur(); wait = start_time; for (i = 0; i < WAIT_SECONDS; i++) { wait = odp_time_sum(wait, second); odp_time_wait_until(wait); } - end_time = time(); + end_time = time_cur(); wait = odp_time_diff(end_time, start_time); lower_limit = time_from_ns(WAIT_SECONDS * ODP_TIME_SEC_IN_NS - @@ -398,39 +399,43 @@ void time_test_wait_ns(void) } } -static void time_test_to_u64(time_cb time) +static void time_test_accuracy(time_cb time_cur, time_from_ns_cb time_from_ns) { - volatile int count = 0; - uint64_t val1, val2; - odp_time_t t1, t2; - - t1 = time(); + int i; + odp_time_t t1, t2, wait, diff; + clock_t c1, c2; + double sec_t, sec_c; + odp_time_t sec = time_from_ns(ODP_TIME_SEC_IN_NS); - val1 = odp_time_to_u64(t1); - CU_ASSERT(val1 > 0); + c1 = clock(); + t1 = time_cur(); - while (count < BUSY_LOOP_CNT) { - count++; - }; + wait = odp_time_sum(t1, sec); + for (i = 0; i < 5; i++) { + odp_time_wait_until(wait); + wait = odp_time_sum(wait, sec); + } - t2 = time(); - val2 = odp_time_to_u64(t2); - CU_ASSERT(val2 > 0); + t2 = time_cur(); + c2 = clock(); - CU_ASSERT(val2 > val1); + diff = odp_time_diff(t2, t1); + sec_t = ((double)odp_time_to_ns(diff)) / ODP_TIME_SEC_IN_NS; + sec_c = ((double)(c2 - c1)) / CLOCKS_PER_SEC; - val1 = odp_time_to_u64(ODP_TIME_NULL); - CU_ASSERT(val1 == 0); + /* Check that ODP time is within +-5% of system time */ + CU_ASSERT(sec_t < sec_c * 1.05); + CU_ASSERT(sec_t > sec_c * 0.95); } -void time_test_local_to_u64(void) +static void time_test_local_accuracy(void) { - time_test_to_u64(odp_time_local); + time_test_accuracy(odp_time_local, odp_time_local_from_ns); } -void time_test_global_to_u64(void) +static void time_test_global_accuracy(void) { - time_test_to_u64(odp_time_global); + time_test_accuracy(odp_time_global, odp_time_global_from_ns); } odp_testinfo_t time_suite_time[] = { @@ -443,14 +448,14 @@ odp_testinfo_t time_suite_time[] = { ODP_TEST_INFO(time_test_local_sum), ODP_TEST_INFO(time_test_local_wait_until), ODP_TEST_INFO(time_test_wait_ns), - ODP_TEST_INFO(time_test_local_to_u64), + ODP_TEST_INFO(time_test_local_accuracy), ODP_TEST_INFO(time_test_global_res), ODP_TEST_INFO(time_test_global_conversion), ODP_TEST_INFO(time_test_global_cmp), ODP_TEST_INFO(time_test_global_diff), ODP_TEST_INFO(time_test_global_sum), ODP_TEST_INFO(time_test_global_wait_until), - ODP_TEST_INFO(time_test_global_to_u64), + ODP_TEST_INFO(time_test_global_accuracy), ODP_TEST_INFO_NULL }; diff --git a/test/common_plat/validation/api/time/time.sh b/test/common_plat/validation/api/time/time.sh new file mode 100755 index 000000000..02bf75a76 --- /dev/null +++ b/test/common_plat/validation/api/time/time.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# +# Copyright (c) 2017, Linaro Limited +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +# directories where time_main binary can be found: +# -in the validation dir when running make check (intree or out of tree) +# -in the script directory, when running after 'make install', or +# -in the validation when running standalone (./time) intree. +# -in the current directory. +# running stand alone out of tree requires setting PATH +PATH=${TEST_DIR}/api/time:$PATH +PATH=$(dirname $0)/../../../../common_plat/validation/api/time:$PATH +PATH=$(dirname $0):$PATH +PATH=`pwd`:$PATH + +time_main_path=$(which time_main${EXEEXT}) +if [ -x "$time_main_path" ] ; then + echo "running with time_main: $time_run_path" +else + echo "cannot find time_main: please set you PATH for it." + exit 1 +fi + +# exit codes expected by automake for skipped tests +TEST_SKIPPED=77 + +time_main${EXEEXT} +ret=$? + +SIGSEGV=139 + +if [ "${TRAVIS}" = "true" ] && [ $ret -ne 0 ] && + [ ${TEST} = "coverage" ] && [ $ret -ne ${SIGSEGV} ]; then + echo "SKIP: skip due significant slowdown under code coverage" + exit ${TEST_SKIPPED} +fi + +exit $ret diff --git a/test/common_plat/validation/api/time/time_main.c b/test/common_plat/validation/api/time/time_main.c index f86d638a5..bf1cfe7bd 100644 --- a/test/common_plat/validation/api/time/time_main.c +++ b/test/common_plat/validation/api/time/time_main.c @@ -4,7 +4,7 @@ * SPDX-License-Identifier: BSD-3-Clause */ -#include "time.h" +#include "time_test.h" int main(int argc, char *argv[]) { diff --git a/test/common_plat/validation/api/time/time.h b/test/common_plat/validation/api/time/time_test.h index e5132a494..109562944 100644 --- a/test/common_plat/validation/api/time/time.h +++ b/test/common_plat/validation/api/time/time_test.h @@ -24,8 +24,6 @@ void time_test_global_sum(void); void time_test_local_wait_until(void); void time_test_global_wait_until(void); void time_test_wait_ns(void); -void time_test_local_to_u64(void); -void time_test_global_to_u64(void); void time_test_monotony(void); /* test arrays: */ diff --git a/test/common_plat/validation/api/traffic_mngr/Makefile.am b/test/common_plat/validation/api/traffic_mngr/Makefile.am index 35e689a02..a012c1b3c 100644 --- a/test/common_plat/validation/api/traffic_mngr/Makefile.am +++ b/test/common_plat/validation/api/traffic_mngr/Makefile.am @@ -1,10 +1,18 @@ include ../Makefile.inc +TESTS_ENVIRONMENT += TEST_DIR=${builddir} + +TESTSCRIPTS = traffic_mngr.sh +TEST_EXTENSIONS = .sh + +TESTS = $(TESTSCRIPTS) + noinst_LTLIBRARIES = libtesttraffic_mngr.la libtesttraffic_mngr_la_SOURCES = traffic_mngr.c -test_PROGRAMS = traffic_mngr_main$(EXEEXT) +bin_PROGRAMS = traffic_mngr_main$(EXEEXT) dist_traffic_mngr_main_SOURCES = traffic_mngr_main.c traffic_mngr_main_LDADD = libtesttraffic_mngr.la -lm $(LIBCUNIT_COMMON) $(LIBODP) -EXTRA_DIST = traffic_mngr.h +EXTRA_DIST = traffic_mngr.h $(TESTSCRIPTS) +dist_check_SCRIPTS = $(TESTSCRIPTS) diff --git a/test/common_plat/validation/api/traffic_mngr/traffic_mngr.sh b/test/common_plat/validation/api/traffic_mngr/traffic_mngr.sh new file mode 100755 index 000000000..4db7ea384 --- /dev/null +++ b/test/common_plat/validation/api/traffic_mngr/traffic_mngr.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# +# Copyright (c) 2017, Linaro Limited +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +# directories where traffic_mngr_main binary can be found: +# -in the validation dir when running make check (intree or out of tree) +# -in the script directory, when running after 'make install', or +# -in the validation when running standalone (./traffic_mngr) intree. +# -in the current directory. +# running stand alone out of tree requires setting PATH +PATH=${TEST_DIR}/api/traffic_mngr:$PATH +PATH=$(dirname $0)/../../../../common_plat/validation/api/traffic_mngr:$PATH +PATH=$(dirname $0):$PATH +PATH=`pwd`:$PATH + +traffic_mngr_main_path=$(which traffic_mngr_main${EXEEXT}) +if [ -x "$traffic_mngr_main_path" ] ; then + echo "running with traffic_mngr_main: $traffic_mngr_run_path" +else + echo "cannot find traffic_mngr_main: please set you PATH for it." + exit 1 +fi + +# exit codes expected by automake for skipped tests +TEST_SKIPPED=77 + +traffic_mngr_main${EXEEXT} +ret=$? + +SIGSEGV=139 + +if [ "${TRAVIS}" = "true" ] && [ $ret -ne 0 ] && [ $ret -ne ${SIGSEGV} ]; then + echo "SKIP: skip due to not isolated environment" + exit ${TEST_SKIPPED} +fi + +exit $ret diff --git a/test/linux-generic/Makefile.am b/test/linux-generic/Makefile.am index 998ee5617..f92083d73 100644 --- a/test/linux-generic/Makefile.am +++ b/test/linux-generic/Makefile.am @@ -28,9 +28,9 @@ TESTS = validation/api/pktio/pktio_run.sh \ $(ALL_API_VALIDATION_DIR)/scheduler/scheduler_main$(EXEEXT) \ $(ALL_API_VALIDATION_DIR)/std_clib/std_clib_main$(EXEEXT) \ $(ALL_API_VALIDATION_DIR)/thread/thread_main$(EXEEXT) \ - $(ALL_API_VALIDATION_DIR)/time/time_main$(EXEEXT) \ + $(ALL_API_VALIDATION_DIR)/time/time.sh \ $(ALL_API_VALIDATION_DIR)/timer/timer_main$(EXEEXT) \ - $(ALL_API_VALIDATION_DIR)/traffic_mngr/traffic_mngr_main$(EXEEXT) \ + $(ALL_API_VALIDATION_DIR)/traffic_mngr/traffic_mngr.sh \ $(ALL_API_VALIDATION_DIR)/shmem/shmem_main$(EXEEXT) \ $(ALL_API_VALIDATION_DIR)/system/system_main$(EXEEXT) \ ring/ring_main$(EXEEXT) @@ -53,11 +53,8 @@ endif if PKTIO_DPDK TESTS += validation/api/pktio/pktio_run_dpdk.sh endif - -if PKTIO_IPC TESTS += pktio_ipc/pktio_ipc_run.sh SUBDIRS += pktio_ipc -endif else #performance tests refer to pktio_env if test_perf diff --git a/test/linux-generic/pktio_ipc/pktio_ipc_run.sh b/test/linux-generic/pktio_ipc/pktio_ipc_run.sh index 52e8d42a0..331ecdc98 100755 --- a/test/linux-generic/pktio_ipc/pktio_ipc_run.sh +++ b/test/linux-generic/pktio_ipc/pktio_ipc_run.sh @@ -20,9 +20,6 @@ PATH=.:$PATH run() { local ret=0 - #if test was interrupted with CTRL+c than files - #might remain in shm. Needed cleanely delete them. - rm -rf /tmp/odp-* 2>&1 > /dev/null echo "==== run pktio_ipc1 then pktio_ipc2 ====" pktio_ipc1${EXEEXT} -t 10 & diff --git a/test/linux-generic/validation/api/pktio/pktio_run.sh b/test/linux-generic/validation/api/pktio/pktio_run.sh index e8b0f936f..19def8c5a 100755 --- a/test/linux-generic/validation/api/pktio/pktio_run.sh +++ b/test/linux-generic/validation/api/pktio/pktio_run.sh @@ -31,6 +31,7 @@ if [ -x "$pktio_main_path" ] ; then echo "running with pktio_main: $pktio_run_path" else echo "cannot find pktio_main: please set you PATH for it." + exit 1 fi # directory where platform test sources are, including scripts diff --git a/test/linux-generic/validation/api/pktio/pktio_run_dpdk.sh b/test/linux-generic/validation/api/pktio/pktio_run_dpdk.sh index fa46fa430..3060dc003 100755 --- a/test/linux-generic/validation/api/pktio/pktio_run_dpdk.sh +++ b/test/linux-generic/validation/api/pktio/pktio_run_dpdk.sh @@ -74,7 +74,7 @@ run() if [ "$ODP_PKTIO_IF0" = "" ]; then setup_pktio_env clean - export ODP_PKTIO_DPDK_PARAMS="--vdev eth_pcap0,iface=$IF0 --vdev eth_pcap1,iface=$IF1" + export ODP_PKTIO_DPDK_PARAMS="--no-pci --vdev eth_pcap0,iface=$IF0 --vdev eth_pcap1,iface=$IF1" export ODP_PKTIO_IF0=0 export ODP_PKTIO_IF1=1 fi |