diff options
| author | Matias Elo <matias.elo@nokia.com> | 2021-04-28 15:52:39 +0300 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-04-28 15:52:39 +0300 |
| commit | 029cb31ef951664c12cfe61563a25f3ff6449e5a (patch) | |
| tree | bb12bb9dcd04e8a1176a782b50a16a085c03963f | |
| parent | c3e252739b6c5812d11ac07ec7df687d927de36e (diff) | |
| parent | 20eb6b4655671fbfb6fd41c7407762cd30f1cfc5 (diff) | |
Merge ODP v1.27.0.0v1.27.0.0_DPDK_19.11
Merge ODP linux-generic v1.27.0.0 into ODP-DPDK.
88 files changed, 4774 insertions, 1185 deletions
diff --git a/.checkpatch.conf b/.checkpatch.conf index 7c179e3e8..fa31c1edf 100644 --- a/.checkpatch.conf +++ b/.checkpatch.conf @@ -11,6 +11,7 @@ --ignore=VOLATILE --ignore=AVOID_EXTERNS --ignore=CONST_STRUCT +--ignore=PREFER_ARRAY_SIZE --ignore=PREFER_KERNEL_TYPES --ignore=CONSTANT_COMPARISON --ignore=BLOCK_COMMENT_STYLE diff --git a/.codecov.yml b/.codecov.yml index 4b3a723c9..d5eddbf7d 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -1,21 +1,19 @@ codecov: notify: - require_ci_to_pass: yes + require_ci_to_pass: no + wait_for_ci: no coverage: - precision: 3 + precision: 2 round: down - range: "50...75" + range: "70...100" status: project: default: - enabled: yes target: 70% threshold: 5% - patch: - default: - enabled: no - changes: no + patch: off + changes: off parsers: gcov: @@ -26,3 +24,6 @@ parsers: macro: no comment: false + +ignore: + - "platform/linux-generic/miniz" diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 8ce63208d..bb7ef6e2e 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -186,7 +186,7 @@ jobs: if: ${{ failure() }} run: find . -name "*.trs" | xargs grep -l '^.test-result. FAIL' | while read trs ; do echo FAILURE detected at $trs; cat ${trs%%.trs}.log ; done - name: Upload to Codecov - uses: codecov/codecov-action@v1.0.15 + uses: codecov/codecov-action@v1.2.1 Run_distcheck: runs-on: ubuntu-18.04 diff --git a/.gitignore b/.gitignore index 772c551f4..0a58c0ee9 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,8 @@ include/odp/autoheader_build.h include/odp/autoheader_external.h include/odp/autoheader_internal.h include/odp/stamp-h* +helper/include/odp/helper/autoheader_external.h +helper/include/odp/helper/stamp-h* lib/ libtool ltmain.sh diff --git a/DEPENDENCIES b/DEPENDENCIES index 30745c1ef..ba63313cb 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -5,6 +5,14 @@ Prerequisites for building the OpenDataPlane (ODP) API Linux distributions tested by the ODP CI. Earlier versions may or may not work. + For CentOS/RedHat distros, configure the system to use Fedora EPEL repos and + third-party packages: + $ sudo yum install epel-release + + Additionally, for CentOS 8 distros, enable the powertools repository: + $ sudo yum install dnf-plugins-core + $ sudo yum config-manager --set-enabled powertools + 2. autotools automake @@ -23,10 +31,10 @@ Prerequisites for building the OpenDataPlane (ODP) API Libraries currently required to link: libconfig, openssl, libatomic On Debian/Ubuntu systems: - $ sudo apt-get install libconfig-dev + $ sudo apt-get install libconfig-dev libatomic On CentOS/RedHat/Fedora systems: - $ sudo yum install libconfig-devel + $ sudo yum install libconfig-devel libatomic It is possible to build ODP without OpenSSL by passing flag --without-openssl to configure script. However this will result in @@ -183,6 +191,7 @@ Prerequisites for building the OpenDataPlane (ODP) API DPDK pktio adds a dependency to NUMA library. # Debian/Ubuntu $ sudo apt-get install libnuma-dev + # CentOS/RedHat/Fedora $ sudo yum install numactl-devel @@ -245,7 +254,10 @@ Prerequisites for building the OpenDataPlane (ODP) API 4.1 Native CUnit install # Debian/Ubuntu - $ apt-get install libcunit1-dev + $ sudo apt-get install libcunit1-dev + + # CentOS/RedHat/Fedora systems + $ sudo yum install CUnit-devel 4.2 Built from src @@ -341,7 +353,10 @@ Prerequisites for building the OpenDataPlane (ODP) API Message sequence diagrams are stored as msc files and the svg versions are generated when the docs are built. # Debian/Ubuntu - $ apt-get install mscgen + $ sudo apt-get install mscgen + + # CentOS 8/RedHat/Fedora + $ sudo yum install mscgen 6.1 API Guide See https://www.doxygen.nl/manual/install.html @@ -350,13 +365,19 @@ Prerequisites for building the OpenDataPlane (ODP) API 6.1.1 HTML # Debian/Ubuntu - $ apt-get install doxygen graphviz + $ sudo apt-get install doxygen graphviz + + # CentOS/RedHat/Fedora + $ sudo yum install doxygen graphviz 6.2 User guides 6.2.1 HTML # Debian/Ubuntu - $ apt-get install asciidoctor source-highlight librsvg2-bin + $ sudo apt-get install asciidoctor source-highlight librsvg2-bin + + # CentOS/RedHat/Fedora + $ sudo yum install asciidoc source-highlight librsvg2 7.0 Submitting patches diff --git a/configure.ac b/configure.ac index ae5759ad0..1987c4c2c 100644 --- a/configure.ac +++ b/configure.ac @@ -3,7 +3,7 @@ AC_PREREQ([2.5]) # ODP API version ########################################################################## m4_define([odpapi_generation_version], [1]) -m4_define([odpapi_major_version], [26]) +m4_define([odpapi_major_version], [27]) m4_define([odpapi_minor_version], [0]) m4_define([odpapi_point_version], [0]) m4_define([odpapi_version], @@ -22,7 +22,7 @@ AC_SUBST(ODP_VERSION_API_MINOR) ########################################################################## m4_define([odph_version_generation], [1]) m4_define([odph_version_major], [0]) -m4_define([odph_version_minor], [5]) +m4_define([odph_version_minor], [6]) m4_define([odph_version], [odph_version_generation.odph_version_major.odph_version_minor]) @@ -43,7 +43,7 @@ AS_IF([test "$ac_cv_env_CFLAGS_set" = ""], [user_cflags=0], [user_cflags=1]) # Initialize automake AM_INIT_AUTOMAKE([1.9 tar-pax subdir-objects foreign nostdinc -Wall -Werror]) AC_CONFIG_SRCDIR([include/odp/api/spec/init.h]) -AM_CONFIG_HEADER([include/odp/autoheader_build.h include/odp/autoheader_external.h include/odp/autoheader_internal.h]) +AM_CONFIG_HEADER([include/odp/autoheader_build.h include/odp/autoheader_external.h include/odp/autoheader_internal.h helper/include/odp/helper/autoheader_external.h]) AC_USE_SYSTEM_EXTENSIONS AC_SYS_LARGEFILE @@ -332,6 +332,7 @@ AM_CONDITIONAL([HAVE_DOXYGEN], [test "x${DOXYGEN}" = "xdoxygen"]) AM_CONDITIONAL([user_guide], [test "x${user_guides}" = "xyes" ]) AM_CONDITIONAL([HAVE_MSCGEN], [test "x${MSCGEN}" = "xmscgen"]) AM_CONDITIONAL([helper_linux], [test x$helper_linux = xyes ]) +AM_CONDITIONAL([helper_cli], [test x$helper_cli = xyes ]) AM_CONDITIONAL([ARCH_IS_ARM], [test "x${ARCH_DIR}" = "xarm"]) AM_CONDITIONAL([ARCH_IS_AARCH64], [test "x${ARCH_DIR}" = "xaarch64"]) AM_CONDITIONAL([ARCH_IS_DEFAULT], [test "x${ARCH_DIR}" = "xdefault"]) @@ -356,15 +357,6 @@ AS_IF([test "x$enable_debug" != "xno"], [ODP_DEBUG=1], AC_DEFINE_UNQUOTED([ODP_DEBUG], [$ODP_DEBUG], [Define to 1 to include additional debug code]) -AC_ARG_ENABLE([helper-debug], - [AS_HELP_STRING([--enable-helper-debug], - [helpers include additional debugging code [default=disabled]])], - [], [AS_IF([test "x$enable_debug" = "xfull"], [enable_helper_debug=yes], - [enable_helper_debug=no])]) -AS_IF([test "x$enable_helper_debug" != "xno"], [ODPH_DEBUG=1], [ODPH_DEBUG=0]) -AC_DEFINE_UNQUOTED([ODPH_DEBUG], [$ODPH_DEBUG], - [Define to 1 to include additional helper debug code]) - ########################################################################## # Enable/disable ODP_DEBUG_PRINT ########################################################################## @@ -378,16 +370,6 @@ AS_IF([test "x$enable_debug_print" != "xno"], [ODP_DEBUG_PRINT=1], AC_DEFINE_UNQUOTED([ODP_DEBUG_PRINT], [$ODP_DEBUG_PRINT], [Define to 1 to display debug information]) -AC_ARG_ENABLE([helper-debug-print], - [AS_HELP_STRING([--enable-helper-debug-print], - [display helper debugging information [default=disabled]])], - [], [AS_IF([test "x$enable_debug" = "xfull"], [enable_helper_debug_print=yes], - [enable_helper_debug_print=no])]) -AS_IF([test "x$enable_helper_debug_print" != "xno"], [ODPH_DEBUG_PRINT=1], - [ODPH_DEBUG_PRINT=0]) -AC_DEFINE_UNQUOTED([ODPH_DEBUG_PRINT], [$ODPH_DEBUG_PRINT], - [Define to 1 to display helper debug information]) - debug_settings="ODP_DEBUG=${ODP_DEBUG}, ODP_DEBUG_PRINT=${ODP_DEBUG_PRINT}, \ ODPH_DEBUG=${ODPH_DEBUG}, ODPH_DEBUG_PRINT=${ODPH_DEBUG_PRINT}" @@ -500,6 +482,7 @@ AC_MSG_RESULT([ ARCH_ABI ${ARCH_ABI} with_platform: ${with_platform} helper_linux: ${helper_linux} + helper_cli: ${helper_cli} prefix: ${prefix} sysconfdir: ${sysconfdir} libdir: ${libdir} diff --git a/doc/application-api-guide/examples.dox b/doc/application-api-guide/examples.dox index 98e66d72b..18817cd63 100644 --- a/doc/application-api-guide/examples.dox +++ b/doc/application-api-guide/examples.dox @@ -10,6 +10,11 @@ */ /** + * @example odp_cli.c + * CLI example application + */ + +/** * @example odp_debug.c * Debug example application */ diff --git a/example/Makefile.am b/example/Makefile.am index fc4623d1f..d6d242cf9 100644 --- a/example/Makefile.am +++ b/example/Makefile.am @@ -18,3 +18,7 @@ SUBDIRS = classifier \ if HAVE_DW_ATOMIC_CMP_EXC SUBDIRS += ipfragreass endif + +if helper_cli +SUBDIRS += cli +endif diff --git a/example/cli/.gitignore b/example/cli/.gitignore new file mode 100644 index 000000000..2a19d7a64 --- /dev/null +++ b/example/cli/.gitignore @@ -0,0 +1,3 @@ +odp_cli +*.log +*.trs diff --git a/example/cli/Makefile.am b/example/cli/Makefile.am new file mode 100644 index 000000000..0e97a09ed --- /dev/null +++ b/example/cli/Makefile.am @@ -0,0 +1,31 @@ +include $(top_srcdir)/example/Makefile.inc + +bin_PROGRAMS = odp_cli + +odp_cli_SOURCES = odp_cli.c + +if test_example +TESTS = odp_cli_run.sh +endif + +EXTRA_DIST = odp_cli_run.sh + +# If building out-of-tree, make check will not copy the scripts and data to the +# $(builddir) assuming that all commands are run locally. However this prevents +# running tests on a remote target using LOG_COMPILER. +# So copy all script and data files explicitly here. +all-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + if [ -e $(srcdir)/$$f ]; then \ + mkdir -p $(builddir)/$$(dirname $$f); \ + cp -f $(srcdir)/$$f $(builddir)/$$f; \ + fi \ + done \ + fi +clean-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + rm -f $(builddir)/$$f; \ + done \ + fi diff --git a/example/cli/odp_cli.c b/example/cli/odp_cli.c new file mode 100644 index 000000000..e3998129c --- /dev/null +++ b/example/cli/odp_cli.c @@ -0,0 +1,176 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* + * ODP CLI Helper Example + * + * This example shows how to start and stop ODP CLI using the CLI helper + * API functions. This example application can also be used to try out + * the CLI by connecting to a running application with a telnet client. + */ + +#include <odp_api.h> +#include <odp/helper/odph_api.h> + +#include <stdio.h> +#include <stdint.h> +#include <signal.h> + +typedef struct { + int time; + char *addr; + uint16_t port; +} options_t; + +static void usage(const char *prog) +{ + printf("\n" + "Usage: %s [options]\n" + "\n" + "OPTIONS:\n" + " -t, --time <sec> Keep CLI open for <sec> seconds. (default -1 (infinite))\n" + " -a, --address <addr> Bind listening socket to IP address <addr>.\n" + " -p, --port <port> Bind listening socket to port <port>.\n" + "\n" + "ODP helper defaults are used for address and port, if the options are\n" + "not given.\n" + "\n", + prog); +} + +static void parse_args(int argc, char *argv[], options_t *opt) +{ + static const struct option longopts[] = { + { "time", required_argument, NULL, 't' }, + { "address", required_argument, NULL, 'a' }, + { "port", required_argument, NULL, 'p' }, + { "help", no_argument, NULL, 'h' }, + { NULL, 0, NULL, 0 } + }; + + static const char *shortopts = "+t:a:p:h"; + + while (1) { + int c = getopt_long(argc, argv, shortopts, longopts, NULL); + + if (c == -1) + break; /* No more options */ + + switch (c) { + case 't': + opt->time = atoi(optarg); + break; + case 'a': + opt->addr = optarg; + break; + case 'p': + opt->port = atoi(optarg); + break; + default: + usage(argv[0]); + exit(EXIT_SUCCESS); + break; + } + } + + optind = 1; /* reset 'extern optind' from the getopt lib */ +} + +static volatile int shutdown_sig; + +static void sig_handler(int signo) +{ + (void)signo; + + shutdown_sig = 1; +} + +int main(int argc, char *argv[]) +{ + signal(SIGINT, sig_handler); + + odph_helper_options_t helper_options; + + /* Let helper collect its own arguments (e.g. --odph_proc) */ + argc = odph_parse_options(argc, argv); + if (odph_options(&helper_options)) { + ODPH_ERR("Error: reading ODP helper options failed.\n"); + exit(EXIT_FAILURE); + } + + odp_init_t init; + + odp_init_param_init(&init); + init.mem_model = helper_options.mem_model; + + options_t opt = { + .time = -1, + .addr = NULL, + .port = 0, + }; + + parse_args(argc, argv, &opt); + + /* Initialize ODP. */ + + odp_instance_t inst; + + if (odp_init_global(&inst, &init, NULL)) { + ODPH_ERR("Global init failed.\n"); + exit(EXIT_FAILURE); + } + + if (odp_init_local(inst, ODP_THREAD_CONTROL)) { + ODPH_ERR("Local init failed.\n"); + exit(EXIT_FAILURE); + } + + /* Prepare CLI parameters. */ + + odph_cli_param_t cli_param; + + odph_cli_param_init(&cli_param); + + if (opt.addr) + cli_param.address = opt.addr; + + if (opt.port) + cli_param.port = opt.port; + + /* Start CLI server. */ + if (odph_cli_start(inst, &cli_param)) { + ODPH_ERR("CLI start failed.\n"); + exit(EXIT_FAILURE); + } + + printf("CLI server started on %s:%d\n", cli_param.address, cli_param.port); + + /* Wait for the given number of seconds. */ + for (int i = 0; (opt.time < 0 || i < opt.time) && !shutdown_sig; i++) + odp_time_wait_ns(ODP_TIME_SEC_IN_NS); + + printf("Stopping CLI server.\n"); + + /* Stop CLI server. */ + if (odph_cli_stop()) { + ODPH_ERR("CLI stop failed.\n"); + exit(EXIT_FAILURE); + } + + /* Terminate ODP. */ + + if (odp_term_local()) { + ODPH_ERR("Local term failed.\n"); + exit(EXIT_FAILURE); + } + + if (odp_term_global(inst)) { + ODPH_ERR("Global term failed.\n"); + exit(EXIT_FAILURE); + } + + return 0; +} diff --git a/example/cli/odp_cli_run.sh b/example/cli/odp_cli_run.sh new file mode 100755 index 000000000..0dc00b793 --- /dev/null +++ b/example/cli/odp_cli_run.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# +# Copyright (c) 2021, Nokia +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# + +./odp_cli${EXEEXT} -t 2 diff --git a/example/debug/odp_debug.c b/example/debug/odp_debug.c index ef9597906..48591134a 100644 --- a/example/debug/odp_debug.c +++ b/example/debug/odp_debug.c @@ -1,20 +1,27 @@ -/* Copyright (c) 2020, Nokia +/* Copyright (c) 2020-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ +#include <stdlib.h> #include <stdio.h> +#include <stdint.h> +#include <inttypes.h> #include <string.h> #include <getopt.h> #include <odp_api.h> +#include <odp/helper/odph_api.h> typedef struct test_global_t { + int system; int shm; - int shm_all; int pool; int queue; + int pktio; + int ipsec; + int timer; } test_global_t; @@ -27,10 +34,13 @@ static void print_usage(void) "are called when no options are given.\n" "\n" "OPTIONS:\n" - " -S, --shm_all Call odp_shm_print_all()\n" + " -S, --system Call odp_sys_info_print() and odp_sys_config_print()\n" " -s, --shm Create a SHM and call odp_shm_print()\n" " -p, --pool Create various types of pools and call odp_pool_print()\n" " -q, --queue Create various types of queues and call odp_queue_print()\n" + " -i, --interface Create packet IO interface (loop) and call odp_pktio_print()\n" + " -I, --ipsec Call odp_ipsec_print()\n" + " -t, --timer Call timer pool, timer and timeout print functions\n" " -h, --help Display help and exit.\n\n"); } @@ -39,14 +49,17 @@ static int parse_options(int argc, char *argv[], test_global_t *global) int opt, long_index; const struct option longopts[] = { - {"shm_all", no_argument, NULL, 'S'}, + {"system", no_argument, NULL, 'S'}, {"shm", no_argument, NULL, 's'}, {"pool", no_argument, NULL, 'p'}, {"queue", no_argument, NULL, 'q'}, + {"interface", no_argument, NULL, 'i'}, + {"ipsec", no_argument, NULL, 'I'}, + {"timer", no_argument, NULL, 't'}, {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} }; - const char *shortopts = "+Sspqh"; + const char *shortopts = "+SspqiIth"; int ret = 0; while (1) { @@ -57,7 +70,7 @@ static int parse_options(int argc, char *argv[], test_global_t *global) switch (opt) { case 'S': - global->shm_all = 1; + global->system = 1; break; case 's': global->shm = 1; @@ -68,6 +81,15 @@ static int parse_options(int argc, char *argv[], test_global_t *global) case 'q': global->queue = 1; break; + case 'i': + global->pktio = 1; + break; + case 'I': + global->ipsec = 1; + break; + case 't': + global->timer = 1; + break; case 'h': default: print_usage(); @@ -78,32 +100,26 @@ static int parse_options(int argc, char *argv[], test_global_t *global) return ret; } -static int shm_debug(test_global_t *global) +static int shm_debug(void) { const char *name = "debug_shm"; odp_shm_t shm = ODP_SHM_INVALID; - if (global->shm) { - shm = odp_shm_reserve(name, 8 * 1024, 64, 0); - if (shm == ODP_SHM_INVALID) { - printf("SHM reserve failed: %s\n", name); - return -1; - } + shm = odp_shm_reserve(name, 8 * 1024, 64, 0); + if (shm == ODP_SHM_INVALID) { + ODPH_ERR("SHM reserve failed: %s\n", name); + return -1; } - if (global->shm_all) { - printf("\n"); - odp_shm_print_all(); - } + printf("\n"); + odp_shm_print_all(); - if (global->shm) { - printf("\n"); - odp_shm_print(shm); + printf("\n"); + odp_shm_print(shm); - if (odp_shm_free(shm)) { - printf("SHM free failed: %s\n", name); - return -1; - } + if (odp_shm_free(shm)) { + ODPH_ERR("SHM free failed: %s\n", name); + return -1; } return 0; @@ -114,7 +130,7 @@ static int buffer_debug(odp_pool_t pool) odp_buffer_t buf = odp_buffer_alloc(pool); if (buf == ODP_BUFFER_INVALID) { - printf("Buffer alloc failed\n"); + ODPH_ERR("Buffer alloc failed\n"); return -1; } @@ -131,13 +147,16 @@ static int packet_debug(odp_pool_t pool, int len) odp_packet_t pkt = odp_packet_alloc(pool, len); if (pkt == ODP_PACKET_INVALID) { - printf("Packet alloc failed\n"); + ODPH_ERR("Packet alloc failed\n"); return -1; } printf("\n"); odp_packet_print(pkt); + printf("\n"); + odp_packet_print_data(pkt, 0, len); + odp_packet_free(pkt); return 0; @@ -159,7 +178,7 @@ static int pool_debug(void) pool = odp_pool_create(name, ¶m); if (pool == ODP_POOL_INVALID) { - printf("Pool create failed: %s\n", name); + ODPH_ERR("Pool create failed: %s\n", name); return -1; } @@ -170,7 +189,7 @@ static int pool_debug(void) return -1; if (odp_pool_destroy(pool)) { - printf("Pool destroy failed: %s\n", name); + ODPH_ERR("Pool destroy failed: %s\n", name); return -1; } @@ -184,7 +203,7 @@ static int pool_debug(void) pool = odp_pool_create(name, ¶m); if (pool == ODP_POOL_INVALID) { - printf("Pool create failed: %s\n", name); + ODPH_ERR("Pool create failed: %s\n", name); return -1; } @@ -195,7 +214,7 @@ static int pool_debug(void) return -1; if (odp_pool_destroy(pool)) { - printf("Pool destroy failed: %s\n", name); + ODPH_ERR("Pool destroy failed: %s\n", name); return -1; } @@ -207,7 +226,7 @@ static int pool_debug(void) pool = odp_pool_create(name, ¶m); if (pool == ODP_POOL_INVALID) { - printf("Pool create failed: %s\n", name); + ODPH_ERR("Pool create failed: %s\n", name); return -1; } @@ -215,7 +234,7 @@ static int pool_debug(void) odp_pool_print(pool); if (odp_pool_destroy(pool)) { - printf("Pool destroy failed: %s\n", name); + ODPH_ERR("Pool destroy failed: %s\n", name); return -1; } @@ -235,21 +254,18 @@ static int queue_debug(void) queue = odp_queue_create(name, ¶m); if (queue == ODP_QUEUE_INVALID) { - printf("Queue create failed: %s\n", name); + ODPH_ERR("Queue create failed: %s\n", name); return -1; } printf("\n"); + odp_queue_print_all(); + + printf("\n"); odp_queue_print(queue); if (odp_queue_destroy(queue)) { - printf("Queue destroy failed: %s\n", name); - return -1; - } - - /* Configure scheduler before creating any scheduled queues */ - if (odp_schedule_config(NULL)) { - printf("Schedule config failed\n"); + ODPH_ERR("Queue destroy failed: %s\n", name); return -1; } @@ -260,7 +276,7 @@ static int queue_debug(void) queue = odp_queue_create(name, ¶m); if (queue == ODP_QUEUE_INVALID) { - printf("Queue create failed: %s\n", name); + ODPH_ERR("Queue create failed: %s\n", name); return -1; } @@ -268,7 +284,172 @@ static int queue_debug(void) odp_queue_print(queue); if (odp_queue_destroy(queue)) { - printf("Queue destroy failed: %s\n", name); + ODPH_ERR("Queue destroy failed: %s\n", name); + return -1; + } + + return 0; +} + +static int pktio_debug(void) +{ + odp_pool_t pool; + odp_pool_param_t pool_param; + odp_pktio_t pktio; + int pkt_len = 100; + + odp_pool_param_init(&pool_param); + pool_param.type = ODP_POOL_PACKET; + pool_param.pkt.num = 10; + pool_param.pkt.len = pkt_len; + + pool = odp_pool_create("debug_pktio_pool", &pool_param); + + if (pool == ODP_POOL_INVALID) { + ODPH_ERR("Pool create failed\n"); + return -1; + } + + pktio = odp_pktio_open("loop", pool, NULL); + + if (pktio == ODP_PKTIO_INVALID) { + ODPH_ERR("Pktio open failed\n"); + return -1; + } + + printf("\n"); + odp_pktio_print(pktio); + + if (odp_pktio_close(pktio)) { + ODPH_ERR("Pktio close failed\n"); + return -1; + } + + if (odp_pool_destroy(pool)) { + ODPH_ERR("Pool destroy failed\n"); + return -1; + } + + return 0; +} + +static int ipsec_debug(void) +{ + printf("\n"); + odp_ipsec_print(); + + return 0; +} + +static int timer_debug(void) +{ + odp_pool_t pool; + odp_pool_param_t pool_param; + odp_timeout_t timeout; + odp_timer_capability_t timer_capa; + odp_timer_pool_t timer_pool; + odp_timer_pool_param_t timer_param; + odp_timer_t timer; + odp_queue_t queue; + odp_queue_param_t queue_param; + odp_event_t event; + uint64_t tick; + uint64_t max_tmo = ODP_TIME_SEC_IN_NS; + uint64_t res = 100 * ODP_TIME_MSEC_IN_NS; + + odp_pool_param_init(&pool_param); + pool_param.type = ODP_POOL_TIMEOUT; + pool_param.tmo.num = 10; + + pool = odp_pool_create("debug_timer", &pool_param); + + if (pool == ODP_POOL_INVALID) { + ODPH_ERR("Pool create failed\n"); + return -1; + } + + timeout = odp_timeout_alloc(pool); + if (timeout == ODP_TIMEOUT_INVALID) { + ODPH_ERR("Timeout alloc failed\n"); + return -1; + } + + if (odp_timer_capability(ODP_CLOCK_CPU, &timer_capa)) { + ODPH_ERR("Timer capa failed\n"); + return -1; + } + + if (timer_capa.max_tmo.max_tmo < max_tmo) + max_tmo = timer_capa.max_tmo.max_tmo; + + if (timer_capa.max_tmo.res_ns > res) + res = timer_capa.max_tmo.res_ns; + + memset(&timer_param, 0, sizeof(timer_param)); + timer_param.res_ns = res; + timer_param.min_tmo = max_tmo / 10; + timer_param.max_tmo = max_tmo; + timer_param.num_timers = 10; + timer_param.clk_src = ODP_CLOCK_CPU; + + timer_pool = odp_timer_pool_create("debug_timer", &timer_param); + + if (timer_pool == ODP_TIMER_POOL_INVALID) { + ODPH_ERR("Timer pool create failed\n"); + return -1; + } + + odp_timer_pool_start(); + + odp_queue_param_init(&queue_param); + if (timer_capa.queue_type_sched) + queue_param.type = ODP_QUEUE_TYPE_SCHED; + + queue = odp_queue_create("debug_timer", &queue_param); + if (queue == ODP_QUEUE_INVALID) { + ODPH_ERR("Queue create failed.\n"); + return -1; + } + + printf("\n"); + odp_timer_pool_print(timer_pool); + + tick = odp_timer_ns_to_tick(timer_pool, max_tmo / 2); + + timer = odp_timer_alloc(timer_pool, queue, (void *)(uintptr_t)0xdeadbeef); + + printf("\n"); + odp_timeout_print(timeout); + + event = odp_timeout_to_event(timeout); + if (odp_timer_set_rel(timer, tick, &event) != ODP_TIMER_SUCCESS) + ODPH_ERR("Timer set failed.\n"); + + printf("\n"); + odp_timer_print(timer); + + event = odp_timer_free(timer); + + if (event == ODP_EVENT_INVALID) { + ODPH_ERR("Timer free failed.\n"); + } else { + timeout = odp_timeout_from_event(event); + + printf("\n"); + odp_timeout_print(timeout); + + odp_timeout_free(timeout); + } + + odp_timer_pool_destroy(timer_pool); + + if (odp_queue_destroy(queue)) { + ODPH_ERR("Queue destroy failed\n"); + return -1; + } + + if (odp_pool_destroy(pool)) { + ODPH_ERR("Pool destroy failed\n"); return -1; } @@ -285,50 +466,80 @@ int main(int argc, char *argv[]) if (argc < 2) { /* If not arguments, run all test cases */ - global->shm_all = 1; + global->system = 1; global->shm = 1; global->pool = 1; global->queue = 1; + global->pktio = 1; + global->ipsec = 1; + global->timer = 1; } else { if (parse_options(argc, argv, global)) - return -1; + exit(EXIT_FAILURE); } if (odp_init_global(&inst, NULL, NULL)) { - printf("Global init failed.\n"); - return -1; + ODPH_ERR("Global init failed.\n"); + exit(EXIT_FAILURE); } if (odp_init_local(inst, ODP_THREAD_CONTROL)) { - printf("Local init failed.\n"); - return -1; + ODPH_ERR("Local init failed.\n"); + exit(EXIT_FAILURE); } - odp_sys_info_print(); + /* Configure scheduler before creating any scheduled queues */ + if (odp_schedule_config(NULL)) { + ODPH_ERR("Schedule config failed\n"); + exit(EXIT_FAILURE); + } - if ((global->shm_all || global->shm) && shm_debug(global)) { - printf("SHM debug failed.\n"); - return -1; + if (global->system) { + printf("\n"); + odp_sys_info_print(); + + printf("\n"); + odp_sys_config_print(); + } + + if (global->shm && shm_debug()) { + ODPH_ERR("SHM debug failed.\n"); + exit(EXIT_FAILURE); } if (global->pool && pool_debug()) { - printf("Pool debug failed.\n"); - return -1; + ODPH_ERR("Pool debug failed.\n"); + exit(EXIT_FAILURE); } if (global->queue && queue_debug()) { - printf("Queue debug failed.\n"); - return -1; + ODPH_ERR("Queue debug failed.\n"); + exit(EXIT_FAILURE); + } + + if (global->pktio && pktio_debug()) { + ODPH_ERR("Packet debug failed.\n"); + exit(EXIT_FAILURE); + } + + if (global->ipsec && ipsec_debug()) { + ODPH_ERR("IPSEC debug failed.\n"); + exit(EXIT_FAILURE); + } + + if (global->timer && timer_debug()) { + ODPH_ERR("Timer debug failed.\n"); + exit(EXIT_FAILURE); } if (odp_term_local()) { - printf("Local term failed.\n"); - return -1; + ODPH_ERR("Local term failed.\n"); + exit(EXIT_FAILURE); } if (odp_term_global(inst)) { - printf("Global term failed.\n"); - return -1; + ODPH_ERR("Global term failed.\n"); + exit(EXIT_FAILURE); } return 0; diff --git a/example/m4/configure.m4 b/example/m4/configure.m4 index a1668e2b3..b02fd72a5 100644 --- a/example/m4/configure.m4 +++ b/example/m4/configure.m4 @@ -20,6 +20,7 @@ AC_ARG_ENABLE([test-example], AM_CONDITIONAL([test_example], [test x$test_example = xyes ]) AC_CONFIG_FILES([example/classifier/Makefile + example/cli/Makefile example/debug/Makefile example/generator/Makefile example/hello/Makefile diff --git a/example/timer/odp_timer_accuracy.c b/example/timer/odp_timer_accuracy.c index cd790a181..cafe362a7 100644 --- a/example/timer/odp_timer_accuracy.c +++ b/example/timer/odp_timer_accuracy.c @@ -1,5 +1,5 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2019, Nokia + * Copyright (c) 2019-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -51,6 +51,7 @@ typedef struct test_global_t { struct { unsigned long long period_ns; unsigned long long res_ns; + unsigned long long res_hz; unsigned long long offset_ns; unsigned long long max_tmo_ns; unsigned long long num; @@ -89,7 +90,9 @@ static void print_usage(void) "\n" "OPTIONS:\n" " -p, --period <nsec> Timeout period in nsec. Default: 200 msec\n" - " -r, --resolution <nsec> Timeout resolution in nsec. Default: period / 10\n" + " -r, --res_ns <nsec> Timeout resolution in nsec. Default: period / 10\n" + " -R, --res_hz <hertz> Timeout resolution in hertz. Note: resolution can be set\n" + " either in nsec or hertz (not both). Default: 0\n" " -f, --first <nsec> First timer offset in nsec. Default: 300 msec\n" " -x, --max_tmo <nsec> Maximum timeout in nsec. When 0, max tmo is calculated from other options. Default: 0\n" " -n, --num <number> Number of timeout periods. Default: 50\n" @@ -114,7 +117,8 @@ static int parse_options(int argc, char *argv[], test_global_t *test_global) int opt, long_index; const struct option longopts[] = { {"period", required_argument, NULL, 'p'}, - {"resolution", required_argument, NULL, 'r'}, + {"res_ns", required_argument, NULL, 'r'}, + {"res_hz", required_argument, NULL, 'R'}, {"first", required_argument, NULL, 'f'}, {"max_tmo", required_argument, NULL, 'x'}, {"num", required_argument, NULL, 'n'}, @@ -128,11 +132,12 @@ static int parse_options(int argc, char *argv[], test_global_t *test_global) {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} }; - const char *shortopts = "+p:r:f:x:n:b:g:m:o:e:s:ih"; + const char *shortopts = "+p:r:R:f:x:n:b:g:m:o:e:s:ih"; int ret = 0; test_global->opt.period_ns = 200 * ODP_TIME_MSEC_IN_NS; test_global->opt.res_ns = 0; + test_global->opt.res_hz = 0; test_global->opt.offset_ns = 300 * ODP_TIME_MSEC_IN_NS; test_global->opt.max_tmo_ns = 0; test_global->opt.num = 50; @@ -157,6 +162,9 @@ static int parse_options(int argc, char *argv[], test_global_t *test_global) case 'r': test_global->opt.res_ns = strtoull(optarg, NULL, 0); break; + case 'R': + test_global->opt.res_hz = strtoull(optarg, NULL, 0); + break; case 'f': test_global->opt.offset_ns = strtoull(optarg, NULL, 0); break; @@ -200,7 +208,8 @@ static int parse_options(int argc, char *argv[], test_global_t *test_global) } } - if (test_global->opt.res_ns == 0) + /* Default resolution */ + if (test_global->opt.res_ns == 0 && test_global->opt.res_hz == 0) test_global->opt.res_ns = test_global->opt.period_ns / 10; test_global->tot_timers = test_global->opt.num * test_global->opt.burst; @@ -224,7 +233,8 @@ static int start_timers(test_global_t *test_global) odp_queue_t queue; odp_queue_param_t queue_param; uint64_t tick, start_tick; - uint64_t period_ns, res_ns, start_ns, nsec, res_capa, offset_ns; + uint64_t period_ns, res_ns, res_hz, start_ns, nsec, offset_ns; + uint64_t max_res_ns, max_res_hz; odp_event_t event; odp_timeout_t timeout; odp_timer_set_t ret; @@ -298,22 +308,39 @@ static int start_timers(test_global_t *test_global) return -1; } - res_capa = timer_capa.highest_res_ns; + max_res_ns = timer_capa.max_res.res_ns; + max_res_hz = timer_capa.max_res.res_hz; offset_ns = test_global->opt.offset_ns; - res_ns = test_global->opt.res_ns; - if (res_ns < res_capa) { - printf("Resolution %" PRIu64 " nsec too high. " - "Highest resolution %" PRIu64 " nsec. " + if (test_global->opt.res_ns) { + res_ns = test_global->opt.res_ns; + res_hz = 0; + } else { + res_ns = 0; + res_hz = test_global->opt.res_hz; + } + + if (res_ns && res_ns < max_res_ns) { + printf("Resolution %" PRIu64 " nsec too high. Highest resolution %" PRIu64 " nsec. " "Default resolution is period / 10.\n\n", - res_ns, res_capa); + res_ns, max_res_ns); + return -1; + } + + if (res_hz && res_hz > max_res_hz) { + printf("Resolution %" PRIu64 " hz too high. Highest resolution %" PRIu64 " hz. " + "Default resolution is period / 10.\n\n", + res_hz, max_res_hz); return -1; } memset(&timer_param, 0, sizeof(odp_timer_pool_param_t)); - timer_param.res_ns = res_ns; + if (res_ns) + timer_param.res_ns = res_ns; + else + timer_param.res_hz = res_hz; if (mode == 0) { timer_param.min_tmo = offset_ns / 2; @@ -339,7 +366,8 @@ static int start_timers(test_global_t *test_global) printf("\nTest parameters:\n"); printf(" clock source: %i\n", test_global->opt.clk_src); - printf(" resolution capa: %" PRIu64 " nsec\n", res_capa); + printf(" max res nsec: %" PRIu64 "\n", max_res_ns); + printf(" max res hertz: %" PRIu64 "\n", max_res_hz); printf(" max timers capa: %" PRIu32 "\n", timer_capa.max_timers); printf(" mode: %i\n", mode); printf(" restart retries: %i\n", test_global->opt.early_retry); @@ -347,7 +375,10 @@ static int start_timers(test_global_t *test_global) printf(" log file: %s\n", test_global->filename); printf(" start offset: %" PRIu64 " nsec\n", offset_ns); printf(" period: %" PRIu64 " nsec\n", period_ns); - printf(" resolution: %" PRIu64 " nsec\n", timer_param.res_ns); + if (res_ns) + printf(" resolution: %" PRIu64 " nsec\n", res_ns); + else + printf(" resolution: %" PRIu64 " hz\n", res_hz); printf(" min timeout: %" PRIu64 " nsec\n", timer_param.min_tmo); printf(" max timeout: %" PRIu64 " nsec\n", timer_param.max_tmo); printf(" num timeout: %" PRIu64 "\n", num_tmo); @@ -494,11 +525,14 @@ static void print_stat(test_global_t *test_global) { uint64_t i; uint64_t tot_timers = test_global->tot_timers; - uint64_t res_ns = test_global->opt.res_ns; test_stat_t *stat = &test_global->stat; test_log_t *log = test_global->log; double ave_after = 0.0; double ave_before = 0.0; + double res_ns = test_global->opt.res_ns; + + if (test_global->opt.res_ns == 0) + res_ns = 1000000000.0 / test_global->opt.res_hz; if (stat->num_after) ave_after = (double)stat->nsec_after_sum / stat->num_after; diff --git a/helper/Makefile.am b/helper/Makefile.am index c0d635dbc..15e968310 100644 --- a/helper/Makefile.am +++ b/helper/Makefile.am @@ -9,13 +9,15 @@ pkgconfig_DATA = libodphelper.pc AM_CPPFLAGS = \ $(ODP_INCLUDES) \ - $(HELPER_INCLUDES) + $(HELPER_INCLUDES) \ + $(LIBCLI_CPPFLAGS) AM_CFLAGS = $(PTHREAD_CFLAGS) AM_LDFLAGS = -version-number '$(ODPHELPER_LIBSO_VERSION)' helperincludedir = $(includedir)/odp/helper/ helperinclude_HEADERS = \ + include/odp/helper/autoheader_external.h\ include/odp/helper/chksum.h\ include/odp/helper/odph_debug.h \ include/odp/helper/eth.h\ @@ -47,6 +49,11 @@ helperlinuxinclude_HEADERS = \ include/odp/helper/linux/process.h endif +if helper_cli +helperinclude_HEADERS += \ + include/odp/helper/cli.h +endif + noinst_HEADERS = \ include/odph_list_internal.h @@ -67,6 +74,12 @@ __LIB__libodphelper_la_SOURCES += \ linux/thread.c endif +if helper_cli +__LIB__libodphelper_la_SOURCES += \ + cli.c +endif + __LIB__libodphelper_la_LIBADD = $(PTHREAD_LIBS) +__LIB__libodphelper_la_LIBADD += $(LIBCLI_LIBS) lib_LTLIBRARIES = $(LIB)/libodphelper.la diff --git a/helper/cli.c b/helper/cli.c new file mode 100644 index 000000000..3aee88aaa --- /dev/null +++ b/helper/cli.c @@ -0,0 +1,499 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/helper/cli.h> +#include <odp_api.h> +#include <odp/helper/odph_api.h> +#include <libcli.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <arpa/inet.h> +#include <netinet/in.h> +#include <unistd.h> +#include <errno.h> +#include <poll.h> + +/* Socketpair socket roles. */ +enum { + SP_READ = 0, + SP_WRITE = 1, +}; + +typedef struct { + volatile int cli_fd; + /* Server thread will exit if this is false. */ + volatile int run; + /* Socketpair descriptors. */ + int sp[2]; + int listen_fd; + /* This lock guards cli_fd and run, which must be accessed atomically. */ + odp_spinlock_t lock; + odph_thread_t thr_server; +} cli_shm_t; + +static const char *shm_name = "_odp_cli"; + +static const odph_cli_param_t param_default = { + .address = "127.0.0.1", + .port = 55555, +}; + +void odph_cli_param_init(odph_cli_param_t *param) +{ + *param = param_default; +} + +static int cmd_show_cpu(struct cli_def *cli, const char *command ODP_UNUSED, + char *argv[] ODP_UNUSED, int argc ODP_UNUSED) +{ + for (int c = 0; c < odp_cpu_count(); c++) { + cli_print(cli, "% 4d: %s %.03f / %.03f GHz", c, + odp_cpu_model_str_id(c), + (float)odp_cpu_hz_id(c) / 1000000000.0, + (float)odp_cpu_hz_max_id(c) / 1000000000.0); + } + + return CLI_OK; +} + +static int cmd_show_version(struct cli_def *cli, const char *command ODP_UNUSED, + char *argv[] ODP_UNUSED, int argc ODP_UNUSED) +{ + cli_print(cli, "ODP API version: %s", odp_version_api_str()); + cli_print(cli, "ODP implementation name: %s", odp_version_impl_name()); + cli_print(cli, "ODP implementation version: %s", + odp_version_impl_str()); + + return CLI_OK; +} + +/* + * Check that number of given arguments matches required number of + * arguments. Print error messages if this is not the case. Return 0 + * on success, -1 otherwise. + */ +static int check_num_args(struct cli_def *cli, int argc, int req_argc) +{ + if (argc < req_argc) { + cli_error(cli, "%% Incomplete command."); + return -1; + } + + if (argc > req_argc) { + cli_error(cli, "%% Extra parameter given to command."); + return -1; + } + + return 0; +} + +static int cmd_call_odp_ipsec_print(struct cli_def *cli, + const char *command ODP_UNUSED, + char *argv[] ODP_UNUSED, int argc) +{ + if (check_num_args(cli, argc, 0)) + return CLI_ERROR; + + odp_ipsec_print(); + + return CLI_OK; +} + +static int cmd_call_odp_shm_print_all(struct cli_def *cli, + const char *command ODP_UNUSED, + char *argv[] ODP_UNUSED, int argc) +{ + if (check_num_args(cli, argc, 0)) + return CLI_ERROR; + + odp_shm_print_all(); + + return CLI_OK; +} + +static int cmd_call_odp_sys_config_print(struct cli_def *cli, + const char *command ODP_UNUSED, + char *argv[] ODP_UNUSED, int argc) +{ + if (check_num_args(cli, argc, 0)) + return CLI_ERROR; + + odp_sys_config_print(); + + return CLI_OK; +} + +static int cmd_call_odp_sys_info_print(struct cli_def *cli, + const char *command ODP_UNUSED, + char *argv[] ODP_UNUSED, int argc) +{ + if (check_num_args(cli, argc, 0)) + return CLI_ERROR; + + odp_sys_info_print(); + + return CLI_OK; +} + +static int cmd_call_odp_pktio_print(struct cli_def *cli, + const char *command ODP_UNUSED, + char *argv[], int argc) +{ + if (check_num_args(cli, argc, 1)) + return CLI_ERROR; + + odp_pktio_t hdl = odp_pktio_lookup(argv[0]); + + if (hdl == ODP_PKTIO_INVALID) { + cli_error(cli, "%% Name not found."); + return CLI_ERROR; + } + + odp_pktio_print(hdl); + + return CLI_OK; +} + +static int cmd_call_odp_pool_print(struct cli_def *cli, + const char *command ODP_UNUSED, char *argv[], + int argc) +{ + if (check_num_args(cli, argc, 1)) + return CLI_ERROR; + + odp_pool_t hdl = odp_pool_lookup(argv[0]); + + if (hdl == ODP_POOL_INVALID) { + cli_error(cli, "%% Name not found."); + return CLI_ERROR; + } + + odp_pool_print(hdl); + + return CLI_OK; +} + +static int cmd_call_odp_queue_print(struct cli_def *cli, + const char *command ODP_UNUSED, + char *argv[], int argc) +{ + if (check_num_args(cli, argc, 1)) + return CLI_ERROR; + + odp_queue_t hdl = odp_queue_lookup(argv[0]); + + if (hdl == ODP_QUEUE_INVALID) { + cli_error(cli, "%% Name not found."); + return CLI_ERROR; + } + + odp_queue_print(hdl); + + return CLI_OK; +} + +static int cmd_call_odp_shm_print(struct cli_def *cli, + const char *command ODP_UNUSED, char *argv[], + int argc) +{ + if (check_num_args(cli, argc, 1)) + return CLI_ERROR; + + odp_shm_t hdl = odp_shm_lookup(argv[0]); + + if (hdl == ODP_SHM_INVALID) { + cli_error(cli, "%% Name not found."); + return CLI_ERROR; + } + + odp_shm_print(hdl); + + return CLI_OK; +} + +static struct cli_def *create_cli(void) +{ + struct cli_command *c; + struct cli_def *cli; + + cli = cli_init(); + cli_set_banner(cli, NULL); + cli_set_hostname(cli, "ODP"); + + c = cli_register_command(cli, NULL, "show", NULL, + PRIVILEGE_UNPRIVILEGED, MODE_EXEC, + "Show information."); + cli_register_command(cli, c, "cpu", cmd_show_cpu, + PRIVILEGE_UNPRIVILEGED, MODE_EXEC, + "Show CPU information."); + cli_register_command(cli, c, "version", cmd_show_version, + PRIVILEGE_UNPRIVILEGED, MODE_EXEC, + "Show version information."); + + c = cli_register_command(cli, NULL, "call", NULL, + PRIVILEGE_UNPRIVILEGED, MODE_EXEC, + "Call ODP API function."); + cli_register_command(cli, c, "odp_ipsec_print", + cmd_call_odp_ipsec_print, + PRIVILEGE_UNPRIVILEGED, MODE_EXEC, NULL); + cli_register_command(cli, c, "odp_pktio_print", + cmd_call_odp_pktio_print, + PRIVILEGE_UNPRIVILEGED, MODE_EXEC, "<name>"); + cli_register_command(cli, c, "odp_pool_print", + cmd_call_odp_pool_print, + PRIVILEGE_UNPRIVILEGED, MODE_EXEC, "<name>"); + cli_register_command(cli, c, "odp_queue_print", + cmd_call_odp_queue_print, + PRIVILEGE_UNPRIVILEGED, MODE_EXEC, "<name>"); + cli_register_command(cli, c, "odp_shm_print_all", + cmd_call_odp_shm_print_all, + PRIVILEGE_UNPRIVILEGED, MODE_EXEC, NULL); + cli_register_command(cli, c, "odp_shm_print", + cmd_call_odp_shm_print, + PRIVILEGE_UNPRIVILEGED, MODE_EXEC, "<name>"); + cli_register_command(cli, c, "odp_sys_config_print", + cmd_call_odp_sys_config_print, + PRIVILEGE_UNPRIVILEGED, MODE_EXEC, NULL); + cli_register_command(cli, c, "odp_sys_info_print", + cmd_call_odp_sys_info_print, + PRIVILEGE_UNPRIVILEGED, MODE_EXEC, NULL); + + return cli; +} + +static int cli_server(void *arg ODP_UNUSED) +{ + cli_shm_t *shm = NULL; + odp_shm_t shm_hdl = odp_shm_lookup(shm_name); + + if (shm_hdl != ODP_SHM_INVALID) + shm = (cli_shm_t *)odp_shm_addr(shm_hdl); + + if (!shm) { + ODPH_ERR("Error: can't start cli server (shm %s not found)\n", shm_name); + return -1; + } + + struct cli_def *cli = create_cli(); + + while (1) { + struct pollfd pfd[2] = { + { .fd = shm->sp[SP_READ], .events = POLLIN, }, + { .fd = shm->listen_fd, .events = POLLIN, }, + }; + + if (poll(pfd, 2, -1) < 0) { + ODPH_ERR("Error: poll(): %s\n", strerror(errno)); + break; + } + + /* + * If we have an event on a socketpair socket, it's + * time to exit. + */ + if (pfd[0].revents) + break; + + /* + * If we don't have an event on the listening socket, poll + * again. + */ + if (!pfd[1].revents) + continue; + + int fd = accept(shm->listen_fd, NULL, 0); + + if (fd < 0) { + if (errno == EAGAIN || errno == EINTR) + continue; + + ODPH_ERR("Error: accept(): %s\n", strerror(errno)); + break; + } + + odp_spinlock_lock(&shm->lock); + if (!shm->run) { + /* + * odph_cli_stop() has been called. Close the + * socket we just accepted and exit. + */ + close(fd); + odp_spinlock_unlock(&shm->lock); + break; + } + shm->cli_fd = fd; + odp_spinlock_unlock(&shm->lock); + /* + * cli_loop() returns only when client is disconnected. One + * possible reason for disconnect is odph_cli_stop(). + */ + cli_loop(cli, shm->cli_fd); + close(shm->cli_fd); + } + + cli_done(cli); + + return 0; +} + +int odph_cli_start(const odp_instance_t instance, + const odph_cli_param_t *param_in) +{ + if (odp_shm_lookup(shm_name) != ODP_SHM_INVALID) { + ODPH_ERR("Error: cli server already running (shm %s exists)\n", shm_name); + return -1; + } + + cli_shm_t *shm = NULL; + odp_shm_t shm_hdl = odp_shm_reserve(shm_name, sizeof(cli_shm_t), 64, + ODP_SHM_SW_ONLY); + + if (shm_hdl != ODP_SHM_INVALID) + shm = (cli_shm_t *)odp_shm_addr(shm_hdl); + + if (!shm) { + ODPH_ERR("Error: failed to reserve shm %s\n", shm_name); + return -1; + } + + memset(shm, 0, sizeof(cli_shm_t)); + odp_spinlock_init(&shm->lock); + shm->sp[SP_READ] = shm->sp[SP_WRITE] = -1; + shm->listen_fd = -1; + shm->cli_fd = -1; + shm->run = 1; + + if (socketpair(PF_LOCAL, SOCK_STREAM, 0, shm->sp)) { + ODPH_ERR("Error: socketpair(): %s\n", strerror(errno)); + goto error; + } + + /* Create listening socket. */ + + shm->listen_fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (shm->listen_fd < 0) { + ODPH_ERR("Error: socket(): %s\n", strerror(errno)); + goto error; + } + + int on = 1; + + if (setsockopt(shm->listen_fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on))) { + ODPH_ERR("Error: setsockopt(): %s\n", strerror(errno)); + goto error; + } + + struct sockaddr_in addr; + + addr.sin_family = AF_INET; + addr.sin_port = htons(param_in->port); + + switch (inet_pton(AF_INET, param_in->address, &addr.sin_addr)) { + case -1: + ODPH_ERR("Error: inet_pton(): %s\n", strerror(errno)); + goto error; + case 0: + ODPH_ERR("Error: inet_pton(): illegal address format\n"); + goto error; + } + + if (bind(shm->listen_fd, (struct sockaddr *)&addr, sizeof(addr))) { + ODPH_ERR("Error: bind(): %s\n", strerror(errno)); + goto error; + } + + if (listen(shm->listen_fd, 1)) { + ODPH_ERR("Error: listen(): %s\n", strerror(errno)); + goto error; + } + + /* Create server thread. */ + + odp_cpumask_t cpumask; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param; + + if (odp_cpumask_default_control(&cpumask, 1) != 1) { + ODPH_ERR("Error: odp_cpumask_default_control() failed\n"); + goto error; + } + + memset(&thr_common, 0, sizeof(thr_common)); + thr_common.instance = instance; + thr_common.cpumask = &cpumask; + + memset(&thr_param, 0, sizeof(thr_param)); + thr_param.thr_type = ODP_THREAD_CONTROL; + thr_param.start = cli_server; + + memset(&shm->thr_server, 0, sizeof(shm->thr_server)); + + if (odph_thread_create(&shm->thr_server, &thr_common, &thr_param, 1) != 1) { + ODPH_ERR("Error: odph_thread_create() failed\n"); + goto error; + } + + return 0; + +error: + close(shm->sp[SP_READ]); + close(shm->sp[SP_WRITE]); + close(shm->listen_fd); + close(shm->cli_fd); + shm->run = 0; + return -1; +} + +int odph_cli_stop(void) +{ + cli_shm_t *shm = NULL; + odp_shm_t shm_hdl = odp_shm_lookup(shm_name); + + if (shm_hdl != ODP_SHM_INVALID) + shm = (cli_shm_t *)odp_shm_addr(shm_hdl); + + if (!shm) { + ODPH_ERR("Error: cli server not running (shm %s not found)\n", shm_name); + return -1; + } + + odp_spinlock_lock(&shm->lock); + shm->run = 0; + /* + * Close the current cli connection. This stops cli_loop(). + */ + close(shm->cli_fd); + odp_spinlock_unlock(&shm->lock); + + /* + * Send a message to the server thread in order to break it out of a + * blocking poll() call. + */ + int stop = 1; + int sent = send(shm->sp[SP_WRITE], &stop, + sizeof(stop), MSG_DONTWAIT | MSG_NOSIGNAL); + + if (sent != sizeof(stop)) { + ODPH_ERR("Error: send() = %d: %s\n", sent, strerror(errno)); + return -1; + } + + if (odph_thread_join(&shm->thr_server, 1) != 1) { + ODPH_ERR("Error: odph_thread_join() failed\n"); + return -1; + } + + close(shm->sp[SP_READ]); + close(shm->sp[SP_WRITE]); + close(shm->listen_fd); + + if (odp_shm_free(shm_hdl)) { + ODPH_ERR("Error: odp_shm_free() failed\n"); + return -1; + } + + return 0; +} diff --git a/helper/include/odp/helper/autoheader_external.h.in b/helper/include/odp/helper/autoheader_external.h.in new file mode 100644 index 000000000..6f5187a5b --- /dev/null +++ b/helper/include/odp/helper/autoheader_external.h.in @@ -0,0 +1,14 @@ + +#ifndef ODPH_AUTOHEADER_EXTERNAL_H_ +#define ODPH_AUTOHEADER_EXTERNAL_H_ + +/* Define to 1 to enable CLI helper */ +#undef ODPH_CLI + +/* Define to 1 to include additional helper debug code */ +#undef ODPH_DEBUG + +/* Define to 1 to display helper debug information */ +#undef ODPH_DEBUG_PRINT + +#endif diff --git a/helper/include/odp/helper/cli.h b/helper/include/odp/helper/cli.h new file mode 100644 index 000000000..41e438857 --- /dev/null +++ b/helper/include/odp/helper/cli.h @@ -0,0 +1,92 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP CLI helper API + * + * This API allows control of ODP CLI server, which may be connected to + * using a telnet client. CLI commands may be used to get information + * from an ODP instance, for debugging purposes. + * + * Many CLI commands output the information to the console, or wherever + * ODP logs have been directed to in global init. + */ + +#ifndef ODPH_CLI_H_ +#define ODPH_CLI_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp_api.h> +#include <odp/helper/ip.h> +#include <stdint.h> + +/** + * @addtogroup odph_cli ODPH CLI + * @{ + */ + +/** ODP CLI server parameters */ +typedef struct { + /** + * A character string containing an IP address. Default is + * "127.0.0.1". + */ + const char *address; + /** TCP port. Default is 55555. */ + uint16_t port; +} odph_cli_param_t; + +/** + * Initialize CLI server params + * + * Initialize an odph_cli_param_t to its default values for all + * fields. + * + * @param[out] param Pointer to parameter structure + */ +void odph_cli_param_init(odph_cli_param_t *param); + +/** + * Start CLI server + * + * Upon successful return from this function, the CLI server will be + * accepting client connections. This function spawns a new thread of + * type ODP_THREAD_CONTROL using odp_cpumask_default_control(). + * + * @param instance ODP instance + * @param param CLI server parameters to use + * @retval 0 Success + * @retval <0 Failure + */ +int odph_cli_start(const odp_instance_t instance, + const odph_cli_param_t *param); + +/** + * Stop CLI server + * + * Stop accepting new client connections and disconnect currently + * connected client. This function terminates the control thread + * created in odph_cli_start(). + * + * @retval 0 Success + * @retval <0 Failure + */ +int odph_cli_stop(void); + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/helper/include/odp/helper/odph_api.h b/helper/include/odp/helper/odph_api.h index f3bcde208..7ab875c6e 100644 --- a/helper/include/odp/helper/odph_api.h +++ b/helper/include/odp/helper/odph_api.h @@ -38,6 +38,12 @@ extern "C" { #include <odp/helper/udp.h> #include <odp/helper/version.h> +#include <odp/helper/autoheader_external.h> + +#ifdef ODPH_CLI +#include <odp/helper/cli.h> +#endif + #ifdef __cplusplus } #endif diff --git a/helper/include/odp/helper/odph_debug.h b/helper/include/odp/helper/odph_debug.h index 2fa89d8ab..39ce3f0d9 100644 --- a/helper/include/odp/helper/odph_debug.h +++ b/helper/include/odp/helper/odph_debug.h @@ -15,7 +15,7 @@ #ifndef ODPH_DEBUG_H_ #define ODPH_DEBUG_H_ -#include <odp/autoheader_external.h> +#include <odp/helper/autoheader_external.h> #include <stdio.h> #include <stdlib.h> diff --git a/helper/libodphelper.pc.in b/helper/libodphelper.pc.in index b14335e5c..be99eeefc 100644 --- a/helper/libodphelper.pc.in +++ b/helper/libodphelper.pc.in @@ -7,5 +7,5 @@ Name: libodphelper Description: Helper for the ODP packet processing engine Version: @PKGCONFIG_VERSION@ Libs: -L${libdir} -lodphelper -Libs.private: +Libs.private: @LIBCLI_LIBS@ Cflags: -I${includedir} diff --git a/helper/m4/configure.m4 b/helper/m4/configure.m4 index c25f8fe68..36629c81e 100644 --- a/helper/m4/configure.m4 +++ b/helper/m4/configure.m4 @@ -1,4 +1,9 @@ ########################################################################## +# Include m4 files +########################################################################## +m4_include([helper/m4/libcli.m4]) + +########################################################################## # Enable/disable test-helper ########################################################################## AC_ARG_ENABLE([test-helper], @@ -17,5 +22,30 @@ AC_ARG_ENABLE([helper-linux], [helper_linux=$enableval], [helper_linux=yes]) +########################################################################## +# Enable/disable ODPH_DEBUG +########################################################################## +AC_ARG_ENABLE([helper-debug], + [AS_HELP_STRING([--enable-helper-debug], + [helpers include additional debugging code [default=disabled]])], + [], [AS_IF([test "x$enable_debug" = "xfull"], [enable_helper_debug=yes], + [enable_helper_debug=no])]) +AS_IF([test "x$enable_helper_debug" != "xno"], [ODPH_DEBUG=1], [ODPH_DEBUG=0]) +AC_DEFINE_UNQUOTED([ODPH_DEBUG], [$ODPH_DEBUG], + [Define to 1 to include additional helper debug code]) + +########################################################################## +# Enable/disable ODPH_DEBUG_PRINT +########################################################################## +AC_ARG_ENABLE([helper-debug-print], + [AS_HELP_STRING([--enable-helper-debug-print], + [display helper debugging information [default=disabled]])], + [], [AS_IF([test "x$enable_debug" = "xfull"], [enable_helper_debug_print=yes], + [enable_helper_debug_print=no])]) +AS_IF([test "x$enable_helper_debug_print" != "xno"], [ODPH_DEBUG_PRINT=1], + [ODPH_DEBUG_PRINT=0]) +AC_DEFINE_UNQUOTED([ODPH_DEBUG_PRINT], [$ODPH_DEBUG_PRINT], + [Define to 1 to display helper debug information]) + AC_CONFIG_FILES([helper/libodphelper.pc helper/test/Makefile]) diff --git a/helper/m4/libcli.m4 b/helper/m4/libcli.m4 new file mode 100644 index 000000000..6748cbb41 --- /dev/null +++ b/helper/m4/libcli.m4 @@ -0,0 +1,43 @@ +########################################################################## +# Set optional libcli path +########################################################################## +AC_ARG_WITH([libcli-path], + [AS_HELP_STRING([--with-libcli-path=DIR], + [path to libcli libs and headers [default=system]])], + [libcli_path_given=yes + LIBCLI_CPPFLAGS="-I$withval/include" + LIBCLI_LIBS="-L$withval/lib" + LIBCLI_RPATH="-R$withval/lib"], + []) + +########################################################################## +# Save and set temporary compilation flags +########################################################################## +OLD_CPPFLAGS=$CPPFLAGS +OLD_LIBS=$LIBS +CPPFLAGS="$LIBCLI_CPPFLAGS $CPPFLAGS" +LIBS="$LIBCLI_LIBS $LIBS" + +######################################################################### +# If libcli is available, enable CLI helper +######################################################################### +helper_cli=no +AC_CHECK_HEADER(libcli.h, + [AC_CHECK_LIB(cli, cli_init, [helper_cli=yes], [], [-lcrypt])], + [AS_IF([test "x$libcli_path_given" = "xyes"], + [AC_MSG_ERROR([libcli not found at the specified path (--with-libcli-path)])])]) + +AS_IF([test "x$helper_cli" != "xno"], + [AC_DEFINE_UNQUOTED([ODPH_CLI], [1], [Define to 1 to enable CLI helper]) + LIBCLI_LIBS="$LIBCLI_RPATH $LIBCLI_LIBS -lcli -lcrypt"], + [LIBCLI_CPPFLAGS="" + LIBCLI_LIBS=""]) + +########################################################################## +# Restore old saved variables +########################################################################## +LIBS=$OLD_LIBS +CPPFLAGS=$OLD_CPPFLAGS + +AC_SUBST([LIBCLI_CPPFLAGS]) +AC_SUBST([LIBCLI_LIBS]) diff --git a/helper/test/.gitignore b/helper/test/.gitignore index 23fab7b30..e1e5ab7b2 100644 --- a/helper/test/.gitignore +++ b/helper/test/.gitignore @@ -1,6 +1,7 @@ *.trs *.log chksum +cli cuckootable iplookuptable odpthreads diff --git a/helper/test/Makefile.am b/helper/test/Makefile.am index 76bfac8fe..57e54cfb8 100644 --- a/helper/test/Makefile.am +++ b/helper/test/Makefile.am @@ -19,6 +19,11 @@ linux_pthread_SOURCES = linux/pthread.c linux_process_SOURCES = linux/process.c endif +if helper_cli +EXECUTABLES += cli +cli_SOURCES = cli.c +endif + COMPILE_ONLY = odpthreads TESTSCRIPTS = odpthreads_as_processes \ diff --git a/helper/test/cli.c b/helper/test/cli.c new file mode 100644 index 000000000..ff3ea2f53 --- /dev/null +++ b/helper/test/cli.c @@ -0,0 +1,62 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_api.h> +#include <odp/helper/odph_api.h> + +int main(int argc, char *argv[]) +{ + odp_instance_t instance; + odph_helper_options_t helper_options; + odp_init_t init_param; + + argc = odph_parse_options(argc, argv); + if (odph_options(&helper_options)) { + ODPH_ERR("Error: reading ODP helper options failed.\n"); + exit(EXIT_FAILURE); + } + + odp_init_param_init(&init_param); + init_param.mem_model = helper_options.mem_model; + + memset(&instance, 0, sizeof(instance)); + + if (odp_init_global(&instance, NULL, NULL)) { + ODPH_ERR("Error: ODP global init failed.\n"); + exit(EXIT_FAILURE); + } + + if (odp_init_local(instance, ODP_THREAD_CONTROL)) { + ODPH_ERR("Error: ODP local init failed.\n"); + exit(EXIT_FAILURE); + } + + odph_cli_param_t cli_param; + + odph_cli_param_init(&cli_param); + + if (odph_cli_start(instance, &cli_param)) { + ODPH_ERR("Error: odph_cli_start() failed.\n"); + exit(EXIT_FAILURE); + } + + if (odph_cli_stop()) { + ODPH_ERR("Error: odph_cli_stop() failed.\n"); + exit(EXIT_FAILURE); + } + + if (odp_term_local()) { + ODPH_ERR("Error: ODP local term failed.\n"); + exit(EXIT_FAILURE); + } + + if (odp_term_global(instance)) { + ODPH_ERR("Error: ODP global term failed.\n"); + exit(EXIT_FAILURE); + } + + return EXIT_SUCCESS; +} diff --git a/helper/test/debug.c b/helper/test/debug.c index 2431b0ecd..3b8a69d8b 100644 --- a/helper/test/debug.c +++ b/helper/test/debug.c @@ -4,7 +4,7 @@ * SPDX-License-Identifier: BSD-3-Clause */ -#include <odp/autoheader_external.h> +#include <odp/helper/autoheader_external.h> #include <odp_api.h> #include <odp/helper/odph_api.h> diff --git a/include/odp/api/abi-default/atomic.h b/include/odp/api/abi-default/atomic.h index e37f254fc..a58f42b8f 100644 --- a/include/odp/api/abi-default/atomic.h +++ b/include/odp/api/abi-default/atomic.h @@ -1,4 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2021, ARM Limited * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -59,6 +60,31 @@ typedef struct ODP_ALIGNED(sizeof(uint64_t)) odp_atomic_u64_s { #endif +#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS + +/** + * @internal + * Atomic 128-bit unsigned integer + */ +typedef struct ODP_ALIGNED(sizeof(odp_u128_t)) odp_atomic_u128_s { + odp_u128_t v; /**< Actual storage for the atomic variable */ +} odp_atomic_u128_t; + +#else + +/** + * @internal + * Atomic 128-bit unsigned integer + */ +typedef struct ODP_ALIGNED(sizeof(odp_u128_t)) odp_atomic_u128_s { + odp_u128_t v; /**< Actual storage for the atomic variable */ + /* Some architectures do not support lock-free operations on 128-bit + * data types. We use a spin lock to ensure atomicity. */ + char lock; /**< Spin lock (if needed) used to ensure atomic access */ +} odp_atomic_u128_t; + +#endif + #ifdef __cplusplus } #endif diff --git a/include/odp/api/spec/atomic.h b/include/odp/api/spec/atomic.h index f1e1818eb..a77527e85 100644 --- a/include/odp/api/spec/atomic.h +++ b/include/odp/api/spec/atomic.h @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2021, ARM Limited * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -25,13 +26,13 @@ extern "C" { * @details * <b> Atomic integers using relaxed memory ordering </b> * - * Atomic integer types (odp_atomic_u32_t and odp_atomic_u64_t) can be used to - * implement e.g. shared counters. If not otherwise documented, operations in - * this API are implemented using <b> RELAXED memory ordering </b> (see memory - * order descriptions in the C11 specification). Relaxed operations do not - * provide synchronization or ordering for other memory accesses (initiated - * before or after the operation), only atomicity of the operation itself is - * guaranteed. + * Atomic integer types (odp_atomic_u32_t, odp_atomic_u64_t and + * odp_atomic_u128_t) can be used to implement e.g. shared counters. If not + * otherwise documented, operations in this API are implemented using + * <b> RELAXED memory ordering </b> (see memory order descriptions in + * the C11 specification). Relaxed operations do not provide synchronization or + * ordering for other memory accesses (initiated before or after the operation), + * only atomicity of the operation itself is guaranteed. * * <b> Operations with non-relaxed memory ordering </b> * @@ -54,6 +55,9 @@ extern "C" { */ /** + * @typedef odp_atomic_u128_t + * Atomic 128-bit unsigned integer + * * @typedef odp_atomic_u64_t * Atomic 64-bit unsigned integer * @@ -189,9 +193,10 @@ void odp_atomic_min_u32(odp_atomic_u32_t *atom, uint32_t new_min); * Compare and swap atomic uint32 variable * * Compares value of atomic variable to the value pointed by 'old_val'. - * If values are equal, the operation writes 'new_val' into the atomic variable - * and returns success. If they are not equal, the operation writes current - * value of atomic variable into 'old_val' and returns failure. + * If the values are equal, the operation writes 'new_val' into the atomic variable + * and returns success. The operation returns failure only when the values are + * not equal (strong CAS operation). The current value of atomic variable is written + * into 'old_val' on failure. * * @param atom Pointer to atomic variable * @param[in,out] old_val Pointer to the old value of the atomic variable. @@ -199,10 +204,8 @@ void odp_atomic_min_u32(odp_atomic_u32_t *atom, uint32_t new_min); * @param new_val New value to be written into the atomic variable * * @return 0 on failure, !0 on success - * */ -int odp_atomic_cas_u32(odp_atomic_u32_t *atom, uint32_t *old_val, - uint32_t new_val); +int odp_atomic_cas_u32(odp_atomic_u32_t *atom, uint32_t *old_val, uint32_t new_val); /** * Exchange value of atomic uint32 variable @@ -345,9 +348,10 @@ void odp_atomic_min_u64(odp_atomic_u64_t *atom, uint64_t new_min); * Compare and swap atomic uint64 variable * * Compares value of atomic variable to the value pointed by 'old_val'. - * If values are equal, the operation writes 'new_val' into the atomic variable - * and returns success. If they are not equal, the operation writes current - * value of atomic variable into 'old_val' and returns failure. + * If the values are equal, the operation writes 'new_val' into the atomic variable + * and returns success. The operation returns failure only when the values are + * not equal (strong CAS operation). The current value of atomic variable is written + * into 'old_val' on failure. * * @param atom Pointer to atomic variable * @param[in,out] old_val Pointer to the old value of the atomic variable. @@ -356,8 +360,7 @@ void odp_atomic_min_u64(odp_atomic_u64_t *atom, uint64_t new_min); * * @return 0 on failure, !0 on success */ -int odp_atomic_cas_u64(odp_atomic_u64_t *atom, uint64_t *old_val, - uint64_t new_val); +int odp_atomic_cas_u64(odp_atomic_u64_t *atom, uint64_t *old_val, uint64_t new_val); /** * Exchange value of atomic uint64 variable @@ -373,6 +376,58 @@ int odp_atomic_cas_u64(odp_atomic_u64_t *atom, uint64_t *old_val, uint64_t odp_atomic_xchg_u64(odp_atomic_u64_t *atom, uint64_t new_val); /* + * 128-bit operations in RELAXED memory ordering + * -------------------------------------------- + */ + +/** + * Initialize atomic odp_u128_t variable + * + * Initializes the atomic variable with 'val'. This operation is not atomic. + * Application must ensure that there's no race condition while initializing + * the variable. + * + * @param atom Pointer to atomic variable + * @param val Value to initialize the variable with + */ +void odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val); + +/** + * Load value of atomic odp_u128_t variable + * + * @param atom Pointer to atomic variable + * + * @return Value of the variable + */ +odp_u128_t odp_atomic_load_u128(odp_atomic_u128_t *atom); + +/** + * Store value to atomic odp_u128_t variable + * + * @param atom Pointer to atomic variable + * @param val Value to store in the variable + */ +void odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val); + +/** + * Compare and swap atomic odp_u128_t variable + * + * Compares value of atomic variable to the value pointed by 'old_val'. + * If the values are equal, the operation writes 'new_val' into the atomic variable + * and returns success. The operation returns failure only when the values are + * not equal (strong CAS operation). The current value of atomic variable is written + * into 'old_val' on failure. + * + * @param atom Pointer to atomic variable + * @param[in,out] old_val Pointer to the old value of the atomic variable. + * Operation updates this value on failure. + * @param new_val New value to be written into the atomic variable + * + * @return 0 on failure, !0 on success + */ +int odp_atomic_cas_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, odp_u128_t new_val); + +/* * 32-bit operations in non-RELAXED memory ordering * ------------------------------------------------ */ @@ -622,6 +677,80 @@ typedef union odp_atomic_op_t { */ int odp_atomic_lock_free_u64(odp_atomic_op_t *atomic_op); +/* + * 128-bit operations in non-RELAXED memory ordering + * ------------------------------------------------ + */ + +/** + * Compare and swap atomic odp_u128_t variable using ACQUIRE memory ordering + * + * Otherwise identical to odp_atomic_cas_u128() but ensures ACQUIRE memory + * ordering on success. Memory ordering is RELAXED on failure. + * + * @param atom Pointer to atomic variable + * @param[in,out] old_val Pointer to the old value of the atomic variable. + * Operation updates this value on failure. + * @param new_val New value to be written into the atomic variable + * + * @return 0 on failure, !0 on success + */ +int odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val); + +/** + * Compare and swap atomic odp_u128_t variable using RELEASE memory ordering + * + * Otherwise identical to odp_atomic_cas_u128() but ensures RELEASE memory + * ordering on success. Memory ordering is RELAXED on failure. + * + * @param atom Pointer to atomic variable + * @param[in,out] old_val Pointer to the old value of the atomic variable. + * Operation updates this value on failure. + * @param new_val New value to be written into the atomic variable + * + * @return 0 on failure, !0 on success + */ +int odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val); + +/** + * Compare and swap atomic odp_u128_t variable using ACQUIRE-and-RELEASE memory + * ordering + * + * Otherwise identical to odp_atomic_cas_u128() but ensures ACQUIRE-and-RELEASE + * memory ordering on success. Memory ordering is RELAXED on failure. + * + * @param atom Pointer to atomic variable + * @param[in,out] old_val Pointer to the old value of the atomic variable. + * Operation updates this value on failure. + * @param new_val New value to be written into the atomic variable + * + * @return 0 on failure, !0 on success + */ +int odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128_t *old_val, + odp_u128_t new_val); + +/** + * Query which atomic odp_atomic_u128_t operations are lock-free + * + * Lock-free implementations have higher performance and scale better than + * implementations using locks. + * + * Init operations (e.g. odp_atomic_init_u128()) are not atomic. This function + * clears the op.init bit but will never set it to one. + * + * Note: 128-bit atomic API includes only init, load, store and CAS operations. + * + * @param atomic_op Pointer to atomic operation structure for storing + * operation flags. All bits are initialized to zero during + * the operation. The parameter is ignored when NULL. + * @retval 0 None of the 128-bit atomic operations are lock-free + * @retval 1 Some of the 128-bit atomic operations are lock-free + * @retval 2 All 128-bit atomic operations are lock-free + */ +int odp_atomic_lock_free_u128(odp_atomic_op_t *atomic_op); + /** * @} */ diff --git a/include/odp/api/spec/ipsec.h b/include/odp/api/spec/ipsec.h index 156d66019..d1856432b 100644 --- a/include/odp/api/spec/ipsec.h +++ b/include/odp/api/spec/ipsec.h @@ -63,6 +63,9 @@ typedef enum odp_ipsec_op_mode_t { * Packet input/output is connected directly to IPSEC inbound/outbound * processing. Application uses asynchronous or inline IPSEC * operations. + * + * Inline processed inbound packets are delivered to the application + * in the same way as packets processed by odp_ipsec_in_enq(). */ ODP_IPSEC_OP_MODE_INLINE, @@ -72,6 +75,46 @@ typedef enum odp_ipsec_op_mode_t { } odp_ipsec_op_mode_t; /** + * IPSEC TEST SA operation + */ +typedef enum odp_ipsec_test_sa_operation_t { + /** Update next sequence number + * + * The seq_num parameter is an outbound SA specific parameter. + * Invoking the odp_ipsec_test_sa_update() API to update this + * field on an inbound SA will cause the API to return failure. + */ + ODP_IPSEC_TEST_SA_UPDATE_SEQ_NUM = 0, + + /** Update highest authenticated sequence number + * + * The antireplay_window_top parameter is inbound SA specific. + * Invoking the odp_ipsec_test_sa_update() API to update this + * field on an outbound SA will cause the API to return failure. + */ + ODP_IPSEC_TEST_SA_UPDATE_ANTIREPLAY_WINDOW_TOP + +} odp_ipsec_test_sa_operation_t; + +/** + * IPSEC TEST SA parameter + */ +typedef union odp_ipsec_test_sa_param_t { + /** Next sequence number + * + * @see ODP_IPSEC_TEST_SA_UPDATE_SEQ_NUM + */ + uint64_t seq_num; + + /** Highest authenticated sequence number + * + * @see ODP_IPSEC_TEST_SA_UPDATE_ANTIREPLAY_WINDOW_TOP + */ + uint64_t antireplay_window_top; + +} odp_ipsec_test_sa_param_t; + +/** * Configuration options for IPSEC inbound processing */ typedef struct odp_ipsec_inbound_config_t { @@ -193,6 +236,28 @@ typedef struct odp_ipsec_outbound_config_t { } odp_ipsec_outbound_config_t; /** + * IPSEC TEST capability + */ +typedef struct odp_ipsec_test_capability_t { + /** Parameters supported for sa_update */ + struct { + /** Next sequence number value + * + * @see ODP_IPSEC_TEST_SA_UPDATE_SEQ_NUM + */ + odp_bool_t seq_num; + + /** Highest authenticated sequence number + * + * @see ODP_IPSEC_TEST_SA_UPDATE_ANTIREPLAY_WINDOW_TOP + */ + odp_bool_t antireplay_window_top; + + } sa_operations; + +} odp_ipsec_test_capability_t; + +/** * IPSEC capability */ typedef struct odp_ipsec_capability_t { @@ -273,6 +338,11 @@ typedef struct odp_ipsec_capability_t { */ odp_support_t inline_ipsec_tm; + /** IPSEC TEST capabilities + * + * @see odp_ipsec_test_sa_update() + */ + odp_ipsec_test_capability_t test; } odp_ipsec_capability_t; /** @@ -452,13 +522,13 @@ typedef struct odp_ipsec_ipv4_param_t { /** IPv4 destination address (NETWORK ENDIAN) */ void *dst_addr; - /** IPv4 Differentiated Services Code Point */ + /** IPv4 Differentiated Services Code Point. The default value is 0. */ uint8_t dscp; - /** IPv4 Don't Fragment bit */ + /** IPv4 Don't Fragment bit. The default value is 0. */ uint8_t df; - /** IPv4 Time To Live */ + /** IPv4 Time To Live. The default value is 255. */ uint8_t ttl; } odp_ipsec_ipv4_param_t; @@ -471,13 +541,13 @@ typedef struct odp_ipsec_ipv6_param_t { /** IPv6 destination address (NETWORK ENDIAN) */ void *dst_addr; - /** IPv6 flow label */ + /** IPv6 flow label. The default value is 0. */ uint32_t flabel; - /** IPv6 Differentiated Services Code Point */ + /** IPv6 Differentiated Services Code Point. The default value is 0. */ uint8_t dscp; - /** IPv6 hop limit */ + /** IPv6 hop limit. The default value is 255. */ uint8_t hlimit; } odp_ipsec_ipv6_param_t; @@ -491,11 +561,11 @@ typedef struct odp_ipsec_ipv6_param_t { * pointers and copied byte-by-byte from memory to the packet. */ typedef struct odp_ipsec_tunnel_param_t { - /** Tunnel type: IPv4 or IPv6 */ + /** Tunnel type: IPv4 or IPv6. The default is IPv4. */ odp_ipsec_tunnel_type_t type; - /** Variant mappings for tunnel parameters */ - union { + /** Tunnel type specific parameters */ + struct { /** IPv4 header parameters */ odp_ipsec_ipv4_param_t ipv4; @@ -511,7 +581,7 @@ typedef struct odp_ipsec_sa_opt_t { /** Extended Sequence Numbers (ESN) * * * 1: Use extended (64 bit) sequence numbers - * * 0: Use normal sequence numbers + * * 0: Use normal sequence numbers (the default value) */ uint32_t esn : 1; @@ -519,7 +589,7 @@ typedef struct odp_ipsec_sa_opt_t { * * * 1: Do UDP encapsulation/decapsulation so that IPSEC packets can * traverse through NAT boxes. - * * 0: No UDP encapsulation + * * 0: No UDP encapsulation (the default value) */ uint32_t udp_encap : 1; @@ -529,7 +599,7 @@ typedef struct odp_ipsec_sa_opt_t { * the outer IP header in encapsulation, and vice versa in * decapsulation. * * 0: Use values from odp_ipsec_tunnel_param_t in encapsulation and - * do not change DSCP field in decapsulation. + * do not change DSCP field in decapsulation (the default value). */ uint32_t copy_dscp : 1; @@ -537,7 +607,7 @@ typedef struct odp_ipsec_sa_opt_t { * * * 1: Copy IPv6 flow label from inner IPv6 header to the * outer IPv6 header. - * * 0: Use value from odp_ipsec_tunnel_param_t + * * 0: Use value from odp_ipsec_tunnel_param_t (the default value) */ uint32_t copy_flabel : 1; @@ -545,7 +615,7 @@ typedef struct odp_ipsec_sa_opt_t { * * * 1: Copy the DF bit from the inner IPv4 header to the outer * IPv4 header. - * * 0: Use value from odp_ipsec_tunnel_param_t + * * 0: Use value from odp_ipsec_tunnel_param_t (the default value) */ uint32_t copy_df : 1; @@ -554,7 +624,7 @@ typedef struct odp_ipsec_sa_opt_t { * * 1: In tunnel mode, decrement inner packet IPv4 TTL or * IPv6 Hop Limit after tunnel decapsulation, or before tunnel * encapsulation. - * * 0: Inner packet is not modified. + * * 0: Inner packet is not modified (the default value) */ uint32_t dec_ttl : 1; @@ -569,6 +639,8 @@ typedef struct odp_ipsec_sa_opt_t { * lifetime expiration is reported: only once, first N or all packets following * the limit crossing. Any number of limits may be used simultaneously. * Use zero when there is no limit. + * + * The default value is zero (i.e. no limit) for all the limits. */ typedef struct odp_ipsec_lifetime_t { /** Soft expiry limits for the session */ @@ -669,7 +741,7 @@ typedef struct odp_ipsec_sa_param_t { /** IPSEC SA direction: inbound or outbound */ odp_ipsec_dir_t dir; - /** IPSEC protocol: ESP or AH */ + /** IPSEC protocol: ESP or AH. The default value is ODP_IPSEC_ESP. */ odp_ipsec_protocol_t proto; /** IPSEC protocol mode: transport or tunnel */ @@ -712,10 +784,12 @@ typedef struct odp_ipsec_sa_param_t { uint32_t context_len; /** IPSEC SA direction dependent parameters */ - union { + struct { /** Inbound specific parameters */ struct { - /** SA lookup mode */ + /** SA lookup mode + * The default value is ODP_IPSEC_LOOKUP_DISABLED. + */ odp_ipsec_lookup_mode_t lookup_mode; /** Additional SA lookup parameters. Values are @@ -732,7 +806,7 @@ typedef struct odp_ipsec_sa_param_t { } lookup_param; /** Minimum anti-replay window size. Use 0 to disable - * anti-replay service. + * anti-replay service. The default value is 0. */ uint32_t antireplay_ws; @@ -765,7 +839,9 @@ typedef struct odp_ipsec_sa_param_t { /** Parameters for tunnel mode */ odp_ipsec_tunnel_param_t tunnel; - /** Fragmentation mode */ + /** Fragmentation mode + * The default value is ODP_IPSEC_FRAG_DISABLED. + */ odp_ipsec_frag_mode_t frag_mode; /** MTU for outbound IP fragmentation offload @@ -814,7 +890,14 @@ typedef struct odp_ipsec_stats_t { * IPSEC SA information */ typedef struct odp_ipsec_sa_info_t { - /** Copy of IPSEC Security Association (SA) parameters */ + /** IPsec SA parameters + * + * This is not necessarily an exact copy of the actual parameter + * structure used in SA creation. The fields that were relevant + * for the SA in the creation phase will have the same values, + * but other fields, such as tunnel parameters for a transport + * mode SA, will have undefined values. + */ odp_ipsec_sa_param_t param; /** IPSEC SA direction dependent parameters */ @@ -1575,6 +1658,12 @@ int odp_ipsec_out(const odp_packet_t pkt_in[], int num_in, * may be processed simultaneously in both modes (initiated by this function * and inline operation). * + * Post-processing may be required after the reception of an IPsec packet + * event to complete IPsec processing for the packet. The post-processing + * happens in the odp_ipsec_result() function that must be called at least + * once before packet data or metadata (other than packet type and subtype) + * may be accessed. + * * @param pkt Packets to be processed * @param num Number of packets to be processed * @param param Inbound operation parameters @@ -1611,6 +1700,12 @@ int odp_ipsec_in_enq(const odp_packet_t pkt[], int num, * The function may be used also in inline processing mode, e.g. for IPSEC * packets for which inline processing is not possible. * + * Post-processing may be required after the reception of an IPsec packet + * event to complete IPsec processing for the packet. The post-processing + * happens in the odp_ipsec_result() function that must be called at least + * once before packet data or metadata (other than packet type and subtype) + * may be accessed. + * * @param pkt Packets to be processed * @param num Number of packets to be processed * @param param Outbound operation parameters @@ -1716,6 +1811,36 @@ int odp_ipsec_result(odp_ipsec_packet_result_t *result, odp_packet_t packet); int odp_ipsec_status(odp_ipsec_status_t *status, odp_event_t event); /** + * IPSEC test API for modifying internal state of an SA. + * + * This function is not meant to be used by normal applications but by special + * test applications that test or debug the operation of the underlying ODP + * implementation. Calling this function may degrade the performance of the + * calling thread, other threads or the IPSEC implementation in general. + * + * Calling this function for an SA at the same time when the SA is used for + * processing traffic or when the SA is being modified through other parts + * of IPSEC API may result in undefined behaviour. + * + * SA state update through this function may not be supported by all ODP + * implementations, ODP instances or SA instances or at every moment. This + * function may return failure for unspecified reasons even when the capability + * call indicated support for updating a particular parameter and previous + * similar calls succeeded. + * + * @param sa IPSEC SA to be updated + * @param op Specifies operation to be performed + * @param param Pointer to IPSEC TEST SA param structure to be + * used for the operation + * + * @return 0 On success + * @retval <0 On failure + */ +int odp_ipsec_test_sa_update(odp_ipsec_sa_t sa, + odp_ipsec_test_sa_operation_t op, + const odp_ipsec_test_sa_param_t *param); + +/** * Update MTU for outbound IP fragmentation * * When IP fragmentation offload is enabled, the SA is created with an MTU. diff --git a/include/odp/api/spec/queue.h b/include/odp/api/spec/queue.h index 0260a7a33..005229e35 100644 --- a/include/odp/api/spec/queue.h +++ b/include/odp/api/spec/queue.h @@ -305,6 +305,14 @@ int odp_queue_info(odp_queue_t queue, odp_queue_info_t *info); void odp_queue_print(odp_queue_t queue); /** + * Print debug info about all queues + * + * Print implementation defined information about all created queues to the ODP + * log. The information is intended to be used for debugging. + */ +void odp_queue_print_all(void); + +/** * @} */ diff --git a/include/odp/api/spec/std_types.h b/include/odp/api/spec/std_types.h index 6e2617b4e..bf3eb77a6 100644 --- a/include/odp/api/spec/std_types.h +++ b/include/odp/api/spec/std_types.h @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2021, ARM Limited * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -51,6 +52,28 @@ typedef uint32_t odp_una_u32_t ODP_ALIGNED(1); typedef uint64_t odp_una_u64_t ODP_ALIGNED(1); /** + * @typedef odp_u128_t + * 128-bit unsigned integer structure + */ + +/** + * 128-bit unsigned integer structure + */ +typedef struct ODP_ALIGNED(16) odp_u128_s { + /** 128 bits in various sizes */ + union { + /** 128 bits as uint64_t words */ + uint64_t u64[2]; + /** 128 bits as uint32_t words */ + uint32_t u32[4]; + /** 128 bits as uint16_t words */ + uint16_t u16[8]; + /** 128 bits as bytes */ + uint8_t u8[16]; + }; +} odp_u128_t; + +/** * @} */ diff --git a/include/odp/api/spec/timer.h b/include/odp/api/spec/timer.h index 22c4fbef8..62151e485 100644 --- a/include/odp/api/spec/timer.h +++ b/include/odp/api/spec/timer.h @@ -383,8 +383,7 @@ int odp_timer_pool_info(odp_timer_pool_t timer_pool, * @return Timer handle on success * @retval ODP_TIMER_INVALID on failure and errno set. */ -odp_timer_t odp_timer_alloc(odp_timer_pool_t timer_pool, odp_queue_t queue, - void *user_ptr); +odp_timer_t odp_timer_alloc(odp_timer_pool_t timer_pool, odp_queue_t queue, const void *user_ptr); /** * Free a timer @@ -566,6 +565,36 @@ odp_timeout_t odp_timeout_alloc(odp_pool_t pool); void odp_timeout_free(odp_timeout_t tmo); /** + * Print timer pool debug information + * + * Prints implementation specific debug information about + * the timer pool to the ODP log. + * + * @param timer_pool Timer pool handle + */ +void odp_timer_pool_print(odp_timer_pool_t timer_pool); + +/** + * Print timer debug information + * + * Prints implementation specific debug information about + * the timer to the ODP log. + * + * @param timer Timer handle + */ +void odp_timer_print(odp_timer_t timer); + +/** + * Print timeout debug information + * + * Prints implementation specific debug information about + * the timeout to the ODP log. + * + * @param tmo Timeout handle + */ +void odp_timeout_print(odp_timeout_t tmo); + +/** * Get printable value for an odp_timer_pool_t * * @param timer_pool odp_timer_pool_t handle to be printed diff --git a/include/odp/autoheader_external.h.in b/include/odp/autoheader_external.h.in index 15626898d..978bc1f2b 100644 --- a/include/odp/autoheader_external.h.in +++ b/include/odp/autoheader_external.h.in @@ -8,10 +8,4 @@ /* Define to 1 to display debug information */ #undef ODP_DEBUG_PRINT -/* Define to 1 to include additional helper debug code */ -#undef ODPH_DEBUG - -/* Define to 1 to display helper debug information */ -#undef ODPH_DEBUG_PRINT - #endif diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 4f6585c14..38a0258a0 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -92,8 +92,9 @@ endif noinst_HEADERS = \ ${top_srcdir}/platform/linux-generic/include/odp_align_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_atomic_internal.h \ - include/odp_buffer_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_bitset.h \ + include/odp_buffer_internal.h \ + ${top_srcdir}/platform/linux-generic/include/odp_chksum_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_classification_internal.h \ include/odp_config_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_debug_internal.h \ @@ -256,7 +257,8 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/aarch64/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/atomic.h \ + arch/aarch64/odp/api/abi/cpu.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ @@ -319,6 +321,7 @@ if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu.h endif noinst_HEADERS += arch/x86/cpu_flags.h \ + arch/x86/odp_cpu.h \ arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h endif diff --git a/platform/linux-dpdk/m4/configure.m4 b/platform/linux-dpdk/m4/configure.m4 index 3bf091082..b86bbbbb5 100644 --- a/platform/linux-dpdk/m4/configure.m4 +++ b/platform/linux-dpdk/m4/configure.m4 @@ -61,7 +61,7 @@ esac # Required for experimental rte_event_port_unlinks_in_progress() API DPDK_CFLAGS="${DPDK_CFLAGS} -DALLOW_EXPERIMENTAL_API" -AS_VAR_APPEND([PLAT_DEP_LIBS], ["${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT}"]) +AS_VAR_APPEND([PLAT_DEP_LIBS], ["${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS}"]) # Add text to the end of configure with platform specific settings. # Make sure it's aligned same as other lines in configure.ac. diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index 35c81edf4..e65a08d0e 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -1,5 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2019, Nokia + * Copyright (c) 2019-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -9,6 +9,7 @@ #include <odp/api/plat/packet_inlines.h> #include <odp_packet_internal.h> #include <odp_debug_internal.h> +#include <odp_chksum_internal.h> #include <odp/api/hints.h> #include <odp/api/byteorder.h> #include <odp/api/plat/byteorder_inlines.h> @@ -1224,86 +1225,25 @@ int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt) return dst_size < src_size; } -/* Simple implementation of ones complement sum. - * Based on RFC1071 and its errata. - */ -typedef union { - uint16_t w; - uint8_t b[2]; -} swap_buf_t; - -static uint32_t segment_sum16_32(const uint8_t *p, - uint32_t len, - uint32_t offset) - -{ - uint32_t sum = 0; - - /* Include second part of 16-bit short word split between segments */ - if (len > 0 && (offset % 2)) { - swap_buf_t sw; - - sw.b[0] = 0; - sw.b[1] = *p++; - sum = sw.w; - len--; - } - - /* - * If pointer is 16-bit aligned, we can do fast path calculation. - * If it is not, we sum hi and lo bytes separately and then sum them. - */ - if ((uintptr_t)p % 2) { - uint32_t sum1 = 0, sum2 = 0; - - while (len > 1) { - sum1 += *p++; - sum2 += *p++; - len -= 2; - } -#if (ODP_BYTE_ORDER == ODP_BIG_ENDIAN) - sum += sum2 + (sum1 << 8); -#else - sum += sum1 + (sum2 << 8); -#endif - } else { - while (len > 1) { - sum += *(const uint16_t *)(uintptr_t)p; - p += 2; - len -= 2; - } - } - - /* Add left-over byte, if any */ - if (len > 0) { - swap_buf_t sw; - - sw.b[0] = *p; - sw.b[1] = 0; - sum += sw.w; - } - - return sum; -} - -static uint32_t packet_sum16_32(odp_packet_hdr_t *pkt_hdr, - uint32_t offset, - uint32_t len) +static uint64_t packet_sum_partial(odp_packet_hdr_t *pkt_hdr, + uint32_t l3_offset, + uint32_t offset, + uint32_t len) { - uint32_t sum = 0; + uint64_t sum = 0; + uint32_t frame_len = odp_packet_len(packet_handle(pkt_hdr)); - if (offset + len > odp_packet_len(packet_handle(pkt_hdr))) + if (offset + len > frame_len) return 0; while (len > 0) { uint32_t seglen = 0; /* GCC */ - void *mapaddr = odp_packet_offset(packet_handle(pkt_hdr), - offset, &seglen, NULL); + void *mapaddr = odp_packet_offset(packet_handle(pkt_hdr), offset, &seglen, NULL); if (seglen > len) seglen = len; - sum += segment_sum16_32(mapaddr, seglen, offset); + sum += chksum_partial(mapaddr, seglen, offset - l3_offset); len -= seglen; offset += seglen; } @@ -1311,20 +1251,14 @@ static uint32_t packet_sum16_32(odp_packet_hdr_t *pkt_hdr, return sum; } -static uint16_t packet_sum_ones_comp16(odp_packet_hdr_t *pkt_hdr, - uint32_t offset, - uint32_t len, - uint32_t l4_part_sum) +static inline uint16_t packet_sum(odp_packet_hdr_t *pkt_hdr, + uint32_t l3_offset, + uint32_t offset, + uint32_t len, + uint64_t sum) { - uint32_t sum = l4_part_sum; - - sum += packet_sum16_32(pkt_hdr, offset, len); - - /* Not more than two additions */ - sum = (sum & 0xffff) + (sum >> 16); - sum = (sum & 0xffff) + (sum >> 16); - - return sum; + sum += packet_sum_partial(pkt_hdr, l3_offset, offset, len); + return chksum_finalize(sum); } static uint32_t packet_sum_crc32c(odp_packet_hdr_t *pkt_hdr, @@ -1441,7 +1375,7 @@ error: static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, odp_proto_chksums_t chksums, - uint32_t *l4_part_sum) + uint64_t *l4_part_sum) { const _odp_ipv4hdr_t *ipv4 = (const _odp_ipv4hdr_t *)*parseptr; uint32_t dstaddr = odp_be_to_cpu_32(ipv4->dst_addr); @@ -1459,7 +1393,7 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, if (chksums.chksum.ipv4) { prs->input_flags.l3_chksum_done = 1; - if (odp_chksum_ones_comp16(ipv4, ihl * 4) != 0xffff) { + if (chksum_finalize(chksum_partial(ipv4, ihl * 4, 0)) != 0xffff) { prs->flags.ip_err = 1; prs->flags.l3_chksum_err = 1; return 0; @@ -1470,8 +1404,8 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += ihl * 4; if (chksums.chksum.udp || chksums.chksum.tcp) - *l4_part_sum = segment_sum16_32((const uint8_t *)&ipv4->src_addr, - 2 * _ODP_IPV4ADDR_LEN, 0); + *l4_part_sum = chksum_partial((const uint8_t *)&ipv4->src_addr, + 2 * _ODP_IPV4ADDR_LEN, 0); if (odp_unlikely(ihl > _ODP_IPV4HDR_IHL_MIN)) prs->input_flags.ipopt = 1; @@ -1501,7 +1435,7 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, uint32_t seg_len, odp_proto_chksums_t chksums, - uint32_t *l4_part_sum) + uint64_t *l4_part_sum) { const _odp_ipv6hdr_t *ipv6 = (const _odp_ipv6hdr_t *)*parseptr; const _odp_ipv6hdr_ext_t *ipv6ext; @@ -1525,8 +1459,8 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_ipv6hdr_t); if (chksums.chksum.udp || chksums.chksum.tcp) - *l4_part_sum = segment_sum16_32((const uint8_t *)&ipv6->src_addr, - 2 * _ODP_IPV6ADDR_LEN, 0); + *l4_part_sum = chksum_partial((const uint8_t *)&ipv6->src_addr, + 2 * _ODP_IPV6ADDR_LEN, 0); /* Skip past any IPv6 extension headers */ if (ipv6->next_hdr == _ODP_IPPROTO_HOPOPTS || @@ -1569,7 +1503,7 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t tcp_len, odp_proto_chksums_t chksums, - uint32_t *l4_part_sum) + uint64_t *l4_part_sum) { const _odp_tcphdr_t *tcp = (const _odp_tcphdr_t *)*parseptr; uint32_t len = tcp->hl * 4; @@ -1595,7 +1529,7 @@ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, */ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, odp_proto_chksums_t chksums, - uint32_t *l4_part_sum) + uint64_t *l4_part_sum) { const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr; uint32_t udplen = odp_be_to_cpu_16(udp->length); @@ -1642,7 +1576,7 @@ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t sctp_len, odp_proto_chksums_t chksums, - uint32_t *l4_part_sum) + uint64_t *l4_part_sum) { if (odp_unlikely(sctp_len < sizeof(_odp_sctphdr_t))) { prs->flags.sctp_err = 1; @@ -1670,7 +1604,7 @@ int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, uint32_t frame_len, uint32_t seg_len, int layer, uint16_t ethtype, odp_proto_chksums_t chksums, - uint32_t *l4_part_sum) + uint64_t *l4_part_sum) { uint8_t ip_proto; @@ -1783,7 +1717,7 @@ int _odp_packet_parse_common(packet_parser_t *prs, const uint8_t *ptr, uint32_t offset; uint16_t ethtype; const uint8_t *parseptr; - uint32_t l4_part_sum; + uint64_t l4_part_sum; parseptr = ptr; offset = 0; @@ -1820,7 +1754,7 @@ static inline int packet_ipv4_chksum(odp_packet_t pkt, if (odp_unlikely(res < 0)) return res; - *chksum = ~odp_chksum_ones_comp16(buf, nleft); + *chksum = ~chksum_finalize(chksum_partial(buf, nleft, 0)); return 0; } @@ -1868,7 +1802,7 @@ static int _odp_packet_tcp_udp_chksum_insert(odp_packet_t pkt, uint16_t proto) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); uint32_t zero = 0; - uint32_t sum; + uint64_t sum; uint16_t l3_ver; uint16_t chksum; uint32_t chksum_offset; @@ -1882,15 +1816,17 @@ static int _odp_packet_tcp_udp_chksum_insert(odp_packet_t pkt, uint16_t proto) odp_packet_copy_to_mem(pkt, pkt_hdr->p.l3_offset, 2, &l3_ver); if (_ODP_IPV4HDR_VER(l3_ver) == _ODP_IPV4) - sum = packet_sum16_32(pkt_hdr, - pkt_hdr->p.l3_offset + - _ODP_IPV4ADDR_OFFSSET, - 2 * _ODP_IPV4ADDR_LEN); + sum = packet_sum_partial(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l3_offset + + _ODP_IPV4ADDR_OFFSSET, + 2 * _ODP_IPV4ADDR_LEN); else - sum = packet_sum16_32(pkt_hdr, - pkt_hdr->p.l3_offset + - _ODP_IPV6ADDR_OFFSSET, - 2 * _ODP_IPV6ADDR_LEN); + sum = packet_sum_partial(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l3_offset + + _ODP_IPV6ADDR_OFFSSET, + 2 * _ODP_IPV6ADDR_LEN); #if ODP_BYTE_ORDER == ODP_BIG_ENDIAN sum += proto; #else @@ -1898,26 +1834,26 @@ static int _odp_packet_tcp_udp_chksum_insert(odp_packet_t pkt, uint16_t proto) #endif if (proto == _ODP_IPPROTO_TCP) { - sum += odp_cpu_to_be_16(frame_len - pkt_hdr->p.l4_offset); + sum += odp_cpu_to_be_16(frame_len - + pkt_hdr->p.l4_offset); chksum_offset = pkt_hdr->p.l4_offset + _ODP_UDP_CSUM_OFFSET; } else { - sum += packet_sum16_32(pkt_hdr, - pkt_hdr->p.l4_offset + - _ODP_UDP_LEN_OFFSET, - 2); + sum += packet_sum_partial(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l4_offset + + _ODP_UDP_LEN_OFFSET, + 2); chksum_offset = pkt_hdr->p.l4_offset + _ODP_UDP_CSUM_OFFSET; } odp_packet_copy_from_mem(pkt, chksum_offset, 2, &zero); - sum += packet_sum16_32(pkt_hdr, - pkt_hdr->p.l4_offset, - frame_len - pkt_hdr->p.l4_offset); - - /* Not more than two additions */ - sum = (sum & 0xffff) + (sum >> 16); - sum = (sum & 0xffff) + (sum >> 16); + sum += packet_sum_partial(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l4_offset, + frame_len - + pkt_hdr->p.l4_offset); - chksum = ~sum; + chksum = ~chksum_finalize(sum); if (proto == _ODP_IPPROTO_UDP && chksum == 0) chksum = 0xffff; @@ -1965,7 +1901,7 @@ int _odp_packet_sctp_chksum_insert(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); uint32_t sum; - uint32_t len = odp_packet_len(pkt); + uint32_t frame_len = odp_packet_len(pkt); if (pkt_hdr->p.l4_offset == ODP_PACKET_OFFSET_INVALID) return -1; @@ -1973,26 +1909,28 @@ int _odp_packet_sctp_chksum_insert(odp_packet_t pkt) sum = 0; odp_packet_copy_from_mem(pkt, pkt_hdr->p.l4_offset + 8, 4, &sum); sum = ~packet_sum_crc32c(pkt_hdr, pkt_hdr->p.l4_offset, - len - pkt_hdr->p.l4_offset, ~0); + frame_len - pkt_hdr->p.l4_offset, + ~0); return odp_packet_copy_from_mem(pkt, pkt_hdr->p.l4_offset + 8, 4, &sum); } static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, odp_proto_chksums_t chksums, - uint32_t l4_part_sum) + uint64_t l4_part_sum) { - uint32_t len = odp_packet_len(packet_handle(pkt_hdr)); + uint32_t frame_len = odp_packet_len(packet_handle(pkt_hdr)); /* UDP chksum == 0 case is covered in parse_udp() */ if (chksums.chksum.udp && pkt_hdr->p.input_flags.udp && !pkt_hdr->p.input_flags.ipfrag && !pkt_hdr->p.input_flags.udp_chksum_zero) { - uint16_t sum = ~packet_sum_ones_comp16(pkt_hdr, - pkt_hdr->p.l4_offset, - len - - pkt_hdr->p.l4_offset, - l4_part_sum); + uint16_t sum = ~packet_sum(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l4_offset, + frame_len - + pkt_hdr->p.l4_offset, + l4_part_sum); pkt_hdr->p.input_flags.l4_chksum_done = 1; if (sum != 0) { @@ -2005,11 +1943,12 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, if (chksums.chksum.tcp && pkt_hdr->p.input_flags.tcp && !pkt_hdr->p.input_flags.ipfrag) { - uint16_t sum = ~packet_sum_ones_comp16(pkt_hdr, - pkt_hdr->p.l4_offset, - len - - pkt_hdr->p.l4_offset, - l4_part_sum); + uint16_t sum = ~packet_sum(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l4_offset, + frame_len - + pkt_hdr->p.l4_offset, + l4_part_sum); pkt_hdr->p.input_flags.l4_chksum_done = 1; if (sum != 0) { @@ -2025,7 +1964,7 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, uint32_t sum = ~packet_sum_crc32c(pkt_hdr, pkt_hdr->p.l4_offset + _ODP_SCTPHDR_LEN, - len - + frame_len - pkt_hdr->p.l4_offset - _ODP_SCTPHDR_LEN, l4_part_sum); @@ -2058,7 +1997,7 @@ int _odp_packet_parse_layer(odp_packet_hdr_t *pkt_hdr, const uint8_t *base = odp_packet_data(pkt); uint32_t offset = 0; uint16_t ethtype; - uint32_t l4_part_sum = 0; + uint64_t l4_part_sum = 0; int rc; if (odp_unlikely(layer == ODP_PROTO_LAYER_NONE)) @@ -2094,7 +2033,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, odp_proto_layer_t layer = param->last_layer; int ret; uint16_t ethtype; - uint32_t l4_part_sum = 0; + uint64_t l4_part_sum = 0; if (proto == ODP_PROTO_NONE || layer == ODP_PROTO_LAYER_NONE) return -1; diff --git a/platform/linux-dpdk/odp_queue_basic.c b/platform/linux-dpdk/odp_queue_basic.c index a006bcc54..4cf619965 100644 --- a/platform/linux-dpdk/odp_queue_basic.c +++ b/platform/linux-dpdk/odp_queue_basic.c @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -726,6 +727,97 @@ static void queue_print(odp_queue_t handle) UNLOCK(queue); } +static void queue_print_all(void) +{ + uint32_t i, index, len, max_len; + const char *name; + int status; + odp_queue_type_t type; + odp_nonblocking_t blocking; + odp_queue_op_mode_t enq_mode; + odp_queue_op_mode_t deq_mode; + odp_queue_order_t order; + const char *status_str; + const char *bl_str; + char type_c, enq_c, deq_c, order_c, sync_c; + const int col_width = 24; + int prio = 0; + odp_schedule_sync_t sync = ODP_SCHED_SYNC_PARALLEL; + + ODP_PRINT("\nList of all queues\n"); + ODP_PRINT("------------------\n"); + ODP_PRINT(" idx %-*s type stat blk enq deq ord len max_len sync prio\n", col_width, "name"); + + for (i = 0; i < CONFIG_MAX_QUEUES; i++) { + queue_entry_t *queue = qentry_from_index(i); + + if (queue->s.status < QUEUE_STATUS_READY) + continue; + + LOCK(queue); + + status = queue->s.status; + index = queue->s.index; + name = queue->s.name; + type = queue->s.type; + blocking = queue->s.param.nonblocking; + enq_mode = queue->s.param.enq_mode; + deq_mode = queue->s.param.deq_mode; + order = queue->s.param.order; + + if (queue->s.queue_lf) { + len = _odp_queue_lf_length(queue->s.queue_lf); + max_len = _odp_queue_lf_max_length(); + } else if (queue->s.spsc) { + len = ring_spsc_length(queue->s.ring_spsc); + max_len = ring_spsc_max_length(queue->s.ring_spsc); + } else if (type == ODP_QUEUE_TYPE_SCHED) { + len = ring_st_length(queue->s.ring_st); + max_len = ring_st_max_length(queue->s.ring_st); + prio = queue->s.param.sched.prio; + sync = queue->s.param.sched.sync; + } else { + len = ring_mpmc_length(queue->s.ring_mpmc); + max_len = ring_mpmc_max_length(queue->s.ring_mpmc); + } + + UNLOCK(queue); + + if (status < QUEUE_STATUS_READY) + continue; + + status_str = (status == QUEUE_STATUS_READY) ? "R" : + ((status == QUEUE_STATUS_SCHED) ? "S" : "NS"); + + type_c = (type == ODP_QUEUE_TYPE_PLAIN) ? 'P' : 'S'; + + bl_str = (blocking == ODP_BLOCKING) ? "B" : + ((blocking == ODP_NONBLOCKING_LF) ? "LF" : "WF"); + + enq_c = (enq_mode == ODP_QUEUE_OP_MT) ? 'S' : + ((enq_mode == ODP_QUEUE_OP_MT_UNSAFE) ? 'U' : 'D'); + + deq_c = (deq_mode == ODP_QUEUE_OP_MT) ? 'S' : + ((deq_mode == ODP_QUEUE_OP_MT_UNSAFE) ? 'U' : 'D'); + + order_c = (order == ODP_QUEUE_ORDER_KEEP) ? 'K' : 'I'; + + ODP_PRINT("%4u %-*s %c %2s %2s", index, col_width, name, type_c, + status_str, bl_str); + ODP_PRINT(" %c %c %c %6u %6u", enq_c, deq_c, order_c, len, max_len); + + if (type == ODP_QUEUE_TYPE_SCHED) { + sync_c = (sync == ODP_SCHED_SYNC_PARALLEL) ? 'P' : + ((sync == ODP_SCHED_SYNC_ATOMIC) ? 'A' : 'O'); + ODP_PRINT(" %c %4i", sync_c, prio); + } + + ODP_PRINT("\n"); + } + + ODP_PRINT("\n"); +} + static inline int _sched_queue_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], int num) { @@ -1094,7 +1186,9 @@ _odp_queue_api_fn_t queue_basic_api = { .queue_to_u64 = queue_to_u64, .queue_param_init = queue_param_init, .queue_info = queue_info, - .queue_print = queue_print + .queue_print = queue_print, + .queue_print_all = queue_print_all + }; /* Functions towards internal components */ diff --git a/platform/linux-dpdk/odp_queue_if.c b/platform/linux-dpdk/odp_queue_if.c index f19716d73..310664fb8 100644 --- a/platform/linux-dpdk/odp_queue_if.c +++ b/platform/linux-dpdk/odp_queue_if.c @@ -88,7 +88,7 @@ uint64_t odp_queue_to_u64(odp_queue_t hdl) void odp_queue_param_init(odp_queue_param_t *param) { - return _odp_queue_api->queue_param_init(param); + _odp_queue_api->queue_param_init(param); } int odp_queue_info(odp_queue_t queue, odp_queue_info_t *info) @@ -98,7 +98,12 @@ int odp_queue_info(odp_queue_t queue, odp_queue_info_t *info) void odp_queue_print(odp_queue_t queue) { - return _odp_queue_api->queue_print(queue); + _odp_queue_api->queue_print(queue); +} + +void odp_queue_print_all(void) +{ + _odp_queue_api->queue_print_all(); } int _odp_queue_init_global(void) diff --git a/platform/linux-dpdk/odp_timer.c b/platform/linux-dpdk/odp_timer.c index 5ae6fa77c..0910d09ac 100644 --- a/platform/linux-dpdk/odp_timer.c +++ b/platform/linux-dpdk/odp_timer.c @@ -1,5 +1,5 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2019-2020, Nokia + * Copyright (c) 2019-2021, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -69,7 +69,7 @@ typedef struct { odp_ticketlock_t lock; int state; uint64_t tick; - void *user_ptr; + const void *user_ptr; odp_queue_t queue; odp_event_t tmo_event; struct timer_pool_s *timer_pool; @@ -507,7 +507,7 @@ uint64_t odp_timer_pool_to_u64(odp_timer_pool_t tp) odp_timer_t odp_timer_alloc(odp_timer_pool_t tp, odp_queue_t queue, - void *user_ptr) + const void *user_ptr) { uint32_t timer_idx; timer_entry_t *timer; @@ -811,7 +811,7 @@ void *odp_timeout_user_ptr(odp_timeout_t tmo) { odp_timeout_hdr_t *timeout_hdr = timeout_to_hdr(tmo); - return timeout_hdr->user_ptr; + return (void *)(uintptr_t)timeout_hdr->user_ptr; } odp_timeout_t odp_timeout_alloc(odp_pool_t pool) @@ -829,3 +829,76 @@ void odp_timeout_free(odp_timeout_t tmo) odp_buffer_free(odp_buffer_from_event(ev)); } + +void odp_timer_pool_print(odp_timer_pool_t timer_pool) +{ + timer_pool_t *tp; + + if (timer_pool == ODP_TIMER_POOL_INVALID) { + ODP_ERR("Bad timer pool handle\n"); + return; + } + + tp = timer_pool_from_hdl(timer_pool); + + ODP_PRINT("\nTimer pool info\n"); + ODP_PRINT("---------------\n"); + ODP_PRINT(" timer pool %p\n", tp); + ODP_PRINT(" name %s\n", tp->name); + ODP_PRINT(" num timers %u\n", tp->cur_timers); + ODP_PRINT(" hwm timers %u\n", tp->hwm_timers); + ODP_PRINT(" num tp %i\n", timer_global->num_timer_pools); + ODP_PRINT("\n"); +} + +void odp_timer_print(odp_timer_t timer_hdl) +{ + timer_entry_t *timer = timer_from_hdl(timer_hdl); + + if (timer_hdl == ODP_TIMER_INVALID) { + ODP_ERR("Bad timer handle\n"); + return; + } + + ODP_PRINT("\nTimer info\n"); + ODP_PRINT("----------\n"); + ODP_PRINT(" timer pool %p\n", timer->timer_pool); + ODP_PRINT(" timer index %" PRIu32 "\n", timer->timer_idx); + ODP_PRINT(" dest queue 0x%" PRIx64 "\n", odp_queue_to_u64(timer->queue)); + ODP_PRINT(" user ptr %p\n", timer->user_ptr); + ODP_PRINT(" state %s\n", + (timer->state == NOT_TICKING) ? "not ticking" : + (timer->state == EXPIRED ? "expired" : "ticking")); + ODP_PRINT("\n"); +} + +void odp_timeout_print(odp_timeout_t tmo) +{ + const odp_timeout_hdr_t *timeout_hdr = timeout_to_hdr(tmo); + odp_timer_t timer_hdl; + timer_pool_t *tp = NULL; + uint32_t idx = 0; + + if (tmo == ODP_TIMEOUT_INVALID) { + ODP_ERR("Bad timeout handle\n"); + return; + } + + timer_hdl = timeout_hdr->timer; + + if (timer_hdl != ODP_TIMER_INVALID) { + timer_entry_t *timer = timer_from_hdl(timer_hdl); + + tp = timer->timer_pool; + idx = timer->timer_idx; + } + + ODP_PRINT("\nTimeout info\n"); + ODP_PRINT("------------\n"); + ODP_PRINT(" tmo handle 0x%" PRIx64 "\n", odp_timeout_to_u64(tmo)); + ODP_PRINT(" timer pool %p\n", tp); + ODP_PRINT(" timer index %u\n", idx); + ODP_PRINT(" expiration %" PRIu64 "\n", timeout_hdr->expiration); + ODP_PRINT(" user ptr %p\n", timeout_hdr->user_ptr); + ODP_PRINT("\n"); +} diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 70ab7ab7c..1d9a3bbdb 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -96,6 +96,7 @@ noinst_HEADERS = \ include/odp_atomic_internal.h \ include/odp_bitset.h \ include/odp_buffer_internal.h \ + include/odp_chksum_internal.h \ include/odp_classification_datamodel.h \ include/odp_classification_internal.h \ include/odp_config_internal.h \ @@ -289,7 +290,8 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/cpu_inlines.h \ arch/default/odp/api/abi/cpu_time.h \ arch/aarch64/odp/api/abi/hash_crc32.h if !ODP_ABI_COMPAT -odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/cpu.h +odpapiabiarchinclude_HEADERS += arch/aarch64/odp/api/abi/atomic.h \ + arch/aarch64/odp/api/abi/cpu.h endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ @@ -352,6 +354,7 @@ if !ODP_ABI_COMPAT odpapiabiarchinclude_HEADERS += arch/x86/odp/api/abi/cpu.h endif noinst_HEADERS += arch/x86/cpu_flags.h \ + arch/x86/odp_cpu.h \ arch/default/odp_cpu.h \ arch/default/odp_cpu_idling.h endif diff --git a/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h new file mode 100644 index 000000000..d1dbf36b8 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp/api/abi/atomic.h @@ -0,0 +1,12 @@ +/* Copyright (c) 2021, ARM Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifdef __ARM_FEATURE_ATOMICS +#define _ODP_LOCK_FREE_128BIT_ATOMICS +#endif + +#include <odp/api/abi-default/atomic.h> + diff --git a/platform/linux-generic/arch/aarch64/odp_atomic.h b/platform/linux-generic/arch/aarch64/odp_atomic.h index 8a0e7ce2b..dbeccebde 100644 --- a/platform/linux-generic/arch/aarch64/odp_atomic.h +++ b/platform/linux-generic/arch/aarch64/odp_atomic.h @@ -1,5 +1,4 @@ -/* Copyright (c) 2017, ARM Limited. All rights reserved. - * +/* Copyright (c) 2017-2021, ARM Limited * Copyright (c) 2017-2018, Linaro Limited * All rights reserved. * @@ -218,4 +217,54 @@ static inline __int128 __lockfree_load_16(__int128 *var, int mo) return old; } +#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS + +/** + * @internal + * Helper macro for lockless atomic CAS operations on 128-bit integers + * @param[in,out] atom Pointer to the 128-bit atomic variable + * @param oper CAS operation + * @param old_val Old value + * @param new_val New value to be swapped + * @return 1 for success and 0 for fail + */ +#define ATOMIC_CAS_OP_128(atom, oper, old_val, new_val, val) \ +({ \ + odp_u128_t _val; \ + odp_atomic_u128_t *_atom = atom; \ + odp_u128_t *_old_val = old_val; \ + odp_u128_t _new_val = new_val; \ + odp_u128_t *ptr = (odp_u128_t *)(_atom); \ + register uint64_t old0 __asm__ ("x0"); \ + register uint64_t old1 __asm__ ("x1"); \ + register uint64_t new0 __asm__ ("x2"); \ + register uint64_t new1 __asm__ ("x3"); \ + old0 = (uint64_t)(_old_val)->u64[0]; \ + old1 = (uint64_t)(_old_val)->u64[1]; \ + new0 = (uint64_t)(_new_val).u64[0]; \ + new1 = (uint64_t)(_new_val).u64[1]; \ + __asm__ volatile(oper " %[old0], %[old1], %[new0], %[new1], [%[ptr]]" \ + : [old0] "+r" (old0), [old1] "+r" (old1) \ + : [new0] "r" (new0), [new1] "r" (new1), \ + [ptr] "r" (ptr) \ + : "memory"); \ + _val.u64[0] = old0; \ + _val.u64[1] = old1; \ + val = _val; \ +}) + +#define ATOMIC_CAS_OP_128_NO_ORDER(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "casp", old_value, new_value, val) + +#define ATOMIC_CAS_OP_128_ACQ(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "caspa", old_value, new_value, val) + +#define ATOMIC_CAS_OP_128_REL(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "caspl", old_value, new_value, val) + +#define ATOMIC_CAS_OP_128_ACQ_REL(atom, old_value, new_value, val) \ + ATOMIC_CAS_OP_128(atom, "caspal", old_value, new_value, val) + +#endif + #endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H */ diff --git a/platform/linux-generic/arch/aarch64/odp_cpu.h b/platform/linux-generic/arch/aarch64/odp_cpu.h index 0da6e3c60..84bc4dffd 100644 --- a/platform/linux-generic/arch/aarch64/odp_cpu.h +++ b/platform/linux-generic/arch/aarch64/odp_cpu.h @@ -59,4 +59,10 @@ do { \ #include "odp_atomic.h" #include "odp_cpu_idling.h" +#ifdef __ARM_FEATURE_UNALIGNED +#define _ODP_UNALIGNED 1 +#else +#define _ODP_UNALIGNED 0 +#endif + #endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H */ diff --git a/platform/linux-generic/arch/aarch64/odp_cpu_idling.h b/platform/linux-generic/arch/aarch64/odp_cpu_idling.h index 8528c8591..a6cea8c63 100644 --- a/platform/linux-generic/arch/aarch64/odp_cpu_idling.h +++ b/platform/linux-generic/arch/aarch64/odp_cpu_idling.h @@ -30,12 +30,6 @@ static inline int wfe(void) return 1; } -static inline void doze(void) -{ - /* When using WFE do not stall the pipeline using other means */ - odp_cpu_pause(); -} - #define monitor128(addr, mo) lld((addr), (mo)) #define monitor64(addr, mo) ll64((addr), (mo)) #define monitor32(addr, mo) ll32((addr), (mo)) diff --git a/platform/linux-generic/arch/arm/odp_cpu.h b/platform/linux-generic/arch/arm/odp_cpu.h index d3d2de34f..82d47325f 100644 --- a/platform/linux-generic/arch/arm/odp_cpu.h +++ b/platform/linux-generic/arch/arm/odp_cpu.h @@ -52,4 +52,10 @@ static inline void _odp_dmb(void) #include "odp_atomic.h" #include "odp_cpu_idling.h" +#ifdef __ARM_FEATURE_UNALIGNED +#define _ODP_UNALIGNED 1 +#else +#define _ODP_UNALIGNED 0 +#endif + #endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H */ diff --git a/platform/linux-generic/arch/arm/odp_cpu_idling.h b/platform/linux-generic/arch/arm/odp_cpu_idling.h index 8528c8591..a6cea8c63 100644 --- a/platform/linux-generic/arch/arm/odp_cpu_idling.h +++ b/platform/linux-generic/arch/arm/odp_cpu_idling.h @@ -30,12 +30,6 @@ static inline int wfe(void) return 1; } -static inline void doze(void) -{ - /* When using WFE do not stall the pipeline using other means */ - odp_cpu_pause(); -} - #define monitor128(addr, mo) lld((addr), (mo)) #define monitor64(addr, mo) ll64((addr), (mo)) #define monitor32(addr, mo) ll32((addr), (mo)) diff --git a/platform/linux-generic/arch/default/odp_cpu.h b/platform/linux-generic/arch/default/odp_cpu.h index 18dd968fb..d8bc125c8 100644 --- a/platform/linux-generic/arch/default/odp_cpu.h +++ b/platform/linux-generic/arch/default/odp_cpu.h @@ -9,6 +9,10 @@ #ifndef ODP_DEFAULT_CPU_H_ #define ODP_DEFAULT_CPU_H_ +#ifndef _ODP_UNALIGNED +#define _ODP_UNALIGNED 0 +#endif + /****************************************************************************** * Atomics *****************************************************************************/ diff --git a/platform/linux-generic/arch/default/odp_cpu_idling.h b/platform/linux-generic/arch/default/odp_cpu_idling.h index 70b8cae6e..9d23ad20d 100644 --- a/platform/linux-generic/arch/default/odp_cpu_idling.h +++ b/platform/linux-generic/arch/default/odp_cpu_idling.h @@ -28,9 +28,4 @@ static inline int wfe(void) #define monitor32(addr, mo) __atomic_load_n((addr), (mo)) #define monitor8(addr, mo) __atomic_load_n((addr), (mo)) -static inline void doze(void) -{ - odp_cpu_pause(); -} - #endif diff --git a/platform/linux-generic/arch/x86/odp_cpu.h b/platform/linux-generic/arch/x86/odp_cpu.h new file mode 100644 index 000000000..8f8f22daf --- /dev/null +++ b/platform/linux-generic/arch/x86/odp_cpu.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2021, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_X86_CPU_H_ +#define ODP_X86_CPU_H_ + +#define _ODP_UNALIGNED 1 + +#include <default/odp_cpu.h> + +#endif diff --git a/platform/linux-generic/include-abi/odp/api/abi/atomic.h b/platform/linux-generic/include-abi/odp/api/abi/atomic.h index 955b99370..13c12a79f 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/atomic.h +++ b/platform/linux-generic/include-abi/odp/api/abi/atomic.h @@ -55,6 +55,31 @@ typedef struct ODP_ALIGNED(sizeof(uint64_t)) odp_atomic_u64_s { #endif +#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS + +/** + * @internal + * Atomic 128-bit unsigned integer + */ +typedef struct ODP_ALIGNED(sizeof(odp_u128_t)) odp_atomic_u128_s { + odp_u128_t v; /**< Actual storage for the atomic variable */ +} odp_atomic_u128_t; + +#else + +/** + * @internal + * Atomic 128-bit unsigned integer + */ +typedef struct ODP_ALIGNED(sizeof(odp_u128_t)) odp_atomic_u128_s { + odp_u128_t v; /**< Actual storage for the atomic variable */ + /* Some architectures do not support lock-free operations on 128-bit + * data types. We use a spin lock to ensure atomicity. */ + char lock; /**< Spin lock (if needed) used to ensure atomic access */ +} odp_atomic_u128_t; + +#endif + /** @ingroup odp_atomic * @{ */ diff --git a/platform/linux-generic/include/odp/api/plat/queue_inline_types.h b/platform/linux-generic/include/odp/api/plat/queue_inline_types.h index bbff4b412..e59c7f55a 100644 --- a/platform/linux-generic/include/odp/api/plat/queue_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/queue_inline_types.h @@ -51,6 +51,8 @@ typedef struct { void (*queue_param_init)(odp_queue_param_t *param); int (*queue_info)(odp_queue_t queue, odp_queue_info_t *info); void (*queue_print)(odp_queue_t queue); + void (*queue_print_all)(void); + } _odp_queue_api_fn_t; /** @endcond */ diff --git a/platform/linux-generic/include/odp_atomic_internal.h b/platform/linux-generic/include/odp_atomic_internal.h index 5ab4a89af..81280b1fa 100644 --- a/platform/linux-generic/include/odp_atomic_internal.h +++ b/platform/linux-generic/include/odp_atomic_internal.h @@ -27,13 +27,6 @@ extern "C" { #endif /** - * Pointer atomic type - */ -typedef struct ODP_ALIGNED(sizeof(void *)) { - void *v; /**< Actual storage for the atomic variable */ -} _odp_atomic_ptr_t; - -/** * Atomic flag (boolean) type * @Note this is not the same as a plain boolean type. * _odp_atomic_flag_t is guaranteed to be able to operate on atomically. @@ -58,562 +51,9 @@ typedef enum { _ODP_MEMMODEL_RLS = __ATOMIC_RELEASE, /** Acquire&release memory ordering, synchronize with acquire loads and release * stores in another (one other) thread */ - _ODP_MEMMODEL_ACQ_RLS = __ATOMIC_ACQ_REL, -/** Sequential consistent memory ordering, synchronize with acquire loads and - * release stores in all threads */ - _ODP_MEMMODEL_SC = __ATOMIC_SEQ_CST -} _odp_memmodel_t; - -/***************************************************************************** - * Operations on 32-bit atomics - * _odp_atomic_u32_load_mm - return current value - * _odp_atomic_u32_store_mm - no return value - * _odp_atomic_u32_xchg_mm - return old value - * _odp_atomic_u32_cmp_xchg_strong_mm - return bool - * _odp_atomic_u32_fetch_add_mm - return old value - * _odp_atomic_u32_add_mm - no return value - * _odp_atomic_u32_fetch_sub_mm - return old value - * _odp_atomic_u32_sub_mm - no return value - *****************************************************************************/ - -/** - * Atomic load of 32-bit atomic variable - * - * @param atom Pointer to a 32-bit atomic variable - * @param mmodel Memory ordering associated with the load operation - * - * @return Value of the variable - */ -static inline uint32_t _odp_atomic_u32_load_mm(const odp_atomic_u32_t *atom, - _odp_memmodel_t mmodel) -{ - return __atomic_load_n(&atom->v, mmodel); -} - -/** - * Atomic store to 32-bit atomic variable - * - * @param[out] atom Pointer to a 32-bit atomic variable - * @param val Value to store in the atomic variable - * @param mmodel Memory order associated with the store operation - */ -static inline void _odp_atomic_u32_store_mm(odp_atomic_u32_t *atom, - uint32_t val, - _odp_memmodel_t mmodel) -{ - __atomic_store_n(&atom->v, val, mmodel); -} - -/** - * Atomic exchange (swap) of 32-bit atomic variable - * - * @param[in,out] atom Pointer to a 32-bit atomic variable - * @param val New value to store in the atomic variable - * @param mmodel Memory order associated with the exchange operation - * - * @return Old value of the variable - */ -static inline uint32_t _odp_atomic_u32_xchg_mm(odp_atomic_u32_t *atom, - uint32_t val, - _odp_memmodel_t mmodel) - -{ - return __atomic_exchange_n(&atom->v, val, mmodel); -} - -/** - * Atomic compare and exchange (swap) of 32-bit atomic variable - * "Strong" semantics, will not fail spuriously. - * - * @param[in,out] atom Pointer to a 32-bit atomic variable - * @param[in,out] exp Pointer to expected value (updated on failure) - * @param val New value to write - * @param success Memory order associated with a successful compare-and-swap - * operation - * @param failure Memory order associated with a failed compare-and-swap - * operation - * - * @retval 1 exchange successul - * @retval 0 exchange failed and '*exp' updated with current value - */ -static inline int _odp_atomic_u32_cmp_xchg_strong_mm(odp_atomic_u32_t *atom, - uint32_t *exp, - uint32_t val, - _odp_memmodel_t success, - _odp_memmodel_t failure) -{ - return __atomic_compare_exchange_n(&atom->v, exp, val, - false/*strong*/, success, failure); -} - -/** - * Atomic fetch and add of 32-bit atomic variable - * - * @param[in,out] atom Pointer to a 32-bit atomic variable - * @param val Value to add to the atomic variable - * @param mmodel Memory order associated with the add operation - * - * @return Value of the atomic variable before the addition - */ -static inline uint32_t _odp_atomic_u32_fetch_add_mm(odp_atomic_u32_t *atom, - uint32_t val, - _odp_memmodel_t mmodel) -{ - return __atomic_fetch_add(&atom->v, val, mmodel); -} - -/** - * Atomic add of 32-bit atomic variable - * - * @param[in,out] atom Pointer to a 32-bit atomic variable - * @param val Value to add to the atomic variable - * @param mmodel Memory order associated with the add operation - */ -static inline void _odp_atomic_u32_add_mm(odp_atomic_u32_t *atom, uint32_t val, - _odp_memmodel_t mmodel) + _ODP_MEMMODEL_ACQ_RLS = __ATOMIC_ACQ_REL -{ - (void)__atomic_fetch_add(&atom->v, val, mmodel); -} - -/** - * Atomic fetch and subtract of 32-bit atomic variable - * - * @param[in,out] atom Pointer to a 32-bit atomic variable - * @param val Value to subtract from the atomic variable - * @param mmodel Memory order associated with the subtract operation - * - * @return Value of the atomic variable before the subtraction - */ -static inline uint32_t _odp_atomic_u32_fetch_sub_mm(odp_atomic_u32_t *atom, - uint32_t val, - _odp_memmodel_t mmodel) -{ - return __atomic_fetch_sub(&atom->v, val, mmodel); -} - -/** - * Atomic subtract of 32-bit atomic variable - * - * @param[in,out] atom Pointer to a 32-bit atomic variable - * @param val Value to subtract from the atomic variable - * @param mmodel Memory order associated with the subtract operation - */ -static inline void _odp_atomic_u32_sub_mm(odp_atomic_u32_t *atom, uint32_t val, - _odp_memmodel_t mmodel) - -{ - (void)__atomic_fetch_sub(&atom->v, val, mmodel); -} - -/***************************************************************************** - * Operations on 64-bit atomics - * _odp_atomic_u64_load_mm - return current value - * _odp_atomic_u64_store_mm - no return value - * _odp_atomic_u64_xchg_mm - return old value - * _odp_atomic_u64_cmp_xchg_strong_mm - return bool - * _odp_atomic_u64_fetch_add_mm - return old value - * _odp_atomic_u64_add_mm - no return value - * _odp_atomic_u64_fetch_sub_mm - return old value - * _odp_atomic_u64_sub_mm - no return value - *****************************************************************************/ - -/* Check if the compiler support lock-less atomic operations on 64-bit types */ -#ifdef ODP_ATOMIC_U64_LOCK -/** - * @internal - * Helper macro for lock-based atomic operations on 64-bit integers - * @param[in,out] atom Pointer to the 64-bit atomic variable - * @param expr Expression used update the variable. - * @param mm Memory order to use. - * @return The old value of the variable. - */ -#define ATOMIC_OP_MM(atom, expr, mm) \ -({ \ - uint64_t old_val; \ - /* Loop while lock is already taken, stop when lock becomes clear */ \ - while (__atomic_test_and_set(&(atom)->lock, \ - (mm) == _ODP_MEMMODEL_SC ? \ - __ATOMIC_SEQ_CST : __ATOMIC_ACQUIRE)) \ - (void)0; \ - old_val = (atom)->v; \ - (expr); /* Perform whatever update is desired */ \ - __atomic_clear(&(atom)->lock, \ - (mm) == _ODP_MEMMODEL_SC ? \ - __ATOMIC_SEQ_CST : __ATOMIC_RELEASE); \ - old_val; /* Return old value */ \ -}) - -/** - * Atomic load of 64-bit atomic variable - * - * @param atom Pointer to a 64-bit atomic variable - * @param mmodel Memory order associated with the load operation - * - * @return Value of the variable - */ -static inline uint64_t _odp_atomic_u64_load_mm(odp_atomic_u64_t *atom, - _odp_memmodel_t mmodel) -{ - return ATOMIC_OP_MM(atom, (void)0, mmodel); -} - -/** - * Atomic store to 64-bit atomic variable - * - * @param[out] atom Pointer to a 64-bit atomic variable - * @param val Value to write to the atomic variable - * @param mmodel Memory order associated with the store operation - */ -static inline void _odp_atomic_u64_store_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) -{ - (void)ATOMIC_OP_MM(atom, atom->v = val, mmodel); -} - -/** - * Atomic exchange (swap) of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val New value to write to the atomic variable - * @param mmodel Memory order associated with the exchange operation - * - * @return Old value of variable - */ -static inline uint64_t _odp_atomic_u64_xchg_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) - -{ - return ATOMIC_OP_MM(atom, atom->v = val, mmodel); -} - -/** - * Atomic compare and exchange (swap) of 64-bit atomic variable - * "Strong" semantics, will not fail spuriously. - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param[in,out] exp Pointer to expected value (updated on failure) - * @param val New value to write - * @param success Memory order associated with a successful compare-and-swap - * operation - * @param failure Memory order associated with a failed compare-and-swap - * operation - * - * @retval 1 exchange successful - * @retval 0 exchange failed and '*exp' updated with current value - */ -static inline int _odp_atomic_u64_cmp_xchg_strong_mm(odp_atomic_u64_t *atom, - uint64_t *exp, - uint64_t val, - _odp_memmodel_t success, - _odp_memmodel_t failure) -{ - /* Possibly we are a bit pessimistic with the memory models */ - odp_bool_t ret_succ; - /* Loop while lock is already taken, stop when lock becomes clear */ - while (__atomic_test_and_set(&(atom)->lock, - (success) == _ODP_MEMMODEL_SC ? - __ATOMIC_SEQ_CST : __ATOMIC_ACQUIRE)) - (void)0; - if (atom->v == *exp) { - atom->v = val; - ret_succ = 1; - } else { - *exp = atom->v; - ret_succ = 0; - } - __atomic_clear(&(atom)->lock, - (ret_succ ? success : failure) == _ODP_MEMMODEL_SC ? - __ATOMIC_SEQ_CST : __ATOMIC_RELEASE); - return ret_succ; -} - -/** - * Atomic fetch and add of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val Value to add to the atomic variable - * @param mmodel Memory order associated with the add operation - * - * @return Value of the atomic variable before the addition - */ -static inline uint64_t _odp_atomic_u64_fetch_add_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) -{ - return ATOMIC_OP_MM(atom, atom->v += val, mmodel); -} - -/** - * Atomic add of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val Value to add to the atomic variable - * @param mmodel Memory order associated with the add operation. - */ -static inline void _odp_atomic_u64_add_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) - -{ - (void)ATOMIC_OP_MM(atom, atom->v += val, mmodel); -} - -/** - * Atomic fetch and subtract of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val Value to subtract from the atomic variable - * @param mmodel Memory order associated with the subtract operation - * - * @return Value of the atomic variable before the subtraction - */ -static inline uint64_t _odp_atomic_u64_fetch_sub_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) -{ - return ATOMIC_OP_MM(atom, atom->v -= val, mmodel); -} - -/** - * Atomic subtract of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val Value to subtract from the atomic variable - * @param mmodel Memory order associated with the subtract operation - */ -static inline void _odp_atomic_u64_sub_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) - -{ - (void)ATOMIC_OP_MM(atom, atom->v -= val, mmodel); -} - -#undef ATOMIC_OP_MM - -#else /* ! ODP_ATOMIC_U64_LOCK */ - -/** - * Atomic load of 64-bit atomic variable - * - * @param atom Pointer to a 64-bit atomic variable - * @param mmodel Memory order associated with the load operation - * - * @return Value of the variable - */ -static inline uint64_t _odp_atomic_u64_load_mm(odp_atomic_u64_t *atom, - _odp_memmodel_t mmodel) -{ - return __atomic_load_n(&atom->v, mmodel); -} - -/** - * Atomic store to 64-bit atomic variable - * - * @param[out] atom Pointer to a 64-bit atomic variable - * @param val Value to write to the atomic variable - * @param mmodel Memory order associated with the store operation - */ -static inline void _odp_atomic_u64_store_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) -{ - __atomic_store_n(&atom->v, val, mmodel); -} - -/** - * Atomic exchange (swap) of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val New value to write to the atomic variable - * @param mmodel Memory order associated with the exchange operation - * - * @return Old value of variable - */ -static inline uint64_t _odp_atomic_u64_xchg_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) - -{ - return __atomic_exchange_n(&atom->v, val, mmodel); -} - -/** - * Atomic compare and exchange (swap) of 64-bit atomic variable - * "Strong" semantics, will not fail spuriously. - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param[in,out] exp Pointer to expected value (updated on failure) - * @param val New value to write - * @param success Memory order associated with a successful compare-and-swap - * operation - * @param failure Memory order associated with a failed compare-and-swap - * operation - * - * @retval 1 exchange successful - * @retval 0 exchange failed and '*exp' updated with current value - */ -static inline int _odp_atomic_u64_cmp_xchg_strong_mm(odp_atomic_u64_t *atom, - uint64_t *exp, - uint64_t val, - _odp_memmodel_t success, - _odp_memmodel_t failure) -{ - return __atomic_compare_exchange_n(&atom->v, exp, val, - false/*strong*/, success, failure); -} - -/** - * Atomic fetch and add of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val Value to add to the atomic variable - * @param mmodel Memory order associated with the add operation - * - * @return Value of the atomic variable before the addition - */ -static inline uint64_t _odp_atomic_u64_fetch_add_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) -{ - return __atomic_fetch_add(&atom->v, val, mmodel); -} - -/** - * Atomic add of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val Value to add to the atomic variable - * @param mmodel Memory order associated with the add operation. - */ -static inline void _odp_atomic_u64_add_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) - -{ - (void)__atomic_fetch_add(&atom->v, val, mmodel); -} - -/** - * Atomic fetch and subtract of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val Value to subtract from the atomic variable - * @param mmodel Memory order associated with the subtract operation - * - * @return Value of the atomic variable before the subtraction - */ -static inline uint64_t _odp_atomic_u64_fetch_sub_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) -{ - return __atomic_fetch_sub(&atom->v, val, mmodel); -} - -/** - * Atomic subtract of 64-bit atomic variable - * - * @param[in,out] atom Pointer to a 64-bit atomic variable - * @param val Value to subtract from the atomic variable - * @param mmodel Memory order associated with the subtract operation - */ -static inline void _odp_atomic_u64_sub_mm(odp_atomic_u64_t *atom, - uint64_t val, - _odp_memmodel_t mmodel) - -{ - (void)__atomic_fetch_sub(&atom->v, val, mmodel); -} - -#endif /* ! ODP_ATOMIC_U64_LOCK */ - -/***************************************************************************** - * Operations on pointer atomics - * _odp_atomic_ptr_init - no return value - * _odp_atomic_ptr_load - return current value - * _odp_atomic_ptr_store - no return value - * _odp_atomic_ptr_xchg - return old value - *****************************************************************************/ - -/** - * Initialization of pointer atomic variable - * - * @param[out] atom Pointer to a pointer atomic variable - * @param val Value to initialize the variable with - */ -static inline void _odp_atomic_ptr_init(_odp_atomic_ptr_t *atom, void *val) -{ - __atomic_store_n(&atom->v, val, __ATOMIC_RELAXED); -} - -/** - * Atomic load of pointer atomic variable - * - * @param atom Pointer to a pointer atomic variable - * @param mmodel Memory order associated with the load operation - * - * @return Value of the variable - */ -static inline void *_odp_atomic_ptr_load(const _odp_atomic_ptr_t *atom, - _odp_memmodel_t mmodel) -{ - return __atomic_load_n(&atom->v, mmodel); -} - -/** - * Atomic store to pointer atomic variable - * - * @param[out] atom Pointer to a pointer atomic variable - * @param val Value to write to the atomic variable - * @param mmodel Memory order associated with the store operation - */ -static inline void _odp_atomic_ptr_store(_odp_atomic_ptr_t *atom, void *val, - _odp_memmodel_t mmodel) -{ - __atomic_store_n(&atom->v, val, mmodel); -} - -/** - * Atomic exchange (swap) of pointer atomic variable - * - * @param[in,out] atom Pointer to a pointer atomic variable - * @param val New value to write - * @param mmodel Memory order associated with the exchange operation - * - * @return Old value of variable - */ -static inline void *_odp_atomic_ptr_xchg(_odp_atomic_ptr_t *atom, void *val, - _odp_memmodel_t mmodel) -{ - return __atomic_exchange_n(&atom->v, val, mmodel); -} - -/** - * Atomic compare and exchange (swap) of pointer atomic variable - * "Strong" semantics, will not fail spuriously. - * - * @param[in,out] atom Pointer to a pointer atomic variable - * @param[in,out] exp Pointer to expected value (updated on failure) - * @param val New value to write - * @param success Memory order associated with a successful compare-and-swap - * operation - * @param failure Memory order associated with a failed compare-and-swap - * operation - * - * @retval 1 exchange successful - * @retval 0 exchange failed and '*exp' updated with current value - */ -static inline int _odp_atomic_ptr_cmp_xchg_strong(_odp_atomic_ptr_t *atom, - void **exp, void *val, - _odp_memmodel_t success, - _odp_memmodel_t failure) -{ - return __atomic_compare_exchange_n(&atom->v, exp, val, - false/*strong*/, success, failure); -} +} _odp_memmodel_t; /***************************************************************************** * Operations on flag atomics diff --git a/platform/linux-generic/include/odp_chksum_internal.h b/platform/linux-generic/include/odp_chksum_internal.h new file mode 100644 index 000000000..5a134ae2d --- /dev/null +++ b/platform/linux-generic/include/odp_chksum_internal.h @@ -0,0 +1,189 @@ +/* Copyright (c) 2020, Nokia + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_CHKSUM_INTERNAL_H_ +#define ODP_CHKSUM_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/hints.h> +#include <odp/api/byteorder.h> +#include <odp_cpu.h> +#include <stdint.h> + +/* + * Compute the final Internet checksum (RFC 1071) based on a partial + * sum. A partial sum can be obtained e.g. by calling + * chksum_partial(). + */ +static inline uint16_t chksum_finalize(uint64_t sum) +{ + sum = (sum >> 32) + (sum & 0xffffffff); + sum = (sum >> 16) + (sum & 0xffff); + /* + * The final & 0xffff is intentionally omitted, the extra bits + * are discarded by the implicit cast to the return type. + */ + return (sum >> 16) + sum; +} + +/* + * Compute a partial checksum. Several partial checksums may be summed + * together. The final checksum may be obtained by calling + * chksum_finalize(). Parameter offset is the offset of this segment + * of data from the start of IP header. + * + * This implementation + * + * - Accepts unaligned data. + * + * - Accepts data at any byte offset from the start of IP header, + * including odd offsets. + * + * - Uses unaligned memory access only if available. + * + * - Is optimized (for skylake, cn96, a53) by trial and error. + * + * The following did not improve performance (in synthetic tests): + * + * - 2 or 4 sub-sums in the main loop (to break dependency chains). + * + * - Aligning to 8 bytes instead of 4 (for ldp instruction). This + * makes the main loop faster on a53 (only), but the extra + * conditional branch has its cost. + * + * - __builtin_assume_aligned(). + */ +static uint64_t chksum_partial(const void *addr, uint32_t len, uint32_t offset) +{ + const uint8_t *b; + const uint16_t *w; + const uint32_t *d; + uint64_t sum = 0; + + /* + * Offset is either even or odd, the rest of it doesn't + * matter. + */ + offset &= 1; + + if (_ODP_UNALIGNED) { + /* + * We have efficient unaligned access. Just read + * dwords starting at the given address. + */ + d = (const uint32_t *)addr; + } else { + /* + * We must avoid unaligned access, so align to 4 bytes + * by summing up the first up to 3 bytes. + */ + b = (const uint8_t *)addr; + + if (odp_unlikely((uintptr_t)b & 1) && len >= 1) { + /* + * Align to 2 bytes by handling an odd + * byte. Since addr is unaligned, the first + * byte goes into the second byte of the sum. + */ + sum += odp_cpu_to_be_16(*b++); + len -= 1; + + /* An odd byte negates the effect of offset. */ + offset ^= 1; + } + + /* + * This cast increases alignment, but it's OK, since + * we've made sure that the pointer value is aligned. + */ + w = (const uint16_t *)(uintptr_t)b; + + if ((uintptr_t)w & 2 && len >= 2) { + /* Align bytes by handling an odd word. */ + sum += *w++; + len -= 2; + } + + /* Increases alignment. */ + d = (const uint32_t *)(uintptr_t)w; + } + + while (len >= 32) { + /* 8 dwords or 32 bytes per round. */ + + sum += *d++; + sum += *d++; + sum += *d++; + sum += *d++; + + sum += *d++; + sum += *d++; + sum += *d++; + sum += *d++; + + len -= 32; + } + + /* Last up to 7 dwords. */ + switch (len >> 2) { + case 7: + sum += *d++; + /* FALLTHROUGH */ + case 6: + sum += *d++; + /* FALLTHROUGH */ + case 5: + sum += *d++; + /* FALLTHROUGH */ + case 4: + sum += *d++; + /* FALLTHROUGH */ + case 3: + sum += *d++; + /* FALLTHROUGH */ + case 2: + sum += *d++; + /* FALLTHROUGH */ + case 1: + sum += *d++; + /* FALLTHROUGH */ + default: + break; + } + + len &= 3; + + w = (const uint16_t *)d; + if (len > 1) { + /* Last word. */ + sum += *w++; + len -= 2; + } + + if (len) { + /* Last byte. */ + b = (const uint8_t *)w; + sum += odp_cpu_to_be_16((uint16_t)*b << 8); + } + + /* + * If offset is odd, our sum is byte-flipped and we need to + * flip odd and even bytes. + */ + if (odp_unlikely(offset)) + sum = ((sum & 0xff00ff00ff00ff) << 8) | ((sum & 0xff00ff00ff00ff00) >> 8); + + return sum; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp_timer_internal.h b/platform/linux-generic/include/odp_timer_internal.h index 17ebf2684..0c2a0fe4b 100644 --- a/platform/linux-generic/include/odp_timer_internal.h +++ b/platform/linux-generic/include/odp_timer_internal.h @@ -29,10 +29,13 @@ typedef struct { /* Requested expiration time */ uint64_t expiration; + /* User ptr inherited from parent timer */ - void *user_ptr; + const void *user_ptr; + /* Parent timer */ odp_timer_t timer; + } odp_timeout_hdr_t; /* A larger decrement value should be used after receiving events compared to diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4 index d4aeb455f..924346c1f 100644 --- a/platform/linux-generic/m4/configure.m4 +++ b/platform/linux-generic/m4/configure.m4 @@ -25,7 +25,7 @@ m4_include([platform/linux-generic/m4/odp_netmap.m4]) m4_include([platform/linux-generic/m4/odp_dpdk.m4]) ODP_SCHEDULER -AS_VAR_APPEND([PLAT_DEP_LIBS], ["${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT}"]) +AS_VAR_APPEND([PLAT_DEP_LIBS], ["${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS}"]) # Add text to the end of configure with platform specific settings. # Make sure it's aligned same as other lines in configure.ac. diff --git a/platform/linux-generic/odp_atomic.c b/platform/linux-generic/odp_atomic.c index 32ddc95c2..59253c645 100644 --- a/platform/linux-generic/odp_atomic.c +++ b/platform/linux-generic/odp_atomic.c @@ -1,10 +1,12 @@ /* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2021, ARM Limited * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ #include <odp/api/atomic.h> +#include <odp_cpu.h> int odp_atomic_lock_free_u64(odp_atomic_op_t *atomic_op) { @@ -24,3 +26,281 @@ int odp_atomic_lock_free_u64(odp_atomic_op_t *atomic_op) return 2; #endif } + +int odp_atomic_lock_free_u128(odp_atomic_op_t *atomic_op) +{ +#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS + if (atomic_op) { + atomic_op->all_bits = 0; + atomic_op->op.load = 1; + atomic_op->op.store = 1; + atomic_op->op.cas = 1; + } + + return 2; +#else + /* All operations have locks */ + if (atomic_op) + atomic_op->all_bits = 0; + + return 0; +#endif +} + +#ifdef _ODP_LOCK_FREE_128BIT_ATOMICS + +static void __atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t new_val) +{ + odp_u128_t old, val; + + old = atom->v; + + while (1) { + ATOMIC_CAS_OP_128_NO_ORDER(atom, &old, new_val, val); + + if ((val.u64[0] == old.u64[0]) && (val.u64[1] == old.u64[1])) + return; + + old = val; + } +} + +static odp_u128_t __atomic_load_u128(odp_atomic_u128_t *atom) +{ + odp_u128_t val, exp; + + exp.u64[0] = 0; + exp.u64[1] = 0; + ATOMIC_CAS_OP_128_NO_ORDER(atom, &exp, exp, val); + return val; +} + +static void __atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t new_val) +{ + odp_u128_t old, val; + + old = atom->v; + + while (1) { + ATOMIC_CAS_OP_128_NO_ORDER(atom, &old, new_val, val); + + if ((val.u64[0] == old.u64[0]) && (val.u64[1] == old.u64[1])) + return; + + old = val; + } +} + +static int __atomic_cas_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + int ret = 0; + odp_u128_t val; + + ATOMIC_CAS_OP_128_NO_ORDER(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + ret = 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return ret; +} + +static int __atomic_cas_acq_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + int ret = 0; + odp_u128_t val; + + ATOMIC_CAS_OP_128_ACQ(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + ret = 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return ret; +} + +static int __atomic_cas_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + int ret = 0; + odp_u128_t val; + + ATOMIC_CAS_OP_128_REL(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + ret = 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return ret; +} + +static int __atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, + odp_u128_t new_val) +{ + int ret = 0; + odp_u128_t val; + + ATOMIC_CAS_OP_128_ACQ_REL(atom, old_val, new_val, val); + + if ((val.u64[0] == old_val->u64[0]) && (val.u64[1] == old_val->u64[1])) + ret = 1; + + old_val->u64[0] = val.u64[0]; + old_val->u64[1] = val.u64[1]; + + return ret; +} + +#else /* Locked version */ + +/** + * @internal + * 128 bit store operation expression for the ATOMIC_OP macro + */ +#define ATOMIC_STORE_OP_128(new_val) \ +({ \ + (_atom)->v = (new_val); \ +}) + +/** + * @internal + * 128 bit CAS operation expression for the ATOMIC_OP macro + */ +#define ATOMIC_CAS_OP_128(ret_ptr, old_val, new_val) \ +({ \ + int *_ret_ptr = ret_ptr; \ + odp_u128_t *_old_val = old_val; \ + odp_u128_t _new_val = new_val; \ + if (((_atom)->v.u64[0] == (_old_val)->u64[0]) && \ + ((_atom)->v.u64[1] == (_old_val)->u64[1])) { \ + (_atom)->v = (_new_val); \ + *(_ret_ptr) = 1; \ + } else { \ + *(_ret_ptr) = 0; \ + } \ +}) + +/** + * @internal + * Helper macro for lock-based atomic operations on 128-bit integers + * @param[in,out] atom Pointer to the 128-bit atomic variable + * @param expr Expression used update the variable. + * @return The old value of the variable. + */ +#define ATOMIC_OP_128(atom, expr) \ +({ \ + odp_u128_t _old_val; \ + odp_atomic_u128_t *_atom = atom; \ + /* Loop while lock is already taken, stop when lock becomes clear */ \ + while (__atomic_test_and_set(&(_atom)->lock, __ATOMIC_ACQUIRE)) \ + (void)0; \ + _old_val = (_atom)->v; \ + (expr); /* Perform whatever update is desired */ \ + __atomic_clear(&(_atom)->lock, __ATOMIC_RELEASE); \ + _old_val; /* Return old value */ \ +}) + +static void __atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + atom->lock = 0; + ATOMIC_OP_128(atom, ATOMIC_STORE_OP_128(val)); +} + +static odp_u128_t __atomic_load_u128(odp_atomic_u128_t *atom) +{ + return ATOMIC_OP_128(atom, (void)0); +} + +static void __atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + ATOMIC_OP_128(atom, ATOMIC_STORE_OP_128(val)); +} + +static int __atomic_cas_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + int ret; + *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, + new_val)); + return ret; +} + +static int __atomic_cas_acq_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, + odp_u128_t new_val) +{ + int ret; + *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, + new_val)); + return ret; +} + +static int __atomic_cas_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, + odp_u128_t new_val) +{ + int ret; + *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, + new_val)); + return ret; +} + +static int __atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, + odp_u128_t new_val) +{ + int ret; + *old_val = ATOMIC_OP_128(atom, ATOMIC_CAS_OP_128(&ret, old_val, + new_val)); + return ret; +} + +#endif + +void odp_atomic_init_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + __atomic_init_u128(atom, val); +} + +odp_u128_t odp_atomic_load_u128(odp_atomic_u128_t *atom) +{ + return __atomic_load_u128(atom); +} + +void odp_atomic_store_u128(odp_atomic_u128_t *atom, odp_u128_t val) +{ + __atomic_store_u128(atom, val); +} + +int odp_atomic_cas_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + return __atomic_cas_u128(atom, old_val, new_val); +} + +int odp_atomic_cas_acq_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + return __atomic_cas_acq_u128(atom, old_val, new_val); +} + +int odp_atomic_cas_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + return __atomic_cas_rel_u128(atom, old_val, new_val); +} + +int odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, + odp_u128_t *old_val, odp_u128_t new_val) +{ + return __atomic_cas_acq_rel_u128(atom, old_val, new_val); +} diff --git a/platform/linux-generic/odp_chksum.c b/platform/linux-generic/odp_chksum.c index d018e33a3..a0336893e 100644 --- a/platform/linux-generic/odp_chksum.c +++ b/platform/linux-generic/odp_chksum.c @@ -6,31 +6,9 @@ #include <odp/api/chksum.h> #include <odp/api/std_types.h> +#include <odp_chksum_internal.h> -/* Simple implementation of ones complement sum. - * Based on RFC1071 and its errata. - */ uint16_t odp_chksum_ones_comp16(const void *p, uint32_t len) { - uint32_t sum = 0; - const uint16_t *data = p; - - while (len > 1) { - sum += *data++; - len -= 2; - } - - /* Add left-over byte, if any */ - if (len > 0) { - uint16_t left_over = 0; - - *(uint8_t *)&left_over = *(const uint8_t *)data; - sum += left_over; - } - - /* Fold 32-bit sum to 16 bits */ - while (sum >> 16) - sum = (sum & 0xffff) + (sum >> 16); - - return sum; + return chksum_finalize(chksum_partial(p, len, 0)); } diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index 5f452c598..bfa410b44 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -691,8 +691,7 @@ static int pmr_create_term(pmr_term_value_t *value, return -1; } - memset(&value->match.value, 0, MAX_PMR_TERM_SIZE); - memset(&value->match.mask, 0, MAX_PMR_TERM_SIZE); + memset(&value->match, 0, sizeof(value->match)); memcpy(&value->match.value, param->match.value, param->val_sz); memcpy(&value->match.mask, param->match.mask, param->val_sz); diff --git a/platform/linux-generic/odp_hash_crc_gen.c b/platform/linux-generic/odp_hash_crc_gen.c index 93ebe0e95..109d15420 100644 --- a/platform/linux-generic/odp_hash_crc_gen.c +++ b/platform/linux-generic/odp_hash_crc_gen.c @@ -66,15 +66,15 @@ static inline uint8_t reflect_u8(uint8_t byte) { uint8_t u8[8]; - u8[0] = (byte & (0x1 << 7)) >> 7; - u8[1] = (byte & (0x1 << 6)) >> 5; - u8[2] = (byte & (0x1 << 5)) >> 3; - u8[3] = (byte & (0x1 << 4)) >> 1; + u8[0] = (byte & (0x1u << 7)) >> 7; + u8[1] = (byte & (0x1u << 6)) >> 5; + u8[2] = (byte & (0x1u << 5)) >> 3; + u8[3] = (byte & (0x1u << 4)) >> 1; - u8[4] = (byte & (0x1 << 3)) << 1; - u8[5] = (byte & (0x1 << 2)) << 3; - u8[6] = (byte & (0x1 << 1)) << 5; - u8[7] = (byte & 0x1) << 7; + u8[4] = (byte & (0x1u << 3)) << 1; + u8[5] = (byte & (0x1u << 2)) << 3; + u8[6] = (byte & (0x1u << 1)) << 5; + u8[7] = (byte & 0x1u) << 7; return u8[0] | u8[1] | u8[2] | u8[3] | u8[4] | u8[5] | u8[6] | u8[7]; } @@ -84,10 +84,10 @@ static inline uint32_t reflect_u32(uint32_t u32) { uint8_t u8[4]; - u8[0] = reflect_u8((u32 & 0xff000000) >> 24); - u8[1] = reflect_u8((u32 & 0x00ff0000) >> 16); - u8[2] = reflect_u8((u32 & 0x0000ff00) >> 8); - u8[3] = reflect_u8(u32 & 0xff); + u8[0] = reflect_u8((u32 & 0xff000000u) >> 24); + u8[1] = reflect_u8((u32 & 0x00ff0000u) >> 16); + u8[2] = reflect_u8((u32 & 0x0000ff00u) >> 8); + u8[3] = reflect_u8(u32 & 0xffu); return (u8[3] << 24) | (u8[2] << 16) | (u8[1] << 8) | u8[0]; } @@ -97,9 +97,9 @@ static inline uint32_t reflect_u24(uint32_t u32) { uint8_t u8[4]; - u8[0] = reflect_u8((u32 & 0xff0000) >> 16); - u8[1] = reflect_u8((u32 & 0x00ff00) >> 8); - u8[2] = reflect_u8(u32 & 0xff); + u8[0] = reflect_u8((u32 & 0xff0000u) >> 16); + u8[1] = reflect_u8((u32 & 0x00ff00u) >> 8); + u8[2] = reflect_u8(u32 & 0xffu); return (u8[2] << 16) | (u8[1] << 8) | u8[0]; } @@ -109,8 +109,8 @@ static inline uint32_t reflect_u16(uint32_t u32) { uint8_t u8[4]; - u8[0] = reflect_u8((u32 & 0xff00) >> 8); - u8[1] = reflect_u8(u32 & 0xff); + u8[0] = reflect_u8((u32 & 0xff00u) >> 8); + u8[1] = reflect_u8(u32 & 0xffu); return (u8[1] << 8) | u8[0]; } @@ -128,8 +128,8 @@ static inline void crc_table_gen(uint32_t poly, int reflect, int width) crc_table->reflect = reflect; shift = width - 8; - mask = 0xffffffff >> (32 - width); - msb = 0x1 << (width - 1); + mask = 0xffffffffu >> (32 - width); + msb = 0x1u << (width - 1); if (reflect) { if (width == 32) @@ -145,7 +145,7 @@ static inline void crc_table_gen(uint32_t poly, int reflect, int width) crc = i; for (bit = 0; bit < 8; bit++) { - if (crc & 0x1) + if (crc & 0x1u) crc = poly ^ (crc >> 1); else crc = crc >> 1; @@ -173,7 +173,7 @@ static inline uint32_t crc_calc(const uint8_t *data, uint32_t data_len, uint32_t mask; shift = width - 8; - mask = 0xffffffff >> (32 - width); + mask = 0xffffffffu >> (32 - width); crc = init_val; @@ -181,7 +181,7 @@ static inline uint32_t crc_calc(const uint8_t *data, uint32_t data_len, byte = data[i]; if (reflect) { - crc = crc_table->crc[(crc ^ byte) & 0xff] ^ (crc >> 8); + crc = crc_table->crc[(crc ^ byte) & 0xffu] ^ (crc >> 8); } else { crc = crc_table->crc[(crc >> shift) ^ byte] ^ (crc << 8); diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index f0bbaf167..f8746f812 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -136,6 +136,7 @@ int odp_ipsec_capability(odp_ipsec_capability_t *capa) capa->max_queues = queue_capa.max_queues; capa->inline_ipsec_tm = ODP_SUPPORT_NO; + capa->test.sa_operations.seq_num = 1; return 0; } @@ -2040,6 +2041,26 @@ err: return in_pkt; } +int odp_ipsec_test_sa_update(odp_ipsec_sa_t sa, + odp_ipsec_test_sa_operation_t sa_op, + const odp_ipsec_test_sa_param_t *sa_param) +{ + ipsec_sa_t *ipsec_sa; + + ipsec_sa = _odp_ipsec_sa_entry_from_hdl(sa); + ODP_ASSERT(NULL != ipsec_sa); + + switch (sa_op) { + case ODP_IPSEC_TEST_SA_UPDATE_SEQ_NUM: + odp_atomic_store_u64(&ipsec_sa->hot.out.seq, sa_param->seq_num); + break; + default: + return -1; + } + + return 0; +} + int odp_ipsec_result(odp_ipsec_packet_result_t *result, odp_packet_t packet) { odp_ipsec_packet_result_t *res; @@ -2142,7 +2163,7 @@ void odp_ipsec_print(void) { ODP_PRINT("\nIPSEC print\n"); ODP_PRINT("-----------\n"); - ODP_PRINT(" max number of SA %u\n", ipsec_config->max_num_sa); + ODP_PRINT(" max number of SA %u\n\n", ipsec_config->max_num_sa); } void odp_ipsec_sa_print(odp_ipsec_sa_t sa) @@ -2151,5 +2172,5 @@ void odp_ipsec_sa_print(odp_ipsec_sa_t sa) ODP_PRINT("\nIPSEC SA print\n"); ODP_PRINT("--------------\n"); - ODP_PRINT(" SPI %u\n", ipsec_sa->spi); + ODP_PRINT(" SPI %u\n\n", ipsec_sa->spi); } diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c index 0e021361c..7cae32703 100644 --- a/platform/linux-generic/odp_ipsec_sad.c +++ b/platform/linux-generic/odp_ipsec_sad.c @@ -318,6 +318,8 @@ void odp_ipsec_sa_param_init(odp_ipsec_sa_param_t *param) { memset(param, 0, sizeof(odp_ipsec_sa_param_t)); param->dest_queue = ODP_QUEUE_INVALID; + param->outbound.tunnel.ipv4.ttl = 255; + param->outbound.tunnel.ipv6.hlimit = 255; } /* Return IV length required for the cipher for IPsec use */ @@ -1007,8 +1009,9 @@ static void ipsec_out_sa_info(ipsec_sa_t *ipsec_sa, odp_ipsec_sa_info_t *sa_info static void ipsec_in_sa_info(ipsec_sa_t *ipsec_sa, odp_ipsec_sa_info_t *sa_info) { - if (ipsec_sa->param.mode == ODP_IPSEC_MODE_TUNNEL) { - uint8_t *dst = sa_info->inbound.lookup_param.dst_addr; + uint8_t *dst = sa_info->inbound.lookup_param.dst_addr; + + if (ipsec_sa->lookup_mode == ODP_IPSEC_LOOKUP_DSTADDR_SPI) { if (ipsec_sa->param.inbound.lookup_param.ip_version == ODP_IPSEC_IPV4) @@ -1018,8 +1021,8 @@ static void ipsec_in_sa_info(ipsec_sa_t *ipsec_sa, odp_ipsec_sa_info_t *sa_info) memcpy(dst, &ipsec_sa->in.lookup_dst_ipv6, ODP_IPV6_ADDR_SIZE); - sa_info->param.inbound.lookup_param.dst_addr = dst; } + sa_info->param.inbound.lookup_param.dst_addr = dst; if (ipsec_sa->antireplay) { sa_info->inbound.antireplay_ws = IPSEC_ANTIREPLAY_WS; diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 2fe7fe67d..752ff8416 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -11,6 +11,7 @@ #include <odp/api/plat/packet_inlines.h> #include <odp_packet_internal.h> #include <odp_debug_internal.h> +#include <odp_chksum_internal.h> #include <odp_errno_define.h> #include <odp/api/hints.h> #include <odp/api/byteorder.h> @@ -1785,73 +1786,12 @@ int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt) return dst_uarea_size < src_uarea_size; } -/* Simple implementation of ones complement sum. - * Based on RFC1071 and its errata. - */ -typedef union { - uint16_t w; - uint8_t b[2]; -} swap_buf_t; - -static uint32_t segment_sum16_32(const uint8_t *p, - uint32_t len, - uint32_t offset) - +static uint64_t packet_sum_partial(odp_packet_hdr_t *pkt_hdr, + uint32_t l3_offset, + uint32_t offset, + uint32_t len) { - uint32_t sum = 0; - - /* Include second part of 16-bit short word split between segments */ - if (len > 0 && (offset % 2)) { - swap_buf_t sw; - - sw.b[0] = 0; - sw.b[1] = *p++; - sum = sw.w; - len--; - } - - /* - * If pointer is 16-bit aligned, we can do fast path calculation. - * If it is not, we sum hi and lo bytes separately and then sum them. - */ - if ((uintptr_t)p % 2) { - uint32_t sum1 = 0, sum2 = 0; - - while (len > 1) { - sum1 += *p++; - sum2 += *p++; - len -= 2; - } -#if (ODP_BYTE_ORDER == ODP_BIG_ENDIAN) - sum += sum2 + (sum1 << 8); -#else - sum += sum1 + (sum2 << 8); -#endif - } else { - while (len > 1) { - sum += *(const uint16_t *)(uintptr_t)p; - p += 2; - len -= 2; - } - } - - /* Add left-over byte, if any */ - if (len > 0) { - swap_buf_t sw; - - sw.b[0] = *p; - sw.b[1] = 0; - sum += sw.w; - } - - return sum; -} - -static uint32_t packet_sum16_32(odp_packet_hdr_t *pkt_hdr, - uint32_t offset, - uint32_t len) -{ - uint32_t sum = 0; + uint64_t sum = 0; if (offset + len > pkt_hdr->frame_len) return 0; @@ -1863,7 +1803,7 @@ static uint32_t packet_sum16_32(odp_packet_hdr_t *pkt_hdr, if (seglen > len) seglen = len; - sum += segment_sum16_32(mapaddr, seglen, offset); + sum += chksum_partial(mapaddr, seglen, offset - l3_offset); len -= seglen; offset += seglen; } @@ -1871,20 +1811,14 @@ static uint32_t packet_sum16_32(odp_packet_hdr_t *pkt_hdr, return sum; } -static uint16_t packet_sum_ones_comp16(odp_packet_hdr_t *pkt_hdr, - uint32_t offset, - uint32_t len, - uint32_t l4_part_sum) +static inline uint16_t packet_sum(odp_packet_hdr_t *pkt_hdr, + uint32_t l3_offset, + uint32_t offset, + uint32_t len, + uint64_t sum) { - uint32_t sum = l4_part_sum; - - sum += packet_sum16_32(pkt_hdr, offset, len); - - /* Not more than two additions */ - sum = (sum & 0xffff) + (sum >> 16); - sum = (sum & 0xffff) + (sum >> 16); - - return sum; + sum += packet_sum_partial(pkt_hdr, l3_offset, offset, len); + return chksum_finalize(sum); } static uint32_t packet_sum_crc32c(odp_packet_hdr_t *pkt_hdr, @@ -1999,7 +1933,7 @@ error: static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, odp_proto_chksums_t chksums, - uint32_t *l4_part_sum) + uint64_t *l4_part_sum) { const _odp_ipv4hdr_t *ipv4 = (const _odp_ipv4hdr_t *)*parseptr; uint32_t dstaddr = odp_be_to_cpu_32(ipv4->dst_addr); @@ -2017,7 +1951,7 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, if (chksums.chksum.ipv4) { prs->input_flags.l3_chksum_done = 1; - if (odp_chksum_ones_comp16(ipv4, ihl * 4) != 0xffff) { + if (chksum_finalize(chksum_partial(ipv4, ihl * 4, 0)) != 0xffff) { prs->flags.ip_err = 1; prs->flags.l3_chksum_err = 1; return 0; @@ -2028,8 +1962,8 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += ihl * 4; if (chksums.chksum.udp || chksums.chksum.tcp) - *l4_part_sum = segment_sum16_32((const uint8_t *)&ipv4->src_addr, - 2 * _ODP_IPV4ADDR_LEN, 0); + *l4_part_sum = chksum_partial((const uint8_t *)&ipv4->src_addr, + 2 * _ODP_IPV4ADDR_LEN, 0); if (odp_unlikely(ihl > _ODP_IPV4HDR_IHL_MIN)) prs->input_flags.ipopt = 1; @@ -2059,7 +1993,7 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, uint32_t *offset, uint32_t frame_len, uint32_t seg_len, odp_proto_chksums_t chksums, - uint32_t *l4_part_sum) + uint64_t *l4_part_sum) { const _odp_ipv6hdr_t *ipv6 = (const _odp_ipv6hdr_t *)*parseptr; const _odp_ipv6hdr_ext_t *ipv6ext; @@ -2083,8 +2017,8 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, *parseptr += sizeof(_odp_ipv6hdr_t); if (chksums.chksum.udp || chksums.chksum.tcp) - *l4_part_sum = segment_sum16_32((const uint8_t *)&ipv6->src_addr, - 2 * _ODP_IPV6ADDR_LEN, 0); + *l4_part_sum = chksum_partial((const uint8_t *)&ipv6->src_addr, + 2 * _ODP_IPV6ADDR_LEN, 0); /* Skip past any IPv6 extension headers */ if (ipv6->next_hdr == _ODP_IPPROTO_HOPOPTS || @@ -2127,7 +2061,7 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr, static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t tcp_len, odp_proto_chksums_t chksums, - uint32_t *l4_part_sum) + uint64_t *l4_part_sum) { const _odp_tcphdr_t *tcp = (const _odp_tcphdr_t *)*parseptr; uint32_t len = tcp->hl * 4; @@ -2153,7 +2087,7 @@ static inline void parse_tcp(packet_parser_t *prs, const uint8_t **parseptr, */ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, odp_proto_chksums_t chksums, - uint32_t *l4_part_sum) + uint64_t *l4_part_sum) { const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr; uint32_t udplen = odp_be_to_cpu_16(udp->length); @@ -2200,7 +2134,7 @@ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr, static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr, uint16_t sctp_len, odp_proto_chksums_t chksums, - uint32_t *l4_part_sum) + uint64_t *l4_part_sum) { if (odp_unlikely(sctp_len < sizeof(_odp_sctphdr_t))) { prs->flags.sctp_err = 1; @@ -2228,7 +2162,7 @@ int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr, uint32_t frame_len, uint32_t seg_len, int layer, uint16_t ethtype, odp_proto_chksums_t chksums, - uint32_t *l4_part_sum) + uint64_t *l4_part_sum) { uint8_t ip_proto; @@ -2341,7 +2275,7 @@ int _odp_packet_parse_common(packet_parser_t *prs, const uint8_t *ptr, uint32_t offset; uint16_t ethtype; const uint8_t *parseptr; - uint32_t l4_part_sum; + uint64_t l4_part_sum; parseptr = ptr; offset = 0; @@ -2378,7 +2312,7 @@ static inline int packet_ipv4_chksum(odp_packet_t pkt, if (odp_unlikely(res < 0)) return res; - *chksum = ~odp_chksum_ones_comp16(buf, nleft); + *chksum = ~chksum_finalize(chksum_partial(buf, nleft, 0)); return 0; } @@ -2426,7 +2360,7 @@ static int _odp_packet_tcp_udp_chksum_insert(odp_packet_t pkt, uint16_t proto) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); uint32_t zero = 0; - uint32_t sum; + uint64_t sum; uint16_t l3_ver; uint16_t chksum; uint32_t chksum_offset; @@ -2439,15 +2373,17 @@ static int _odp_packet_tcp_udp_chksum_insert(odp_packet_t pkt, uint16_t proto) odp_packet_copy_to_mem(pkt, pkt_hdr->p.l3_offset, 2, &l3_ver); if (_ODP_IPV4HDR_VER(l3_ver) == _ODP_IPV4) - sum = packet_sum16_32(pkt_hdr, - pkt_hdr->p.l3_offset + - _ODP_IPV4ADDR_OFFSSET, - 2 * _ODP_IPV4ADDR_LEN); + sum = packet_sum_partial(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l3_offset + + _ODP_IPV4ADDR_OFFSSET, + 2 * _ODP_IPV4ADDR_LEN); else - sum = packet_sum16_32(pkt_hdr, - pkt_hdr->p.l3_offset + - _ODP_IPV6ADDR_OFFSSET, - 2 * _ODP_IPV6ADDR_LEN); + sum = packet_sum_partial(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l3_offset + + _ODP_IPV6ADDR_OFFSSET, + 2 * _ODP_IPV6ADDR_LEN); #if ODP_BYTE_ORDER == ODP_BIG_ENDIAN sum += proto; #else @@ -2459,24 +2395,22 @@ static int _odp_packet_tcp_udp_chksum_insert(odp_packet_t pkt, uint16_t proto) pkt_hdr->p.l4_offset); chksum_offset = pkt_hdr->p.l4_offset + _ODP_UDP_CSUM_OFFSET; } else { - sum += packet_sum16_32(pkt_hdr, - pkt_hdr->p.l4_offset + - _ODP_UDP_LEN_OFFSET, - 2); + sum += packet_sum_partial(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l4_offset + + _ODP_UDP_LEN_OFFSET, + 2); chksum_offset = pkt_hdr->p.l4_offset + _ODP_UDP_CSUM_OFFSET; } odp_packet_copy_from_mem(pkt, chksum_offset, 2, &zero); - sum += packet_sum16_32(pkt_hdr, - pkt_hdr->p.l4_offset, - pkt_hdr->frame_len - - pkt_hdr->p.l4_offset); - - /* Not more than two additions */ - sum = (sum & 0xffff) + (sum >> 16); - sum = (sum & 0xffff) + (sum >> 16); + sum += packet_sum_partial(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l4_offset, + pkt_hdr->frame_len - + pkt_hdr->p.l4_offset); - chksum = ~sum; + chksum = ~chksum_finalize(sum); if (proto == _ODP_IPPROTO_UDP && chksum == 0) chksum = 0xffff; @@ -2538,18 +2472,19 @@ int _odp_packet_sctp_chksum_insert(odp_packet_t pkt) static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, odp_proto_chksums_t chksums, - uint32_t l4_part_sum) + uint64_t l4_part_sum) { /* UDP chksum == 0 case is covered in parse_udp() */ if (chksums.chksum.udp && pkt_hdr->p.input_flags.udp && !pkt_hdr->p.input_flags.ipfrag && !pkt_hdr->p.input_flags.udp_chksum_zero) { - uint16_t sum = ~packet_sum_ones_comp16(pkt_hdr, - pkt_hdr->p.l4_offset, - pkt_hdr->frame_len - - pkt_hdr->p.l4_offset, - l4_part_sum); + uint16_t sum = ~packet_sum(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l4_offset, + pkt_hdr->frame_len - + pkt_hdr->p.l4_offset, + l4_part_sum); pkt_hdr->p.input_flags.l4_chksum_done = 1; if (sum != 0) { @@ -2562,11 +2497,12 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr, if (chksums.chksum.tcp && pkt_hdr->p.input_flags.tcp && !pkt_hdr->p.input_flags.ipfrag) { - uint16_t sum = ~packet_sum_ones_comp16(pkt_hdr, - pkt_hdr->p.l4_offset, - pkt_hdr->frame_len - - pkt_hdr->p.l4_offset, - l4_part_sum); + uint16_t sum = ~packet_sum(pkt_hdr, + pkt_hdr->p.l3_offset, + pkt_hdr->p.l4_offset, + pkt_hdr->frame_len - + pkt_hdr->p.l4_offset, + l4_part_sum); pkt_hdr->p.input_flags.l4_chksum_done = 1; if (sum != 0) { @@ -2613,7 +2549,7 @@ int _odp_packet_parse_layer(odp_packet_hdr_t *pkt_hdr, const uint8_t *base = packet_data(pkt_hdr); uint32_t offset = 0; uint16_t ethtype; - uint32_t l4_part_sum = 0; + uint64_t l4_part_sum = 0; int rc; if (odp_unlikely(layer == ODP_PROTO_LAYER_NONE)) @@ -2649,7 +2585,7 @@ int odp_packet_parse(odp_packet_t pkt, uint32_t offset, odp_proto_layer_t layer = param->last_layer; int ret; uint16_t ethtype; - uint32_t l4_part_sum = 0; + uint64_t l4_part_sum = 0; if (proto == ODP_PROTO_NONE || layer == ODP_PROTO_LAYER_NONE) return -1; diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 9363b315a..442aa0d94 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -1656,8 +1656,7 @@ void odp_pktio_print(odp_pktio_t hdl) return; } - len += snprintf(&str[len], n - len, - "pktio\n"); + len += snprintf(&str[len], n - len, "Pktio info\n----------\n"); len += snprintf(&str[len], n - len, " name %s\n", entry->s.name); len += snprintf(&str[len], n - len, diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 0ffad0807..2d2ed3c7f 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -767,6 +767,97 @@ static void queue_print(odp_queue_t handle) UNLOCK(queue); } +static void queue_print_all(void) +{ + uint32_t i, index, len, max_len; + const char *name; + int status; + odp_queue_type_t type; + odp_nonblocking_t blocking; + odp_queue_op_mode_t enq_mode; + odp_queue_op_mode_t deq_mode; + odp_queue_order_t order; + const char *status_str; + const char *bl_str; + char type_c, enq_c, deq_c, order_c, sync_c; + const int col_width = 24; + int prio = 0; + odp_schedule_sync_t sync = ODP_SCHED_SYNC_PARALLEL; + + ODP_PRINT("\nList of all queues\n"); + ODP_PRINT("------------------\n"); + ODP_PRINT(" idx %-*s type stat blk enq deq ord len max_len sync prio\n", col_width, "name"); + + for (i = 0; i < CONFIG_MAX_QUEUES; i++) { + queue_entry_t *queue = qentry_from_index(i); + + if (queue->s.status < QUEUE_STATUS_READY) + continue; + + LOCK(queue); + + status = queue->s.status; + index = queue->s.index; + name = queue->s.name; + type = queue->s.type; + blocking = queue->s.param.nonblocking; + enq_mode = queue->s.param.enq_mode; + deq_mode = queue->s.param.deq_mode; + order = queue->s.param.order; + + if (queue->s.queue_lf) { + len = _odp_queue_lf_length(queue->s.queue_lf); + max_len = _odp_queue_lf_max_length(); + } else if (queue->s.spsc) { + len = ring_spsc_length(&queue->s.ring_spsc); + max_len = queue->s.ring_mask + 1; + } else if (type == ODP_QUEUE_TYPE_SCHED) { + len = ring_st_length(&queue->s.ring_st); + max_len = queue->s.ring_mask + 1; + prio = queue->s.param.sched.prio; + sync = queue->s.param.sched.sync; + } else { + len = ring_mpmc_length(&queue->s.ring_mpmc); + max_len = queue->s.ring_mask + 1; + } + + UNLOCK(queue); + + if (status < QUEUE_STATUS_READY) + continue; + + status_str = (status == QUEUE_STATUS_READY) ? "R" : + ((status == QUEUE_STATUS_SCHED) ? "S" : "NS"); + + type_c = (type == ODP_QUEUE_TYPE_PLAIN) ? 'P' : 'S'; + + bl_str = (blocking == ODP_BLOCKING) ? "B" : + ((blocking == ODP_NONBLOCKING_LF) ? "LF" : "WF"); + + enq_c = (enq_mode == ODP_QUEUE_OP_MT) ? 'S' : + ((enq_mode == ODP_QUEUE_OP_MT_UNSAFE) ? 'U' : 'D'); + + deq_c = (deq_mode == ODP_QUEUE_OP_MT) ? 'S' : + ((deq_mode == ODP_QUEUE_OP_MT_UNSAFE) ? 'U' : 'D'); + + order_c = (order == ODP_QUEUE_ORDER_KEEP) ? 'K' : 'I'; + + ODP_PRINT("%4u %-*s %c %2s %2s", index, col_width, name, type_c, + status_str, bl_str); + ODP_PRINT(" %c %c %c %6u %6u", enq_c, deq_c, order_c, len, max_len); + + if (type == ODP_QUEUE_TYPE_SCHED) { + sync_c = (sync == ODP_SCHED_SYNC_PARALLEL) ? 'P' : + ((sync == ODP_SCHED_SYNC_ATOMIC) ? 'A' : 'O'); + ODP_PRINT(" %c %4i", sync_c, prio); + } + + ODP_PRINT("\n"); + } + + ODP_PRINT("\n"); +} + static inline int _sched_queue_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[], int num) { @@ -1137,7 +1228,9 @@ _odp_queue_api_fn_t _odp_queue_basic_api = { .queue_to_u64 = queue_to_u64, .queue_param_init = queue_param_init, .queue_info = queue_info, - .queue_print = queue_print + .queue_print = queue_print, + .queue_print_all = queue_print_all + }; /* Functions towards internal components */ diff --git a/platform/linux-generic/odp_queue_if.c b/platform/linux-generic/odp_queue_if.c index d4b1c550c..9ebd5db86 100644 --- a/platform/linux-generic/odp_queue_if.c +++ b/platform/linux-generic/odp_queue_if.c @@ -88,7 +88,7 @@ uint64_t odp_queue_to_u64(odp_queue_t hdl) void odp_queue_param_init(odp_queue_param_t *param) { - return _odp_queue_api->queue_param_init(param); + _odp_queue_api->queue_param_init(param); } int odp_queue_info(odp_queue_t queue, odp_queue_info_t *info) @@ -98,7 +98,12 @@ int odp_queue_info(odp_queue_t queue, odp_queue_info_t *info) void odp_queue_print(odp_queue_t queue) { - return _odp_queue_api->queue_print(queue); + _odp_queue_api->queue_print(queue); +} + +void odp_queue_print_all(void) +{ + _odp_queue_api->queue_print_all(); } int _odp_queue_init_global(void) diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c index 02bf9fbc2..d70e174e0 100644 --- a/platform/linux-generic/odp_queue_scalable.c +++ b/platform/linux-generic/odp_queue_scalable.c @@ -15,6 +15,7 @@ #include <odp/api/sync.h> #include <odp/api/plat/sync_inlines.h> #include <odp/api/traffic_mngr.h> +#include <odp/api/cpu.h> #include <odp_config_internal.h> #include <odp_debug_internal.h> @@ -465,7 +466,7 @@ static int queue_destroy(odp_queue_t handle) sevl(); while (wfe() && monitor32((uint32_t *)&q->qschst.numevts, __ATOMIC_RELAXED) != 0) - doze(); + odp_cpu_pause(); } if (q->schedq != NULL) { @@ -573,7 +574,7 @@ static inline int _odp_queue_enq(sched_elem_t *q, sevl(); while (wfe() && monitor32(&q->cons_write, __ATOMIC_RELAXED) != old_write) - doze(); + odp_cpu_pause(); } /* Signal consumers that events are available (release events) @@ -803,7 +804,7 @@ inline int _odp_queue_deq(sched_elem_t *q, odp_buffer_hdr_t *buf_hdr[], int num) sevl(); while (wfe() && monitor32(&q->prod_read, __ATOMIC_RELAXED) != old_read) - doze(); + odp_cpu_pause(); } /* Signal producers that empty slots are available @@ -1013,6 +1014,76 @@ static void queue_print(odp_queue_t handle) UNLOCK(&queue->s.lock); } +static void queue_print_all(void) +{ + uint32_t i, index; + const char *name; + int status; + odp_queue_type_t type; + odp_nonblocking_t blocking; + odp_queue_op_mode_t enq_mode; + odp_queue_op_mode_t deq_mode; + odp_queue_order_t order; + const char *bl_str; + char type_c, enq_c, deq_c, order_c, sync_c; + const int col_width = 24; + int prio = 0; + odp_schedule_sync_t sync = ODP_SCHED_SYNC_PARALLEL; + + ODP_PRINT("\nList of all queues\n"); + ODP_PRINT("------------------\n"); + ODP_PRINT(" idx %-*s type blk enq deq ord sync prio\n", col_width, "name"); + + for (i = 0; i < CONFIG_MAX_QUEUES; i++) { + queue_entry_t *queue = &queue_tbl->queue[i]; + + if (queue->s.status != QUEUE_STATUS_READY) + continue; + + LOCK(&queue->s.lock); + + status = queue->s.status; + index = queue->s.index; + name = queue->s.name; + type = queue->s.type; + blocking = queue->s.param.nonblocking; + enq_mode = queue->s.param.enq_mode; + deq_mode = queue->s.param.deq_mode; + order = queue->s.param.order; + + UNLOCK(&queue->s.lock); + + if (status != QUEUE_STATUS_READY) + continue; + + type_c = (type == ODP_QUEUE_TYPE_PLAIN) ? 'P' : 'S'; + + bl_str = (blocking == ODP_BLOCKING) ? "B" : + ((blocking == ODP_NONBLOCKING_LF) ? "LF" : "WF"); + + enq_c = (enq_mode == ODP_QUEUE_OP_MT) ? 'S' : + ((enq_mode == ODP_QUEUE_OP_MT_UNSAFE) ? 'U' : 'D'); + + deq_c = (deq_mode == ODP_QUEUE_OP_MT) ? 'S' : + ((deq_mode == ODP_QUEUE_OP_MT_UNSAFE) ? 'U' : 'D'); + + order_c = (order == ODP_QUEUE_ORDER_KEEP) ? 'K' : 'I'; + + ODP_PRINT("%4u %-*s %c %2s", index, col_width, name, type_c, bl_str); + ODP_PRINT(" %c %c %c", enq_c, deq_c, order_c); + + if (type == ODP_QUEUE_TYPE_SCHED) { + sync_c = (sync == ODP_SCHED_SYNC_PARALLEL) ? 'P' : + ((sync == ODP_SCHED_SYNC_ATOMIC) ? 'A' : 'O'); + ODP_PRINT(" %c %4i", sync_c, prio); + } + + ODP_PRINT("\n"); + } + + ODP_PRINT("\n"); +} + static uint64_t queue_to_u64(odp_queue_t hdl) { return _odp_pri(hdl); @@ -1099,7 +1170,8 @@ _odp_queue_api_fn_t _odp_queue_scalable_api = { .queue_to_u64 = queue_to_u64, .queue_param_init = queue_param_init, .queue_info = queue_info, - .queue_print = queue_print + .queue_print = queue_print, + .queue_print_all = queue_print_all }; /* Functions towards internal components */ diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 0396123f3..32a21442d 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -1633,7 +1633,7 @@ static int schedule_num_grps(void) static void schedule_get_config(schedule_config_t *config) { - *config = *(&sched->config_if); + *config = sched->config_if; }; static int schedule_capability(odp_schedule_capability_t *capa) diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 4da5c378d..2ba40256b 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -227,7 +227,7 @@ void _odp_sched_update_enq(sched_elem_t *q, uint32_t actual) while (wfe() && monitor8(&q->qschst.cur_ticket, __ATOMIC_ACQUIRE) != ticket) - doze(); + odp_cpu_pause(); } /* Enqueue at end of scheduler queue */ /* We are here because of empty-to-non-empty transition @@ -375,7 +375,7 @@ sched_update_deq(sched_elem_t *q, while (wfe() && monitor8(&q->qschst.cur_ticket, __ATOMIC_ACQUIRE) != ticket) - doze(); + odp_cpu_pause(); } /* We are here because of non-empty-to-empty transition or * WRR budget exhausted @@ -502,7 +502,7 @@ static inline void sched_update_popd(sched_elem_t *elem) sevl(); while (wfe() && monitor8(&elem->qschst.cur_ticket, __ATOMIC_ACQUIRE) != ticket) - doze(); + odp_cpu_pause(); } sched_update_popd_sc(elem); atomic_store_release(&elem->qschst.cur_ticket, ticket + 1, @@ -1074,7 +1074,7 @@ restart_same: while (wfe() && monitor32(&rwin->turn, __ATOMIC_ACQUIRE) != sn) - doze(); + odp_cpu_pause(); } #ifdef CONFIG_QSCHST_LOCK LOCK(&elem->qschlock); @@ -1162,7 +1162,7 @@ static void schedule_order_lock(uint32_t lock_index) while (wfe() && monitor32(&rctx->rwin->olock[lock_index], __ATOMIC_ACQUIRE) != rctx->sn) - doze(); + odp_cpu_pause(); } } @@ -1579,7 +1579,7 @@ static int schedule_group_destroy(odp_schedule_group_t group) !bitset_is_eql(wanted, bitset_monitor(&sg->thr_actual[p], __ATOMIC_RELAXED))) - doze(); + odp_cpu_pause(); } /* Else ignore because no ODP queues on this prio */ } @@ -2157,7 +2157,7 @@ static void order_lock(void) */ while (wfe() && monitor32(&rwin->hc.head, __ATOMIC_ACQUIRE) != sn) - doze(); + odp_cpu_pause(); } } diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index 90ba101cd..cad391ed4 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -1037,7 +1037,7 @@ static int schedule_capability(odp_schedule_capability_t *capa) static void get_config(schedule_config_t *config) { - *config = *(&sched_global->config_if); + *config = sched_global->config_if; }; /* Fill in scheduler interface */ diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index ce33c6787..8b2ba16c2 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -26,6 +26,7 @@ #include <odp/api/align.h> #include <odp_align_internal.h> #include <odp/api/atomic.h> +#include <odp/api/plat/atomic_inlines.h> #include <odp_atomic_internal.h> #include <odp/api/buffer.h> #include <odp/api/cpu.h> @@ -106,7 +107,7 @@ ODP_STATIC_ASSERT(sizeof(tick_buf_t) == 16, "sizeof(tick_buf_t) == 16"); #endif typedef struct { - void *user_ptr; + const void *user_ptr; odp_queue_t queue;/* Used for free list when timer is free */ } _odp_timer_t; @@ -186,10 +187,7 @@ static __thread timer_local_t timer_local; static void itimer_init(timer_pool_t *tp); static void itimer_fini(timer_pool_t *tp); -static void timer_init(_odp_timer_t *tim, - tick_buf_t *tb, - odp_queue_t _q, - void *_up) +static void timer_init(_odp_timer_t *tim, tick_buf_t *tb, odp_queue_t _q, const void *_up) { tim->queue = _q; tim->user_ptr = _up; @@ -200,7 +198,7 @@ static void timer_init(_odp_timer_t *tim, #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 tb->exp_tck.v = TMO_INACTIVE; #else - _odp_atomic_u64_store_mm(&tb->exp_tck, TMO_INACTIVE, _ODP_MEMMODEL_RLS); + odp_atomic_store_rel_u64(&tb->exp_tck, TMO_INACTIVE); #endif } @@ -487,9 +485,7 @@ static void odp_timer_pool_del(timer_pool_t *tp) ODP_ABORT("Failed to free shared memory (%d)\n", rc); } -static inline odp_timer_t timer_alloc(timer_pool_t *tp, - odp_queue_t queue, - void *user_ptr) +static inline odp_timer_t timer_alloc(timer_pool_t *tp, odp_queue_t queue, const void *user_ptr) { odp_timer_t hdl; @@ -504,13 +500,12 @@ static inline odp_timer_t timer_alloc(timer_pool_t *tp, tp->first_free = get_next_free(tim); /* Initialize timer */ timer_init(tim, &tp->tick_buf[idx], queue, user_ptr); - if (odp_unlikely(tp->num_alloc > - odp_atomic_load_u32(&tp->high_wm))) + if (odp_unlikely(tp->num_alloc > odp_atomic_load_u32(&tp->high_wm))) { /* Update high_wm last with release model to * ensure timer initialization is visible */ - _odp_atomic_u32_store_mm(&tp->high_wm, - tp->num_alloc, - _ODP_MEMMODEL_RLS); + odp_atomic_store_rel_u32(&tp->high_wm, tp->num_alloc); + } + hdl = tp_idx_to_handle(tp, idx); /* Add timer to queue */ _odp_queue_fn->timer_add(queue); @@ -602,8 +597,8 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_buffer_t *tmo_buf, uint64_t old; /* Swap in new expiration tick, get back old tick which * will indicate active/inactive timer state */ - old = _odp_atomic_u64_xchg_mm(&tb->exp_tck, abs_tck, - _ODP_MEMMODEL_RLX); + old = odp_atomic_xchg_u64(&tb->exp_tck, abs_tck); + if ((old & TMO_INACTIVE) != 0) { /* Timer was inactive (cancelled or expired), * we can't reset a timer without a timeout buffer. @@ -615,12 +610,7 @@ static bool timer_reset(uint32_t idx, uint64_t abs_tck, odp_buffer_t *tmo_buf, * reset or cancelled the timer. Without any * synchronization between the threads, we have a * data race and the behavior is undefined */ - (void)_odp_atomic_u64_cmp_xchg_strong_mm( - &tb->exp_tck, - &abs_tck, - old, - _ODP_MEMMODEL_RLX, - _ODP_MEMMODEL_RLX); + (void)odp_atomic_cas_u64(&tb->exp_tck, &abs_tck, old); success = false; } #else /* Target supports neither 128-bit nor 64-bit CAS => use lock */ @@ -806,7 +796,7 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) uint64_t exp_tck; #ifdef ODP_ATOMIC_U128 /* Atomic re-read for correctness */ - exp_tck = _odp_atomic_u64_load_mm(&tb->exp_tck, _ODP_MEMMODEL_RLX); + exp_tck = odp_atomic_load_u64(&tb->exp_tck); /* Re-check exp_tck */ if (odp_likely(exp_tck <= tick)) { /* Attempt to grab timeout buffer, replace with inactive timer @@ -888,8 +878,7 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) static inline void timer_pool_scan(timer_pool_t *tp, uint64_t tick) { tick_buf_t *array = &tp->tick_buf[0]; - uint32_t high_wm = _odp_atomic_u32_load_mm(&tp->high_wm, - _ODP_MEMMODEL_ACQ); + uint32_t high_wm = odp_atomic_load_acq_u32(&tp->high_wm); uint32_t i; ODP_ASSERT(high_wm <= tp->param.num_timers); @@ -1364,9 +1353,7 @@ uint64_t odp_timer_pool_to_u64(odp_timer_pool_t tpid) return _odp_pri(tpid); } -odp_timer_t odp_timer_alloc(odp_timer_pool_t tpid, - odp_queue_t queue, - void *user_ptr) +odp_timer_t odp_timer_alloc(odp_timer_pool_t tpid, odp_queue_t queue, const void *user_ptr) { timer_pool_t *tp = timer_pool_from_hdl(tpid); @@ -1497,7 +1484,7 @@ uint64_t odp_timeout_tick(odp_timeout_t tmo) void *odp_timeout_user_ptr(odp_timeout_t tmo) { - return timeout_hdr(tmo)->user_ptr; + return (void *)(uintptr_t)timeout_hdr(tmo)->user_ptr; } odp_timeout_t odp_timeout_alloc(odp_pool_t pool) @@ -1516,6 +1503,81 @@ void odp_timeout_free(odp_timeout_t tmo) odp_buffer_free(odp_buffer_from_event(ev)); } +void odp_timer_pool_print(odp_timer_pool_t timer_pool) +{ + timer_pool_t *tp; + + if (timer_pool == ODP_TIMER_POOL_INVALID) { + ODP_ERR("Bad timer pool handle\n"); + return; + } + + tp = timer_pool_from_hdl(timer_pool); + + ODP_PRINT("\nTimer pool info\n"); + ODP_PRINT("---------------\n"); + ODP_PRINT(" timer pool %p\n", tp); + ODP_PRINT(" tp index %u\n", tp->tp_idx); + ODP_PRINT(" num timers %u\n", tp->num_alloc); + ODP_PRINT(" num tp %i\n", timer_global->num_timer_pools); + ODP_PRINT(" inline timers %i\n", timer_global->use_inline_timers); + ODP_PRINT("\n"); +} + +void odp_timer_print(odp_timer_t timer) +{ + timer_pool_t *tp; + uint32_t idx; + _odp_timer_t *tim; + + if (timer == ODP_TIMER_INVALID) { + ODP_ERR("Bad timer handle\n"); + return; + } + + tp = handle_to_tp(timer); + idx = handle_to_idx(timer, tp); + tim = &tp->timers[idx]; + + ODP_PRINT("\nTimer info\n"); + ODP_PRINT("----------\n"); + ODP_PRINT(" timer pool %p\n", tp); + ODP_PRINT(" timer index %u\n", idx); + ODP_PRINT(" dest queue 0x%" PRIx64 "\n", odp_queue_to_u64(tim->queue)); + ODP_PRINT(" user ptr %p\n", tim->user_ptr); + ODP_PRINT("\n"); +} + +void odp_timeout_print(odp_timeout_t tmo) +{ + const odp_timeout_hdr_t *tmo_hdr; + odp_timer_t timer; + timer_pool_t *tp = NULL; + uint32_t idx = 0; + + if (tmo == ODP_TIMEOUT_INVALID) { + ODP_ERR("Bad timeout handle\n"); + return; + } + + tmo_hdr = timeout_hdr(tmo); + timer = tmo_hdr->timer; + + if (timer != ODP_TIMER_INVALID) { + tp = handle_to_tp(timer); + idx = handle_to_idx(timer, tp); + } + + ODP_PRINT("\nTimeout info\n"); + ODP_PRINT("------------\n"); + ODP_PRINT(" tmo handle 0x%" PRIx64 "\n", odp_timeout_to_u64(tmo)); + ODP_PRINT(" timer pool %p\n", tp); + ODP_PRINT(" timer index %u\n", idx); + ODP_PRINT(" expiration %" PRIu64 "\n", tmo_hdr->expiration); + ODP_PRINT(" user ptr %p\n", tmo_hdr->user_ptr); + ODP_PRINT("\n"); +} + int _odp_timer_init_global(const odp_init_t *params) { odp_shm_t shm; diff --git a/test/common/odp_cunit_common.c b/test/common/odp_cunit_common.c index 25895d628..62418c356 100644 --- a/test/common/odp_cunit_common.c +++ b/test/common/odp_cunit_common.c @@ -1,5 +1,6 @@ /* Copyright (c) 2014-2018, Linaro Limited * Copyright (c) 2019, Nokia + * Copyright (c) 2021, Marvell * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -273,6 +274,39 @@ static int default_term_func(void) return odp_cunit_print_inactive(); } +static void _cunit_test_setup_func(void) +{ + CU_AllTestsCompleteMessageHandler all_test_comp_handler; + CU_SuiteCompleteMessageHandler suite_comp_handler; + CU_pFailureRecord failrec; + CU_pSuite suite; + + if (!getenv("ODP_CUNIT_FAIL_IMMEDIATE")) + return; + + if (CU_get_number_of_failure_records() == 0) + return; + + /* User wants the suite to fail immediately once a test hits an error */ + suite = CU_get_current_suite(); + failrec = CU_get_failure_list(); + + printf("Force aborting as a previous test failed\n"); + + /* Call the Cleanup functions before aborting */ + suite->pCleanupFunc(); + + suite_comp_handler = CU_get_suite_complete_handler(); + if (suite_comp_handler) + suite_comp_handler(suite, failrec); + + all_test_comp_handler = CU_get_all_test_complete_handler(); + if (all_test_comp_handler) + all_test_comp_handler(failrec); + + exit(EXIT_FAILURE); +} + /* * Register suites and tests with CUnit. * @@ -292,7 +326,9 @@ static int cunit_register_suites(odp_suiteinfo_t testsuites[]) if (sinfo->term_func) term_func = sinfo->term_func; - suite = CU_add_suite(sinfo->name, _cunit_suite_init, term_func); + suite = CU_add_suite_with_setup_and_teardown(sinfo->name, _cunit_suite_init, + term_func, _cunit_test_setup_func, + NULL); if (!suite) return CU_get_error(); diff --git a/test/performance/.gitignore b/test/performance/.gitignore index 80396e5d9..0e6d9ef57 100644 --- a/test/performance/.gitignore +++ b/test/performance/.gitignore @@ -1,6 +1,7 @@ *.log *.trs odp_atomic +odp_atomic_perf odp_bench_packet odp_cpu_bench odp_crypto diff --git a/test/performance/Makefile.am b/test/performance/Makefile.am index 624795f8b..7566ab8a8 100644 --- a/test/performance/Makefile.am +++ b/test/performance/Makefile.am @@ -2,7 +2,8 @@ include $(top_srcdir)/test/Makefile.inc TESTS_ENVIRONMENT += TEST_DIR=${builddir} -EXECUTABLES = odp_bench_packet \ +EXECUTABLES = odp_atomic_perf \ + odp_bench_packet \ odp_cpu_bench \ odp_crypto \ odp_ipsec \ @@ -39,6 +40,7 @@ endif bin_PROGRAMS = $(EXECUTABLES) $(COMPILE_ONLY) +odp_atomic_perf_SOURCES = odp_atomic_perf.c odp_bench_packet_SOURCES = odp_bench_packet.c odp_cpu_bench_SOURCES = odp_cpu_bench.c odp_crypto_SOURCES = odp_crypto.c diff --git a/test/performance/odp_atomic_perf.c b/test/performance/odp_atomic_perf.c new file mode 100644 index 000000000..2ed88a5e8 --- /dev/null +++ b/test/performance/odp_atomic_perf.c @@ -0,0 +1,1184 @@ +/* Copyright (c) 2021, Nokia + * + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdio.h> +#include <string.h> +#include <stdint.h> +#include <inttypes.h> +#include <stdlib.h> +#include <getopt.h> + +#include <odp_api.h> +#include <odp/helper/odph_api.h> + +/* Default number of test rounds */ +#define NUM_ROUNDS 1000000u + +/* Initial value for atomic variables */ +#define INIT_VAL 1234567 + +/* Max number of workers if num_cpu=0 */ +#define DEFAULT_MAX_WORKERS 10 + +#define TEST_INFO(name, test, validate, op_type) \ + {name, test, validate, op_type} + +/* Test function template */ +typedef void (*test_fn_t)(void *val, void *out, uint32_t num_round); +/* Test result validation function template */ +typedef int (*validate_fn_t)(void *val, void *out, uint32_t num_round, + uint32_t num_worker, int private); + +typedef enum { + OP_32BIT, + OP_64BIT +} op_bit_t; + +/* Command line options */ +typedef struct test_options_t { + uint32_t num_cpu; + uint32_t num_round; + int private; + +} test_options_t; + +/* Cache aligned atomics for private mode operation */ +typedef struct ODP_ALIGNED_CACHE test_atomic_t { + union { + odp_atomic_u32_t u32; + odp_atomic_u64_t u64; + }; +} test_atomic_t; + +typedef struct test_global_t test_global_t; + +/* Worker thread context */ +typedef struct test_thread_ctx_t { + test_global_t *global; + test_fn_t func; + uint64_t nsec; + uint32_t idx; + op_bit_t type; + +} test_thread_ctx_t; + +/* Global data */ +struct test_global_t { + test_options_t test_options; + odp_barrier_t barrier; + union { + odp_atomic_u32_t atomic_u32; + odp_atomic_u64_t atomic_u64; + }; + odp_cpumask_t cpumask; + odph_thread_t thread_tbl[ODP_THREAD_COUNT_MAX]; + test_thread_ctx_t thread_ctx[ODP_THREAD_COUNT_MAX]; + test_atomic_t atomic_private[ODP_THREAD_COUNT_MAX]; + union { + uint32_t u32; + uint64_t u64; + } output[ODP_THREAD_COUNT_MAX]; +}; + +typedef struct { + const char *name; + test_fn_t test_fn; + validate_fn_t validate_fn; + op_bit_t type; +} test_case_t; + +static test_global_t *test_global; + +static inline void test_atomic_load_u32(void *val, void *out, uint32_t num_round) +{ + odp_atomic_u32_t *atomic_val = val; + uint32_t *result = out; + uint32_t ret = 0; + + for (uint32_t i = 0; i < num_round; i++) + ret += odp_atomic_load_u32(atomic_val); + + *result = ret; +} + +static inline void test_atomic_load_u64(void *val, void *out, uint32_t num_round) +{ + odp_atomic_u64_t *atomic_val = val; + uint64_t *result = out; + uint64_t ret = 0; + + for (uint32_t i = 0; i < num_round; i++) + ret += odp_atomic_load_u64(atomic_val); + + *result = ret; +} + +static inline int validate_atomic_init_val_u32(void *val, void *out, uint32_t num_round, + uint32_t num_worker ODP_UNUSED, + int private ODP_UNUSED) +{ + odp_atomic_u32_t *atomic_val = val; + uint32_t *result = out; + + return (odp_atomic_load_u32(atomic_val) != INIT_VAL) || + (*result != (uint32_t)INIT_VAL * num_round); +} + +static inline int validate_atomic_init_val_u64(void *val, void *out ODP_UNUSED, + uint32_t num_round ODP_UNUSED, + uint32_t worker ODP_UNUSED, int private ODP_UNUSED) +{ + odp_atomic_u64_t *atomic_val = val; + uint64_t *result = out; + + return (odp_atomic_load_u64(atomic_val) != INIT_VAL) || + (*result != (uint64_t)INIT_VAL * num_round); +} + +static inline void test_atomic_store_u32(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u32_t *atomic_val = val; + uint32_t new_val = INIT_VAL + 1; + + for (uint32_t i = 0; i < num_round; i++) + odp_atomic_store_u32(atomic_val, new_val++); +} + +static inline void test_atomic_store_u64(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u64_t *atomic_val = val; + uint64_t new_val = INIT_VAL + 1; + + for (uint32_t i = 0; i < num_round; i++) + odp_atomic_store_u64(atomic_val, new_val++); +} + +static inline int validate_atomic_num_round_u32(void *val, void *out ODP_UNUSED, uint32_t num_round, + uint32_t worker ODP_UNUSED, int private ODP_UNUSED) +{ + odp_atomic_u32_t *atomic_val = val; + + return odp_atomic_load_u32(atomic_val) != ((uint32_t)INIT_VAL + num_round); +} + +static inline int validate_atomic_num_round_u64(void *val, void *out ODP_UNUSED, uint32_t num_round, + uint32_t worker ODP_UNUSED, int private ODP_UNUSED) +{ + odp_atomic_u64_t *atomic_val = val; + + return odp_atomic_load_u64(atomic_val) != ((uint64_t)INIT_VAL + num_round); +} + +static inline void test_atomic_fetch_add_u32(void *val, void *out, uint32_t num_round) +{ + odp_atomic_u32_t *atomic_val = val; + uint32_t *result = out; + uint32_t ret = 0; + + for (uint32_t i = 0; i < num_round; i++) + ret += odp_atomic_fetch_add_u32(atomic_val, 1); + + *result = ret; +} + +static inline void test_atomic_fetch_add_u64(void *val, void *out, uint32_t num_round) +{ + odp_atomic_u64_t *atomic_val = val; + uint64_t *result = out; + uint64_t ret = 0; + + for (uint32_t i = 0; i < num_round; i++) + ret += odp_atomic_fetch_add_u64(atomic_val, 1); + + *result = ret; +} + +static inline int validate_atomic_add_round_u32(void *val, void *out ODP_UNUSED, uint32_t num_round, + uint32_t num_worker, int private) +{ + odp_atomic_u32_t *atomic_val = val; + + if (private) + return odp_atomic_load_u32(atomic_val) != ((uint32_t)INIT_VAL + num_round); + + return odp_atomic_load_u32(atomic_val) != (INIT_VAL + (num_worker * num_round)); +} + +static inline int validate_atomic_add_round_u64(void *val, void *out ODP_UNUSED, uint32_t num_round, + uint32_t num_worker, int private) +{ + odp_atomic_u64_t *atomic_val = val; + + if (private) + return odp_atomic_load_u64(atomic_val) != ((uint64_t)INIT_VAL + num_round); + + return odp_atomic_load_u64(atomic_val) != (INIT_VAL + ((uint64_t)num_worker * num_round)); +} + +static inline void test_atomic_add_u32(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u32_t *atomic_val = val; + + for (uint32_t i = 0; i < num_round; i++) + odp_atomic_add_u32(atomic_val, 1); +} + +static inline void test_atomic_add_u64(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u64_t *atomic_val = val; + + for (uint32_t i = 0; i < num_round; i++) + odp_atomic_add_u64(atomic_val, 1); +} + +static inline void test_atomic_fetch_sub_u32(void *val, void *out, uint32_t num_round) +{ + odp_atomic_u32_t *atomic_val = val; + uint32_t *result = out; + uint32_t ret = 0; + + for (uint32_t i = 0; i < num_round; i++) + ret += odp_atomic_fetch_sub_u32(atomic_val, 1); + + *result = ret; +} + +static inline void test_atomic_fetch_sub_u64(void *val, void *out, uint32_t num_round) +{ + odp_atomic_u64_t *atomic_val = val; + uint64_t *result = out; + uint64_t ret = 0; + + for (uint32_t i = 0; i < num_round; i++) + ret += odp_atomic_fetch_sub_u64(atomic_val, 1); + + *result = ret; +} + +static inline int validate_atomic_sub_round_u32(void *val, void *out ODP_UNUSED, uint32_t num_round, + uint32_t num_worker, int private) +{ + odp_atomic_u32_t *atomic_val = val; + + if (private) + return odp_atomic_load_u32(atomic_val) != ((uint32_t)INIT_VAL - num_round); + + return odp_atomic_load_u32(atomic_val) != ((uint32_t)INIT_VAL - (num_worker * num_round)); +} + +static inline int validate_atomic_sub_round_u64(void *val, void *out ODP_UNUSED, uint32_t num_round, + uint32_t num_worker, int private) +{ + odp_atomic_u64_t *atomic_val = val; + + if (private) + return odp_atomic_load_u64(atomic_val) != ((uint64_t)INIT_VAL - num_round); + + return odp_atomic_load_u64(atomic_val) != ((uint64_t)INIT_VAL - + ((uint64_t)num_worker * num_round)); +} + +static inline void test_atomic_sub_u32(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u32_t *atomic_val = val; + + for (uint32_t i = 0; i < num_round; i++) + odp_atomic_sub_u32(atomic_val, 1); +} + +static inline void test_atomic_sub_u64(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u64_t *atomic_val = val; + + for (uint32_t i = 0; i < num_round; i++) + odp_atomic_sub_u64(atomic_val, 1); +} + +static inline void test_atomic_fetch_inc_u32(void *val, void *out, uint32_t num_round) +{ + odp_atomic_u32_t *atomic_val = val; + uint32_t *result = out; + uint32_t ret = 0; + + for (uint32_t i = 0; i < num_round; i++) + ret += odp_atomic_fetch_inc_u32(atomic_val); + + *result = ret; +} + +static inline void test_atomic_fetch_inc_u64(void *val, void *out, uint32_t num_round) +{ + odp_atomic_u64_t *atomic_val = val; + uint64_t *result = out; + uint64_t ret = 0; + + for (uint32_t i = 0; i < num_round; i++) + ret += odp_atomic_fetch_inc_u64(atomic_val); + + *result = ret; +} + +static inline void test_atomic_inc_u32(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u32_t *atomic_val = val; + + for (uint32_t i = 0; i < num_round; i++) + odp_atomic_inc_u32(atomic_val); +} + +static inline void test_atomic_inc_u64(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u64_t *atomic_val = val; + + for (uint32_t i = 0; i < num_round; i++) + odp_atomic_inc_u64(atomic_val); +} + +static inline void test_atomic_fetch_dec_u32(void *val, void *out, uint32_t num_round) +{ + odp_atomic_u32_t *atomic_val = val; + uint32_t *result = out; + uint32_t ret = 0; + + for (uint32_t i = 0; i < num_round; i++) + ret += odp_atomic_fetch_dec_u32(atomic_val); + + *result = ret; +} + +static inline void test_atomic_fetch_dec_u64(void *val, void *out, uint32_t num_round) +{ + odp_atomic_u64_t *atomic_val = val; + uint64_t *result = out; + uint64_t ret = 0; + + for (uint32_t i = 0; i < num_round; i++) + ret += odp_atomic_fetch_dec_u64(atomic_val); + + *result = ret; +} + +static inline void test_atomic_dec_u32(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u32_t *atomic_val = val; + + for (uint32_t i = 0; i < num_round; i++) + odp_atomic_dec_u32(atomic_val); +} + +static inline void test_atomic_dec_u64(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u64_t *atomic_val = val; + + for (uint32_t i = 0; i < num_round; i++) + odp_atomic_dec_u64(atomic_val); +} + +static inline void test_atomic_max_u32(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u32_t *atomic_val = val; + uint32_t new_max = INIT_VAL + 1; + + for (uint32_t i = 0; i < num_round; i++) + odp_atomic_max_u32(atomic_val, new_max++); +} + +static inline void test_atomic_max_u64(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u64_t *atomic_val = val; + uint64_t new_max = INIT_VAL + 1; + + for (uint32_t i = 0; i < num_round; i++) + odp_atomic_max_u64(atomic_val, new_max++); +} + +static inline int validate_atomic_max_u32(void *val, void *out ODP_UNUSED, uint32_t num_round, + uint32_t num_worker ODP_UNUSED, int private ODP_UNUSED) +{ + uint32_t result = odp_atomic_load_u32((odp_atomic_u32_t *)val); + + return (result != ((uint32_t)INIT_VAL + num_round)) && (result != UINT32_MAX); +} + +static inline int validate_atomic_max_u64(void *val, void *out ODP_UNUSED, uint32_t num_round, + uint32_t num_worker ODP_UNUSED, int private ODP_UNUSED) +{ + uint64_t result = odp_atomic_load_u64((odp_atomic_u64_t *)val); + + return (result != ((uint64_t)INIT_VAL + num_round)) && (result != UINT64_MAX); +} + +static inline void test_atomic_min_u32(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u32_t *atomic_val = val; + uint32_t new_min = INIT_VAL - 1; + + for (uint32_t i = 0; i < num_round; i++) + odp_atomic_min_u32(atomic_val, new_min--); +} + +static inline void test_atomic_min_u64(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u64_t *atomic_val = val; + uint64_t new_min = INIT_VAL - 1; + + for (uint32_t i = 0; i < num_round; i++) + odp_atomic_min_u64(atomic_val, new_min--); +} + +static inline int validate_atomic_min_u32(void *val, void *out ODP_UNUSED, uint32_t num_round, + uint32_t num_worker ODP_UNUSED, int private ODP_UNUSED) +{ + uint32_t result = odp_atomic_load_u32((odp_atomic_u32_t *)val); + + return result != ((uint32_t)INIT_VAL - num_round) && result != 0; +} + +static inline int validate_atomic_min_u64(void *val, void *out ODP_UNUSED, uint32_t num_round, + uint32_t num_worker ODP_UNUSED, int private ODP_UNUSED) +{ + uint64_t result = odp_atomic_load_u64((odp_atomic_u64_t *)val); + + return result != ((uint64_t)INIT_VAL - num_round) && result != 0; +} + +static inline void test_atomic_cas_u32(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u32_t *atomic_val = val; + uint32_t new_val = INIT_VAL + 1; + uint32_t old_val = INIT_VAL; + + for (uint32_t i = 0; i < num_round; i++) { + if (odp_atomic_cas_u32(atomic_val, &old_val, new_val)) + old_val = new_val++; + } +} + +static inline void test_atomic_cas_u64(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u64_t *atomic_val = val; + uint64_t new_val = INIT_VAL + 1; + uint64_t old_val = INIT_VAL; + + for (uint32_t i = 0; i < num_round; i++) { + if (odp_atomic_cas_u64(atomic_val, &old_val, new_val)) + old_val = new_val++; + } +} + +static inline int validate_atomic_cas_u32(void *val, void *out ODP_UNUSED, uint32_t num_round, + uint32_t num_worker ODP_UNUSED, int private) +{ + uint32_t result = odp_atomic_load_u32((odp_atomic_u32_t *)val); + + if (private) + return result != ((uint32_t)INIT_VAL + num_round); + + return result > ((uint32_t)INIT_VAL + num_round); +} + +static inline int validate_atomic_cas_u64(void *val, void *out ODP_UNUSED, uint32_t num_round, + uint32_t num_worker ODP_UNUSED, int private) +{ + uint64_t result = odp_atomic_load_u64((odp_atomic_u64_t *)val); + + if (private) + return result != ((uint64_t)INIT_VAL + num_round); + + return result > ((uint64_t)INIT_VAL + num_round); +} + +static inline void test_atomic_xchg_u32(void *val, void *out, uint32_t num_round) +{ + odp_atomic_u32_t *atomic_val = val; + uint32_t new_val = INIT_VAL + 1; + uint32_t *result = out; + uint32_t ret = 0; + + for (uint32_t i = 0; i < num_round; i++) + ret += odp_atomic_xchg_u32(atomic_val, new_val++); + + *result = ret; +} + +static inline void test_atomic_xchg_u64(void *val, void *out, uint32_t num_round) +{ + odp_atomic_u64_t *atomic_val = val; + uint64_t new_val = INIT_VAL + 1; + uint64_t *result = out; + uint64_t ret = 0; + + for (uint32_t i = 0; i < num_round; i++) + ret += odp_atomic_xchg_u64(atomic_val, new_val++); + + *result = ret; +} + +static inline void test_atomic_load_acq_u32(void *val, void *out, uint32_t num_round) +{ + odp_atomic_u32_t *atomic_val = val; + uint32_t *result = out; + uint32_t ret = 0; + + for (uint32_t i = 0; i < num_round; i++) + ret += odp_atomic_load_acq_u32(atomic_val); + + *result = ret; +} + +static inline void test_atomic_load_acq_u64(void *val, void *out, uint32_t num_round) +{ + odp_atomic_u64_t *atomic_val = val; + uint64_t *result = out; + uint64_t ret = 0; + + for (uint32_t i = 0; i < num_round; i++) + ret += odp_atomic_load_acq_u64(atomic_val); + + *result = ret; +} + +static inline void test_atomic_store_rel_u32(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u32_t *atomic_val = val; + uint32_t new_val = INIT_VAL + 1; + + for (uint32_t i = 0; i < num_round; i++) + odp_atomic_store_rel_u32(atomic_val, new_val++); +} + +static inline void test_atomic_store_rel_u64(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u64_t *atomic_val = val; + uint64_t new_val = INIT_VAL + 1; + + for (uint32_t i = 0; i < num_round; i++) + odp_atomic_store_rel_u64(atomic_val, new_val++); +} + +static inline void test_atomic_add_rel_u32(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u32_t *atomic_val = val; + + for (uint32_t i = 0; i < num_round; i++) + odp_atomic_add_rel_u32(atomic_val, 1); +} + +static inline void test_atomic_add_rel_u64(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u64_t *atomic_val = val; + + for (uint32_t i = 0; i < num_round; i++) + odp_atomic_add_rel_u64(atomic_val, 1); +} + +static inline void test_atomic_sub_rel_u32(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u32_t *atomic_val = val; + + for (uint32_t i = 0; i < num_round; i++) + odp_atomic_sub_rel_u32(atomic_val, 1); +} + +static inline void test_atomic_sub_rel_u64(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u64_t *atomic_val = val; + + for (uint32_t i = 0; i < num_round; i++) + odp_atomic_sub_rel_u64(atomic_val, 1); +} + +static inline void test_atomic_cas_acq_u32(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u32_t *atomic_val = val; + uint32_t new_val = INIT_VAL + 1; + uint32_t old_val = INIT_VAL; + + for (uint32_t i = 0; i < num_round; i++) { + if (odp_atomic_cas_acq_u32(atomic_val, &old_val, new_val)) + old_val = new_val++; + } +} + +static inline void test_atomic_cas_acq_u64(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u64_t *atomic_val = val; + uint64_t new_val = INIT_VAL + 1; + uint64_t old_val = INIT_VAL; + + for (uint32_t i = 0; i < num_round; i++) { + if (odp_atomic_cas_acq_u64(atomic_val, &old_val, new_val)) + old_val = new_val++; + } +} + +static inline void test_atomic_cas_rel_u32(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u32_t *atomic_val = val; + uint32_t new_val = INIT_VAL + 1; + uint32_t old_val = INIT_VAL; + + for (uint32_t i = 0; i < num_round; i++) { + if (odp_atomic_cas_rel_u32(atomic_val, &old_val, new_val)) + old_val = new_val++; + } +} + +static inline void test_atomic_cas_rel_u64(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u64_t *atomic_val = val; + uint64_t new_val = INIT_VAL + 1; + uint64_t old_val = INIT_VAL; + + for (uint32_t i = 0; i < num_round; i++) { + if (odp_atomic_cas_rel_u64(atomic_val, &old_val, new_val)) + old_val = new_val++; + } +} + +static inline void test_atomic_cas_acq_rel_u32(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u32_t *atomic_val = val; + uint32_t new_val = INIT_VAL + 1; + uint32_t old_val = INIT_VAL; + + for (uint32_t i = 0; i < num_round; i++) { + if (odp_atomic_cas_acq_rel_u32(atomic_val, &old_val, new_val)) + old_val = new_val++; + } +} + +static inline void test_atomic_cas_acq_rel_u64(void *val, void *out ODP_UNUSED, uint32_t num_round) +{ + odp_atomic_u64_t *atomic_val = val; + uint64_t new_val = INIT_VAL + 1; + uint64_t old_val = INIT_VAL; + + for (uint32_t i = 0; i < num_round; i++) { + if (odp_atomic_cas_acq_rel_u64(atomic_val, &old_val, new_val)) + old_val = new_val++; + } +} + +static void print_usage(void) +{ + printf("\n" + "Atomic operations performance test\n" + "\n" + "Usage: odp_atomic_perf [options]\n" + "\n" + " -c, --num_cpu Number of CPUs (worker threads). 0: all available CPUs (or max %d) (default)\n" + " -r, --num_round Number of rounds (default %u)\n" + " -p, --private 0: The same atomic variable is shared between threads (default)\n" + " 1: Atomic variables are private to each thread\n" + " -h, --help This help\n" + "\n", DEFAULT_MAX_WORKERS, NUM_ROUNDS); +} + +static void print_info(test_options_t *test_options) +{ + odp_atomic_op_t atomic_ops; + + printf("\nAtomic operations performance test configuration:\n"); + printf(" num cpu %u\n", test_options->num_cpu); + printf(" num rounds %u\n", test_options->num_round); + printf(" private %i\n", test_options->private); + printf("\n"); + + atomic_ops.all_bits = 0; + odp_atomic_lock_free_u64(&atomic_ops); + + printf("\nAtomic operations lock-free:\n"); + printf(" odp_atomic_load_u64: %" PRIu32 "\n", atomic_ops.op.load); + printf(" odp_atomic_store_u64: %" PRIu32 "\n", atomic_ops.op.store); + printf(" odp_atomic_fetch_add_u64: %" PRIu32 "\n", atomic_ops.op.fetch_add); + printf(" odp_atomic_add_u64: %" PRIu32 "\n", atomic_ops.op.add); + printf(" odp_atomic_fetch_sub_u64: %" PRIu32 "\n", atomic_ops.op.fetch_sub); + printf(" odp_atomic_sub_u64: %" PRIu32 "\n", atomic_ops.op.sub); + printf(" odp_atomic_fetch_inc_u64: %" PRIu32 "\n", atomic_ops.op.fetch_inc); + printf(" odp_atomic_inc_u64: %" PRIu32 "\n", atomic_ops.op.inc); + printf(" odp_atomic_fetch_dec_u64: %" PRIu32 "\n", atomic_ops.op.fetch_dec); + printf(" odp_atomic_dec_u64: %" PRIu32 "\n", atomic_ops.op.dec); + printf(" odp_atomic_min_u64: %" PRIu32 "\n", atomic_ops.op.min); + printf(" odp_atomic_max_u64: %" PRIu32 "\n", atomic_ops.op.max); + printf(" odp_atomic_cas_u64: %" PRIu32 "\n", atomic_ops.op.cas); + printf(" odp_atomic_xchg_u64: %" PRIu32 "\n", atomic_ops.op.xchg); + printf("\n\n"); +} + +static int parse_options(int argc, char *argv[], test_options_t *test_options) +{ + int opt; + int long_index; + int ret = 0; + + static const struct option longopts[] = { + {"num_cpu", required_argument, NULL, 'c'}, + {"num_round", required_argument, NULL, 'r'}, + {"private", required_argument, NULL, 'p'}, + {"help", no_argument, NULL, 'h'}, + {NULL, 0, NULL, 0} + }; + + static const char *shortopts = "+c:r:p:h"; + + memset(test_options, 0, sizeof(test_options_t)); + test_options->num_cpu = 0; + test_options->num_round = NUM_ROUNDS; + test_options->private = 0; + + while (1) { + opt = getopt_long(argc, argv, shortopts, longopts, &long_index); + + if (opt == -1) + break; + + switch (opt) { + case 'c': + test_options->num_cpu = atoi(optarg); + break; + case 'r': + test_options->num_round = atol(optarg); + break; + case 'p': + test_options->private = atoi(optarg); + break; + case 'h': + /* fall through */ + default: + print_usage(); + ret = -1; + break; + } + } + + if (test_options->num_round < 1) { + ODPH_ERR("Invalid number of test rounds: %" PRIu32 "\n", test_options->num_round); + return -1; + } + + return ret; +} + +static int set_num_cpu(test_global_t *global) +{ + int ret, max_num; + test_options_t *test_options = &global->test_options; + int num_cpu = test_options->num_cpu; + + /* One thread used for the main thread */ + if (num_cpu > ODP_THREAD_COUNT_MAX - 1) { + ODPH_ERR("Too many workers. Maximum is %i.\n", ODP_THREAD_COUNT_MAX - 1); + return -1; + } + + max_num = num_cpu; + if (num_cpu == 0) { + max_num = ODP_THREAD_COUNT_MAX - 1; + if (max_num > DEFAULT_MAX_WORKERS) + max_num = DEFAULT_MAX_WORKERS; + } + + ret = odp_cpumask_default_worker(&global->cpumask, max_num); + + if (num_cpu && ret != num_cpu) { + ODPH_ERR("Too many workers. Max supported %i.\n", ret); + return -1; + } + + /* Zero: all available workers */ + if (num_cpu == 0) { + if (ret > max_num) { + ODPH_ERR("Too many cpus from odp_cpumask_default_worker(): %i\n", ret); + return -1; + } + + num_cpu = ret; + test_options->num_cpu = num_cpu; + } + + odp_barrier_init(&global->barrier, num_cpu); + + return 0; +} + +static int init_test(test_global_t *global, const char *name, op_bit_t type) +{ + printf("TEST: %s\n", name); + + if (type == OP_32BIT) + odp_atomic_init_u32(&global->atomic_u32, INIT_VAL); + else if (type == OP_64BIT) + odp_atomic_init_u64(&global->atomic_u64, INIT_VAL); + else + return -1; + + for (int i = 0; i < ODP_THREAD_COUNT_MAX; i++) { + if (type == OP_32BIT) { + global->output[i].u32 = 0; + odp_atomic_init_u32(&global->atomic_private[i].u32, INIT_VAL); + } else { + global->output[i].u64 = 0; + odp_atomic_init_u64(&global->atomic_private[i].u64, INIT_VAL); + } + } + return 0; +} + +static int run_test(void *arg) +{ + uint64_t nsec; + odp_time_t t1, t2; + test_thread_ctx_t *thread_ctx = arg; + test_global_t *global = thread_ctx->global; + test_options_t *test_options = &global->test_options; + uint32_t num_round = test_options->num_round; + uint32_t idx = thread_ctx->idx; + test_fn_t test_func = thread_ctx->func; + op_bit_t type = thread_ctx->type; + void *val; + void *out; + uint32_t out_u32 = 0; + uint64_t out_u64 = 0; + + if (type == OP_32BIT) { + val = &global->atomic_u32; + out = &out_u32; + } else { + val = &global->atomic_u64; + out = &out_u64; + } + + if (global->test_options.private) { + if (type == OP_32BIT) + val = &global->atomic_private[idx].u32; + else + val = &global->atomic_private[idx].u64; + } + + /* Start all workers at the same time */ + odp_barrier_wait(&global->barrier); + + t1 = odp_time_local(); + + test_func(val, out, num_round); + + t2 = odp_time_local(); + + nsec = odp_time_diff_ns(t2, t1); + + /* Update stats */ + thread_ctx->nsec = nsec; + if (type == OP_32BIT) + global->output[idx].u32 = out_u32; + else + global->output[idx].u64 = out_u64; + + return 0; +} + +static int start_workers(test_global_t *global, odp_instance_t instance, + test_fn_t func, op_bit_t type) +{ + odph_thread_common_param_t param; + int i, ret; + test_options_t *test_options = &global->test_options; + int num_cpu = test_options->num_cpu; + odph_thread_param_t thr_param[num_cpu]; + + memset(¶m, 0, sizeof(odph_thread_common_param_t)); + param.instance = instance; + param.cpumask = &global->cpumask; + + memset(thr_param, 0, sizeof(thr_param)); + for (i = 0; i < num_cpu; i++) { + test_thread_ctx_t *thread_ctx = &global->thread_ctx[i]; + + thread_ctx->global = global; + thread_ctx->idx = i; + thread_ctx->func = func; + thread_ctx->type = type; + + thr_param[i].thr_type = ODP_THREAD_WORKER; + thr_param[i].start = run_test; + thr_param[i].arg = thread_ctx; + } + + ret = odph_thread_create(global->thread_tbl, ¶m, thr_param, num_cpu); + if (ret != num_cpu) { + ODPH_ERR("Failed to create all threads %i\n", ret); + return -1; + } + + return 0; +} + +static int validate_results(test_global_t *global, validate_fn_t validate, op_bit_t type) +{ + int i; + test_options_t *test_options = &global->test_options; + uint32_t num_round = test_options->num_round; + int num_cpu = test_options->num_cpu; + int private = global->test_options.private; + void *val; + void *out; + + for (i = 0; i < num_cpu; i++) { + if (type == OP_32BIT) { + out = &global->output[i].u32; + val = &global->atomic_u32; + if (private) + val = &global->atomic_private[i].u32; + } else { + out = &global->output[i].u64; + val = &global->atomic_u64; + if (private) + val = &global->atomic_private[i].u64; + } + + if (validate(val, out, num_round, num_cpu, private)) + return -1; + } + return 0; +} + +static void print_stat(test_global_t *global) +{ + int i, num; + double nsec_ave; + test_options_t *test_options = &global->test_options; + int num_cpu = test_options->num_cpu; + uint32_t num_round = test_options->num_round; + uint64_t nsec_sum = 0; + + for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) + nsec_sum += global->thread_ctx[i].nsec; + + if (nsec_sum == 0) { + printf("No results.\n"); + return; + } + + nsec_ave = nsec_sum / num_cpu; + num = 0; + + printf("---------------------------------------------\n"); + printf("Per thread results (Millions of ops per sec):\n"); + printf("---------------------------------------------\n"); + printf(" 1 2 3 4 5 6 7 8 9 10"); + + for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) { + if (global->thread_ctx[i].nsec) { + if ((num % 10) == 0) + printf("\n "); + + printf("%8.2f ", num_round / (global->thread_ctx[i].nsec / 1000.0)); + num++; + } + } + printf("\n\n"); + + printf("Average results over %i threads:\n", num_cpu); + printf("---------------------------------------\n"); + printf(" duration: %8.2f sec\n", nsec_ave / ODP_TIME_SEC_IN_NS); + printf(" operations per cpu: %8.2fM ops/sec\n", num_round / (nsec_ave / 1000.0)); + printf(" total operations: %8.2fM ops/sec\n", + (num_cpu * num_round) / (nsec_ave / 1000.0)); + printf("\n\n"); +} + +/** + * Test functions + */ +static test_case_t test_suite[] = { + TEST_INFO("odp_atomic_load_u32", test_atomic_load_u32, + validate_atomic_init_val_u32, OP_32BIT), + TEST_INFO("odp_atomic_store_u32", test_atomic_store_u32, + validate_atomic_num_round_u32, OP_32BIT), + TEST_INFO("odp_atomic_fetch_add_u32", test_atomic_fetch_add_u32, + validate_atomic_add_round_u32, OP_32BIT), + TEST_INFO("odp_atomic_add_u32", test_atomic_add_u32, + validate_atomic_add_round_u32, OP_32BIT), + TEST_INFO("odp_atomic_fetch_sub_u32", test_atomic_fetch_sub_u32, + validate_atomic_sub_round_u32, OP_32BIT), + TEST_INFO("odp_atomic_sub_u32", test_atomic_sub_u32, + validate_atomic_sub_round_u32, OP_32BIT), + TEST_INFO("odp_atomic_fetch_inc_u32", test_atomic_fetch_inc_u32, + validate_atomic_add_round_u32, OP_32BIT), + TEST_INFO("odp_atomic_inc_u32", test_atomic_inc_u32, + validate_atomic_add_round_u32, OP_32BIT), + TEST_INFO("odp_atomic_fetch_dec_u32", test_atomic_fetch_dec_u32, + validate_atomic_sub_round_u32, OP_32BIT), + TEST_INFO("odp_atomic_dec_u32", test_atomic_dec_u32, + validate_atomic_sub_round_u32, OP_32BIT), + TEST_INFO("odp_atomic_max_u32", test_atomic_max_u32, + validate_atomic_max_u32, OP_32BIT), + TEST_INFO("odp_atomic_min_u32", test_atomic_min_u32, + validate_atomic_min_u32, OP_32BIT), + TEST_INFO("odp_atomic_cas_u32", test_atomic_cas_u32, + validate_atomic_cas_u32, OP_32BIT), + TEST_INFO("odp_atomic_xchg_u32", test_atomic_xchg_u32, + validate_atomic_num_round_u32, OP_32BIT), + TEST_INFO("odp_atomic_load_acq_u32", test_atomic_load_acq_u32, + validate_atomic_init_val_u32, OP_32BIT), + TEST_INFO("odp_atomic_store_rel_u32", test_atomic_store_rel_u32, + validate_atomic_num_round_u32, OP_32BIT), + TEST_INFO("odp_atomic_add_rel_u32", test_atomic_add_rel_u32, + validate_atomic_add_round_u32, OP_32BIT), + TEST_INFO("odp_atomic_sub_rel_u32", test_atomic_sub_rel_u32, + validate_atomic_sub_round_u32, OP_32BIT), + TEST_INFO("odp_atomic_cas_acq_u32", test_atomic_cas_acq_u32, + validate_atomic_cas_u32, OP_32BIT), + TEST_INFO("odp_atomic_cas_rel_u32", test_atomic_cas_rel_u32, + validate_atomic_cas_u32, OP_32BIT), + TEST_INFO("odp_atomic_cas_acq_rel_u32", test_atomic_cas_acq_rel_u32, + validate_atomic_cas_u32, OP_32BIT), + TEST_INFO("odp_atomic_load_u64", test_atomic_load_u64, + validate_atomic_init_val_u64, OP_64BIT), + TEST_INFO("odp_atomic_store_u64", test_atomic_store_u64, + validate_atomic_num_round_u64, OP_64BIT), + TEST_INFO("odp_atomic_fetch_add_u64", test_atomic_fetch_add_u64, + validate_atomic_add_round_u64, OP_64BIT), + TEST_INFO("odp_atomic_add_u64", test_atomic_add_u64, + validate_atomic_add_round_u64, OP_64BIT), + TEST_INFO("odp_atomic_fetch_sub_u64", test_atomic_fetch_sub_u64, + validate_atomic_sub_round_u64, OP_64BIT), + TEST_INFO("odp_atomic_sub_u64", test_atomic_sub_u64, + validate_atomic_sub_round_u64, OP_64BIT), + TEST_INFO("odp_atomic_fetch_inc_u64", test_atomic_fetch_inc_u64, + validate_atomic_add_round_u64, OP_64BIT), + TEST_INFO("odp_atomic_inc_u64", test_atomic_inc_u64, + validate_atomic_add_round_u64, OP_64BIT), + TEST_INFO("odp_atomic_fetch_dec_u64", test_atomic_fetch_dec_u64, + validate_atomic_sub_round_u64, OP_64BIT), + TEST_INFO("odp_atomic_dec_u64", test_atomic_dec_u64, + validate_atomic_sub_round_u64, OP_64BIT), + TEST_INFO("odp_atomic_max_u64", test_atomic_max_u64, + validate_atomic_max_u64, OP_64BIT), + TEST_INFO("odp_atomic_min_u64", test_atomic_min_u64, + validate_atomic_min_u64, OP_64BIT), + TEST_INFO("odp_atomic_cas_u64", test_atomic_cas_u64, + validate_atomic_cas_u64, OP_64BIT), + TEST_INFO("odp_atomic_xchg_u64", test_atomic_xchg_u64, + validate_atomic_num_round_u64, OP_64BIT), + TEST_INFO("odp_atomic_load_acq_u64", test_atomic_load_acq_u64, + validate_atomic_init_val_u64, OP_64BIT), + TEST_INFO("odp_atomic_store_rel_u64", test_atomic_store_rel_u64, + validate_atomic_num_round_u64, OP_64BIT), + TEST_INFO("odp_atomic_add_rel_u64", test_atomic_add_rel_u64, + validate_atomic_add_round_u64, OP_64BIT), + TEST_INFO("odp_atomic_sub_rel_u64", test_atomic_sub_rel_u64, + validate_atomic_sub_round_u64, OP_64BIT), + TEST_INFO("odp_atomic_cas_acq_u64", test_atomic_cas_acq_u64, + validate_atomic_cas_u64, OP_64BIT), + TEST_INFO("odp_atomic_cas_rel_u64", test_atomic_cas_rel_u64, + validate_atomic_cas_u64, OP_64BIT), + TEST_INFO("odp_atomic_cas_acq_rel_u64", test_atomic_cas_acq_rel_u64, + validate_atomic_cas_u64, OP_64BIT) +}; + +int main(int argc, char **argv) +{ + odp_instance_t instance; + odp_init_t init; + odp_shm_t shm; + test_options_t test_options; + int num_tests, i; + + if (parse_options(argc, argv, &test_options)) + exit(EXIT_FAILURE); + + /* List features not to be used */ + odp_init_param_init(&init); + init.not_used.feat.cls = 1; + init.not_used.feat.compress = 1; + init.not_used.feat.crypto = 1; + init.not_used.feat.ipsec = 1; + init.not_used.feat.schedule = 1; + init.not_used.feat.stash = 1; + init.not_used.feat.timer = 1; + init.not_used.feat.tm = 1; + + /* Init ODP before calling anything else */ + if (odp_init_global(&instance, &init, NULL)) { + ODPH_ERR("Global init failed.\n"); + exit(EXIT_FAILURE); + } + + /* Init this thread */ + if (odp_init_local(instance, ODP_THREAD_CONTROL)) { + ODPH_ERR("Local init failed.\n"); + exit(EXIT_FAILURE); + } + + /* Reserve memory for global data from shared mem */ + shm = odp_shm_reserve("test_global", sizeof(test_global_t), + ODP_CACHE_LINE_SIZE, 0); + + if (shm == ODP_SHM_INVALID) { + ODPH_ERR("Shared memory reserve failed.\n"); + exit(EXIT_FAILURE); + } + + test_global = odp_shm_addr(shm); + if (test_global == NULL) { + ODPH_ERR("Shared memory alloc failed.\n"); + exit(EXIT_FAILURE); + } + memset(test_global, 0, sizeof(test_global_t)); + test_global->test_options = test_options; + + odp_sys_info_print(); + + if (set_num_cpu(test_global)) + exit(EXIT_FAILURE); + + print_info(&test_global->test_options); + + /* Loop all test cases */ + num_tests = sizeof(test_suite) / sizeof(test_suite[0]); + + for (i = 0; i < num_tests; i++) { + /* Initialize test variables */ + if (init_test(test_global, test_suite[i].name, test_suite[i].type)) { + ODPH_ERR("Failed to initialize atomics.\n"); + exit(EXIT_FAILURE); + } + + /* Start workers */ + if (start_workers(test_global, instance, test_suite[i].test_fn, test_suite[i].type)) + exit(EXIT_FAILURE); + + /* Wait workers to exit */ + odph_thread_join(test_global->thread_tbl, test_global->test_options.num_cpu); + + print_stat(test_global); + + /* Validate test results */ + if (validate_results(test_global, test_suite[i].validate_fn, test_suite[i].type)) { + ODPH_ERR("Test %s result validation failed.\n", test_suite[i].name); + exit(EXIT_FAILURE); + } + } + + if (odp_shm_free(shm)) { + ODPH_ERR("Shm free failed.\n"); + exit(EXIT_FAILURE); + } + + if (odp_term_local()) { + ODPH_ERR("Local terminate failed.\n"); + exit(EXIT_FAILURE); + } + + if (odp_term_global(instance)) { + ODPH_ERR("Global terminate failed.\n"); + exit(EXIT_FAILURE); + } + + return 0; +} diff --git a/test/performance/odp_mem_perf.c b/test/performance/odp_mem_perf.c index a833a04e8..0470d337a 100644 --- a/test/performance/odp_mem_perf.c +++ b/test/performance/odp_mem_perf.c @@ -135,7 +135,7 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) static int set_num_cpu(test_global_t *global) { - int ret; + int ret, max_num; test_options_t *test_options = &global->test_options; int num_cpu = test_options->num_cpu; @@ -145,7 +145,11 @@ static int set_num_cpu(test_global_t *global) return -1; } - ret = odp_cpumask_default_worker(&global->cpumask, num_cpu); + max_num = num_cpu; + if (num_cpu == 0) + max_num = ODP_THREAD_COUNT_MAX - 1; + + ret = odp_cpumask_default_worker(&global->cpumask, max_num); if (num_cpu && ret != num_cpu) { ODPH_ERR("Too many workers. Max supported %i.\n", ret); @@ -154,6 +158,11 @@ static int set_num_cpu(test_global_t *global) /* Zero: all available workers */ if (num_cpu == 0) { + if (ret > max_num) { + ODPH_ERR("Too many cpus from odp_cpumask_default_worker(): %i\n", ret); + return -1; + } + num_cpu = ret; test_options->num_cpu = num_cpu; } diff --git a/test/performance/odp_sched_latency_run.sh b/test/performance/odp_sched_latency_run.sh index bcccd77a9..372fdb166 100755 --- a/test/performance/odp_sched_latency_run.sh +++ b/test/performance/odp_sched_latency_run.sh @@ -14,9 +14,13 @@ ALL=0 run() { echo odp_sched_latency_run starts requesting $1 worker threads - echo =============================================== + echo ========================================================= - $TEST_DIR/odp_sched_latency${EXEEXT} -c $1 || exit $? + if [ $(nproc) -lt $1 ]; then + echo "Not enough CPU cores. Skipping test." + else + $TEST_DIR/odp_sched_latency${EXEEXT} -c $1 || exit $? + fi } run 1 diff --git a/test/performance/odp_scheduling_run.sh b/test/performance/odp_scheduling_run.sh index 082dc4521..2b4281ee9 100755 --- a/test/performance/odp_scheduling_run.sh +++ b/test/performance/odp_scheduling_run.sh @@ -14,14 +14,17 @@ ALL=0 run() { echo odp_scheduling_run starts requesting $1 worker threads - echo =============================================== + echo ====================================================== - $TEST_DIR/odp_scheduling${EXEEXT} -c $1 - - RET_VAL=$? - if [ $RET_VAL -ne 0 ]; then - echo odp_scheduling FAILED - exit $RET_VAL + if [ $(nproc) -lt $1 ]; then + echo "Not enough CPU cores. Skipping test." + else + $TEST_DIR/odp_scheduling${EXEEXT} -c $1 + RET_VAL=$? + if [ $RET_VAL -ne 0 ]; then + echo odp_scheduling FAILED + exit $RET_VAL + fi fi } diff --git a/test/validation/api/atomic/atomic.c b/test/validation/api/atomic/atomic.c index 36484295f..907624bb0 100644 --- a/test/validation/api/atomic/atomic.c +++ b/test/validation/api/atomic/atomic.c @@ -28,11 +28,13 @@ #define CHECK_MAX_MIN (1 << 0) #define CHECK_XCHG (1 << 2) +#define CHECK_CAS_128 (1 << 4) typedef __volatile uint32_t volatile_u32_t; typedef __volatile uint64_t volatile_u64_t; typedef struct { + odp_atomic_u128_t a128u; odp_atomic_u64_t a64u; odp_atomic_u64_t a64u_min; odp_atomic_u64_t a64u_max; @@ -489,6 +491,90 @@ static void test_atomic_non_relaxed_64(void) } } +static void test_atomic_relaxed_128(void) +{ + int i, ret; + odp_u128_t old, new; + odp_atomic_u128_t *a128u = &global_mem->a128u; + + odp_barrier_wait(&global_mem->global_barrier); + + for (i = 0; i < CNT; i++) { + old = odp_atomic_load_u128(a128u); + + do { + new.u64[0] = old.u64[0] + 2; + new.u64[1] = old.u64[1] + 1; + + ret = odp_atomic_cas_u128(a128u, &old, new); + + } while (ret == 0); + } +} + +static void test_atomic_non_relaxed_128_acq(void) +{ + int i, ret; + odp_u128_t old, new; + odp_atomic_u128_t *a128u = &global_mem->a128u; + + odp_barrier_wait(&global_mem->global_barrier); + + for (i = 0; i < CNT; i++) { + old = odp_atomic_load_u128(a128u); + + do { + new.u64[0] = old.u64[0] + 2; + new.u64[1] = old.u64[1] + 1; + + ret = odp_atomic_cas_acq_u128(a128u, &old, new); + + } while (ret == 0); + } +} + +static void test_atomic_non_relaxed_128_rel(void) +{ + int i, ret; + odp_u128_t old, new; + odp_atomic_u128_t *a128u = &global_mem->a128u; + + odp_barrier_wait(&global_mem->global_barrier); + + for (i = 0; i < CNT; i++) { + old = odp_atomic_load_u128(a128u); + + do { + new.u64[0] = old.u64[0] + 2; + new.u64[1] = old.u64[1] + 1; + + ret = odp_atomic_cas_rel_u128(a128u, &old, new); + + } while (ret == 0); + } +} + +static void test_atomic_non_relaxed_128_acq_rel(void) +{ + int i, ret; + odp_u128_t old, new; + odp_atomic_u128_t *a128u = &global_mem->a128u; + + odp_barrier_wait(&global_mem->global_barrier); + + for (i = 0; i < CNT; i++) { + old = odp_atomic_load_u128(a128u); + + do { + new.u64[0] = old.u64[0] + 2; + new.u64[1] = old.u64[1] + 1; + + ret = odp_atomic_cas_acq_rel_u128(a128u, &old, new); + + } while (ret == 0); + } +} + static void test_atomic_inc_dec_32(void) { test_atomic_inc_32(); @@ -561,6 +647,14 @@ static void test_atomic_cas_inc_dec_64(void) test_atomic_cas_dec_64(); } +static void test_atomic_cas_inc_128(void) +{ + test_atomic_relaxed_128(); + test_atomic_non_relaxed_128_acq(); + test_atomic_non_relaxed_128_rel(); + test_atomic_non_relaxed_128_acq_rel(); +} + static void test_atomic_init(void) { odp_atomic_init_u32(&global_mem->a32u, 0); @@ -571,6 +665,12 @@ static void test_atomic_init(void) odp_atomic_init_u64(&global_mem->a64u_max, 0); odp_atomic_init_u32(&global_mem->a32u_xchg, 0); odp_atomic_init_u64(&global_mem->a64u_xchg, 0); + + odp_u128_t a128u_tmp; + + a128u_tmp.u64[0] = 0; + a128u_tmp.u64[1] = 0; + odp_atomic_init_u128(&global_mem->a128u, a128u_tmp); } static void test_atomic_store(void) @@ -583,6 +683,12 @@ static void test_atomic_store(void) odp_atomic_store_u64(&global_mem->a64u_max, U64_INIT_VAL); odp_atomic_store_u32(&global_mem->a32u_xchg, U32_INIT_VAL); odp_atomic_store_u64(&global_mem->a64u_xchg, U64_INIT_VAL); + + odp_u128_t a128u_tmp; + + a128u_tmp.u64[0] = U64_INIT_VAL; + a128u_tmp.u64[1] = U64_INIT_VAL; + odp_atomic_store_u128(&global_mem->a128u, a128u_tmp); } static void test_atomic_validate(int check) @@ -590,6 +696,20 @@ static void test_atomic_validate(int check) CU_ASSERT(U32_INIT_VAL == odp_atomic_load_u32(&global_mem->a32u)); CU_ASSERT(U64_INIT_VAL == odp_atomic_load_u64(&global_mem->a64u)); + odp_u128_t a128u_tmp; + + a128u_tmp = odp_atomic_load_u128(&global_mem->a128u); + + if (check & CHECK_CAS_128) { + uint64_t iterations = 0; + + iterations = a128u_tmp.u64[0] - a128u_tmp.u64[1]; + CU_ASSERT(iterations == 4 * CNT * global_mem->g_num_threads); + } else { + CU_ASSERT(U64_INIT_VAL == a128u_tmp.u64[0]); + CU_ASSERT(U64_INIT_VAL == a128u_tmp.u64[1]); + } + if (check & CHECK_MAX_MIN) { CU_ASSERT(odp_atomic_load_u32(&global_mem->a32u_max) > odp_atomic_load_u32(&global_mem->a32u_min)); @@ -763,6 +883,7 @@ static int test_atomic_cas_inc_dec_thread(void *arg UNUSED) per_thread_mem = thread_init(); test_atomic_cas_inc_dec_32(); test_atomic_cas_inc_dec_64(); + test_atomic_cas_inc_128(); thread_finalize(per_thread_mem); @@ -807,51 +928,9 @@ static void test_atomic_functional(int func_ptr(void *), int check) test_atomic_validate(check); } -static void atomic_test_atomic_inc_dec(void) -{ - test_atomic_functional(test_atomic_inc_dec_thread, 0); -} - -static void atomic_test_atomic_add_sub(void) -{ - test_atomic_functional(test_atomic_add_sub_thread, 0); -} - -static void atomic_test_atomic_fetch_inc_dec(void) -{ - test_atomic_functional(test_atomic_fetch_inc_dec_thread, 0); -} - -static void atomic_test_atomic_fetch_add_sub(void) -{ - test_atomic_functional(test_atomic_fetch_add_sub_thread, 0); -} - -static void atomic_test_atomic_max_min(void) -{ - test_atomic_functional(test_atomic_max_min_thread, CHECK_MAX_MIN); -} - -static void atomic_test_atomic_cas_inc_dec(void) -{ - test_atomic_functional(test_atomic_cas_inc_dec_thread, 0); -} - -static void atomic_test_atomic_xchg(void) -{ - test_atomic_functional(test_atomic_xchg_thread, CHECK_XCHG); -} - -static void atomic_test_atomic_non_relaxed(void) -{ - test_atomic_functional(test_atomic_non_relaxed_thread, - CHECK_MAX_MIN | CHECK_XCHG); -} - -static void atomic_test_atomic_op_lock_free(void) +static void test_atomic_op_lock_free_set(void) { odp_atomic_op_t atomic_op; - int ret_null, ret; memset(&atomic_op, 0xff, sizeof(odp_atomic_op_t)); atomic_op.all_bits = 0; @@ -897,6 +976,12 @@ static void atomic_test_atomic_op_lock_free(void) CU_ASSERT(atomic_op.all_bits != 0); atomic_op.op.dec = 0; CU_ASSERT(atomic_op.all_bits == 0); +} + +static void test_atomic_op_lock_free_64(void) +{ + odp_atomic_op_t atomic_op; + int ret_null, ret; memset(&atomic_op, 0xff, sizeof(odp_atomic_op_t)); ret = odp_atomic_lock_free_u64(&atomic_op); @@ -954,6 +1039,93 @@ static void atomic_test_atomic_op_lock_free(void) } } +static void test_atomic_op_lock_free_128(void) +{ + odp_atomic_op_t atomic_op; + int ret_null, ret; + + memset(&atomic_op, 0xff, sizeof(odp_atomic_op_t)); + ret = odp_atomic_lock_free_u128(&atomic_op); + ret_null = odp_atomic_lock_free_u128(NULL); + + CU_ASSERT(ret == ret_null); + + /* Init operation is not atomic by the spec. Call to + * odp_atomic_lock_free_u128() zeros it but never sets it. */ + + if (ret == 0) { + /* none are lock free */ + CU_ASSERT(atomic_op.all_bits == 0); + CU_ASSERT(atomic_op.op.init == 0); + CU_ASSERT(atomic_op.op.load == 0); + CU_ASSERT(atomic_op.op.store == 0); + CU_ASSERT(atomic_op.op.cas == 0); + } + + if (ret == 1) { + /* some are lock free */ + CU_ASSERT(atomic_op.all_bits != 0); + CU_ASSERT(atomic_op.op.init == 0); + } + + if (ret == 2) { + /* all are lock free */ + CU_ASSERT(atomic_op.all_bits != 0); + CU_ASSERT(atomic_op.op.init == 0); + CU_ASSERT(atomic_op.op.load == 1); + CU_ASSERT(atomic_op.op.store == 1); + CU_ASSERT(atomic_op.op.cas == 1); + } +} + +static void atomic_test_atomic_inc_dec(void) +{ + test_atomic_functional(test_atomic_inc_dec_thread, 0); +} + +static void atomic_test_atomic_add_sub(void) +{ + test_atomic_functional(test_atomic_add_sub_thread, 0); +} + +static void atomic_test_atomic_fetch_inc_dec(void) +{ + test_atomic_functional(test_atomic_fetch_inc_dec_thread, 0); +} + +static void atomic_test_atomic_fetch_add_sub(void) +{ + test_atomic_functional(test_atomic_fetch_add_sub_thread, 0); +} + +static void atomic_test_atomic_max_min(void) +{ + test_atomic_functional(test_atomic_max_min_thread, CHECK_MAX_MIN); +} + +static void atomic_test_atomic_cas_inc_dec(void) +{ + test_atomic_functional(test_atomic_cas_inc_dec_thread, CHECK_CAS_128); +} + +static void atomic_test_atomic_xchg(void) +{ + test_atomic_functional(test_atomic_xchg_thread, CHECK_XCHG); +} + +static void atomic_test_atomic_non_relaxed(void) +{ + test_atomic_functional(test_atomic_non_relaxed_thread, + CHECK_MAX_MIN | CHECK_XCHG); +} + +static void atomic_test_atomic_op_lock_free(void) +{ + test_atomic_op_lock_free_set(); + test_atomic_op_lock_free_64(); + test_atomic_op_lock_free_128(); +} + odp_testinfo_t atomic_suite_atomic[] = { ODP_TEST_INFO(atomic_test_atomic_inc_dec), ODP_TEST_INFO(atomic_test_atomic_add_sub), diff --git a/test/validation/api/chksum/chksum.c b/test/validation/api/chksum/chksum.c index ce905c04b..86306ab0b 100644 --- a/test/validation/api/chksum/chksum.c +++ b/test/validation/api/chksum/chksum.c @@ -313,10 +313,122 @@ static void chksum_ones_complement_udp_long(void) CU_ASSERT(res == UDP_LONG_CHKSUM); } +static uint16_t chksum_rfc1071(const void *p, uint32_t len) +{ + uint32_t sum = 0; + const uint16_t *data = p; + + while (len > 1) { + sum += *data++; + len -= 2; + } + + /* Add left-over byte, if any */ + if (len > 0) { + uint16_t left_over = 0; + + *(uint8_t *)&left_over = *(const uint8_t *)data; + sum += left_over; + } + + /* Fold 32-bit sum to 16 bits */ + while (sum >> 16) + sum = (sum & 0xffff) + (sum >> 16); + + return sum; +} + +/* + * 64-bit KISS RNGs + * George Marsaglia + * https://www.thecodingforums.com/threads/64-bit-kiss-rngs.673657 + */ + +static unsigned long long x = 1234567890987654321ULL, c = 123456123456123456ULL, + y = 362436362436362436ULL, z = 1066149217761810ULL, t; + +#define MWC (t = (x << 58) + c, c = (x >> 6), x += t, c += (x < t), x) +#define XSH (y ^= (y << 13), y ^= (y >> 17), y ^= (y << 43)) +#define CNG (z = 6906969069LL * z + 1234567) +#define KISS (MWC + XSH + CNG) + +/* + * Test with pseudorandom data and different data lengths and alignments. + */ +static void chksum_ones_complement_pseudorandom(void) +{ + const int size = 32 * 1024; + const unsigned long page = 4096; + /* Allocate some extra pages for alignment and length. */ + uint8_t *buf = (uint8_t *)malloc(size + page * 4); + uint8_t *data = (uint8_t *)(((uintptr_t)buf + (page - 1)) & ~(page - 1)); + + for (int i = 0; i < (size + (int)page * 3) / 8; i++) + ((uint64_t *)(uintptr_t)data)[i] = KISS; + + /* Test data lengths from 1 to more than 9000 bytes. */ + for (int len = 1; len < 10000; len++) { + /* + * To avoid spending too much time on long data, the number of + * rounds goes down as data length goes up. + */ + int rounds = 1000000000 / (len * len + 1000000); + + for (int i = 0; i < rounds; i++) { + /* Align p to two bytes. */ + uint8_t *p = data + (KISS & (size - 1) & ~1UL); + /* + * Generate some fresh random bits at the start of the + * data to be checksummed. + */ + uint64_t rnd = KISS; + + memcpy(p, &rnd, sizeof(rnd)); + CU_ASSERT(chksum_rfc1071(p, len) == + odp_chksum_ones_comp16(p, len)); + } + } + + free(buf); +} + +/* + * Test with very long data with most of the bits set. The idea is to + * maximize the number of carries. + */ +static void chksum_ones_complement_very_long(void) +{ + const int size = 64 * 1024; + const unsigned long page = 4096; + /* Allocate two extra pages for alignment. */ + uint8_t *buf = (uint8_t *)malloc(size + page * 2); + uint8_t *data = (uint8_t *)(((uintptr_t)buf + (page - 1)) & ~(page - 1)); + + /* Start with all bits set. */ + memset(data, 0xff, size + page); + + for (int i = 0; i < 100; i++) { + for (int len = size - 8; len <= size; len++) { + /* Alignment 0, 2, 4, 6, 8. */ + for (int a = 0; a <= 8; a += 2) + CU_ASSERT(chksum_rfc1071(data + a, len) == + odp_chksum_ones_comp16(data + a, len)); + } + + /* Turn off some random bits in the data. */ + uint64_t rnd = KISS; + ((uint8_t *)data)[rnd & (size - 1)] &= (rnd >> 32) & 0xff; + } + + free(buf); +} + odp_testinfo_t chksum_suite[] = { ODP_TEST_INFO(chksum_ones_complement_ip), ODP_TEST_INFO(chksum_ones_complement_udp), ODP_TEST_INFO(chksum_ones_complement_udp_long), + ODP_TEST_INFO(chksum_ones_complement_pseudorandom), + ODP_TEST_INFO(chksum_ones_complement_very_long), ODP_TEST_INFO_NULL, }; diff --git a/test/validation/api/ipsec/ipsec.c b/test/validation/api/ipsec/ipsec.c index 43322e36c..1458e9953 100644 --- a/test/validation/api/ipsec/ipsec.c +++ b/test/validation/api/ipsec/ipsec.c @@ -261,6 +261,18 @@ int ipsec_check_esp_chacha20_poly1305(void) ODP_AUTH_ALG_CHACHA20_POLY1305, 0); } +int ipsec_check_test_sa_update_seq_num(void) +{ + odp_ipsec_capability_t capa; + + odp_ipsec_capability(&capa); + + if (!capa.test.sa_operations.seq_num) + return ODP_TEST_INACTIVE; + + return ODP_TEST_ACTIVE; +} + void ipsec_sa_param_fill(odp_ipsec_sa_param_t *param, odp_bool_t in, odp_bool_t ah, @@ -734,6 +746,48 @@ static int ipsec_send_out_one(const ipsec_test_part *part, return num_out; } +int ipsec_test_sa_update_seq_num(odp_ipsec_sa_t sa, uint32_t seq_num) +{ + odp_ipsec_test_sa_operation_t sa_op; + odp_ipsec_test_sa_param_t sa_param; + + sa_op = ODP_IPSEC_TEST_SA_UPDATE_SEQ_NUM; + sa_param.seq_num = seq_num; + + return odp_ipsec_test_sa_update(sa, sa_op, &sa_param); +} + +static void ipsec_pkt_seq_num_check(odp_packet_t pkt, uint32_t seq_num) +{ + uint32_t l3_off = odp_packet_l3_offset(pkt); + uint32_t l4_off; + odph_ipv4hdr_t ip; + + CU_ASSERT_NOT_EQUAL_FATAL(ODP_PACKET_OFFSET_INVALID, l3_off); + CU_ASSERT_EQUAL_FATAL(0, odp_packet_copy_to_mem(pkt, l3_off, sizeof(ip), + &ip)); + + if (ODPH_IPV4HDR_VER(ip.ver_ihl) == ODPH_IPV4) { + l4_off = l3_off + (ODPH_IPV4HDR_IHL(ip.ver_ihl) * 4); + + if (ip.proto == ODPH_IPPROTO_ESP) { + odph_esphdr_t esp; + + odp_packet_copy_to_mem(pkt, l4_off, sizeof(esp), &esp); + CU_ASSERT_EQUAL(odp_be_to_cpu_32(esp.seq_no), seq_num); + } else if (ip.proto == ODPH_IPPROTO_AH) { + odph_ahhdr_t ah; + + odp_packet_copy_to_mem(pkt, l4_off, sizeof(ah), &ah); + CU_ASSERT_EQUAL(odp_be_to_cpu_32(ah.seq_no), seq_num); + } else { + CU_FAIL("Unexpected IP Proto"); + } + } else { + CU_FAIL("Unexpected IP Version"); + } +} + static void ipsec_pkt_proto_err_set(odp_packet_t pkt) { uint32_t l3_off = odp_packet_l3_offset(pkt); @@ -898,6 +952,9 @@ void ipsec_check_out_in_one(const ipsec_test_part *part, CU_ASSERT_FATAL(odp_packet_len(pkto[i]) <= sizeof(pkt_in.data)); + if (part->flags.test_sa_seq_num) + ipsec_pkt_seq_num_check(pkto[i], part->out[i].seq_num); + if (part->flags.stats == IPSEC_TEST_STATS_PROTO_ERR) ipsec_pkt_proto_err_set(pkto[i]); diff --git a/test/validation/api/ipsec/ipsec.h b/test/validation/api/ipsec/ipsec.h index a9213b420..3bbcb7b64 100644 --- a/test/validation/api/ipsec/ipsec.h +++ b/test/validation/api/ipsec/ipsec.h @@ -59,6 +59,7 @@ typedef struct { odp_bool_t lookup; odp_bool_t ah; odp_bool_t inline_hdr_in_packet; + odp_bool_t test_sa_seq_num; enum ipsec_test_stats stats; } ipsec_test_flags; @@ -73,6 +74,7 @@ typedef struct { const ipsec_test_packet *pkt_res; odp_proto_l3_type_t l3_type; odp_proto_l4_type_t l4_type; + uint32_t seq_num; } out[1]; struct { odp_ipsec_op_status_t status; @@ -101,6 +103,7 @@ void ipsec_check_out_one(const ipsec_test_part *part, odp_ipsec_sa_t sa); void ipsec_check_out_in_one(const ipsec_test_part *part, odp_ipsec_sa_t sa, odp_ipsec_sa_t sa_in); +int ipsec_test_sa_update_seq_num(odp_ipsec_sa_t sa, uint32_t seq_num); int ipsec_check(odp_bool_t ah, odp_cipher_alg_t cipher, @@ -128,5 +131,6 @@ int ipsec_check_esp_null_aes_gmac_128(void); int ipsec_check_esp_null_aes_gmac_192(void); int ipsec_check_esp_null_aes_gmac_256(void); int ipsec_check_esp_chacha20_poly1305(void); +int ipsec_check_test_sa_update_seq_num(void); #endif diff --git a/test/validation/api/ipsec/ipsec_test_out.c b/test/validation/api/ipsec/ipsec_test_out.c index b4065d667..6f285d59a 100644 --- a/test/validation/api/ipsec/ipsec_test_out.c +++ b/test/validation/api/ipsec/ipsec_test_out.c @@ -460,6 +460,21 @@ static void test_out_in_common(ipsec_test_flags *flags, test_ipsec_stats_zero_assert(&stats); } + if (flags->test_sa_seq_num) { + int rc; + + test.out[0].seq_num = 0x1235; + rc = ipsec_test_sa_update_seq_num(sa_out, test.out[0].seq_num); + + /* Skip further checks related to this specific test if the + * SA update call was not successful. + */ + if (rc < 0) { + printf("\t >> skipped"); + test.flags.test_sa_seq_num = false; + } + } + ipsec_check_out_in_one(&test, sa_out, sa_in); if (flags->stats == IPSEC_TEST_STATS_SUCCESS) { @@ -1284,6 +1299,7 @@ static void test_sa_info(void) param_in.inbound.antireplay_ws = 32; sa_in = odp_ipsec_sa_create(¶m_in); + CU_ASSERT_FATAL(sa_in != ODP_IPSEC_SA_INVALID); memset(&info_out, 0, sizeof(info_out)); CU_ASSERT_EQUAL_FATAL(0, odp_ipsec_sa_info(sa_out, &info_out)); @@ -1362,6 +1378,46 @@ static void test_sa_info(void) ipsec_sa_destroy(sa_out); ipsec_sa_destroy(sa_in); + + /* + * Additional check for SA lookup parameters. Let's use transport + * mode SA and ODP_IPSEC_DSTADD_SPI lookup mode. + */ + ipsec_sa_param_fill(¶m_in, + true, false, 123, NULL, + ODP_CIPHER_ALG_AES_CBC, &key_a5_128, + ODP_AUTH_ALG_SHA1_HMAC, &key_5a_160, + NULL, NULL); + param_in.inbound.lookup_mode = ODP_IPSEC_LOOKUP_DSTADDR_SPI; + param_in.inbound.lookup_param.ip_version = ODP_IPSEC_IPV4; + param_in.inbound.lookup_param.dst_addr = &dst; + sa_in = odp_ipsec_sa_create(¶m_in); + CU_ASSERT_FATAL(sa_in != ODP_IPSEC_SA_INVALID); + + memset(&info_in, 0, sizeof(info_in)); + CU_ASSERT_FATAL(odp_ipsec_sa_info(sa_in, &info_in) == 0); + + CU_ASSERT(info_in.param.inbound.lookup_mode == + ODP_IPSEC_LOOKUP_DSTADDR_SPI); + CU_ASSERT_FATAL(info_in.param.inbound.lookup_param.dst_addr == + &info_in.inbound.lookup_param.dst_addr); + CU_ASSERT(!memcmp(info_in.param.inbound.lookup_param.dst_addr, + &dst, + ODP_IPV4_ADDR_SIZE)); + ipsec_sa_destroy(sa_in); +} + +static void test_test_sa_update_seq_num(void) +{ + ipsec_test_flags flags; + + memset(&flags, 0, sizeof(flags)); + flags.display_algo = true; + flags.test_sa_seq_num = true; + + test_esp_out_in_all(&flags); + + printf("\n "); } static void ipsec_test_capability(void) @@ -1371,6 +1427,53 @@ static void ipsec_test_capability(void) CU_ASSERT(odp_ipsec_capability(&capa) == 0); } +static void ipsec_test_default_values(void) +{ + odp_ipsec_config_t config; + odp_ipsec_sa_param_t sa_param; + + memset(&config, 0x55, sizeof(config)); + memset(&sa_param, 0x55, sizeof(sa_param)); + + odp_ipsec_config_init(&config); + CU_ASSERT(config.inbound.lookup.min_spi == 0); + CU_ASSERT(config.inbound.lookup.max_spi == UINT32_MAX); + CU_ASSERT(config.inbound.lookup.spi_overlap == 0); + CU_ASSERT(config.inbound.retain_outer == ODP_PROTO_LAYER_NONE); + CU_ASSERT(config.inbound.parse_level == ODP_PROTO_LAYER_NONE); + CU_ASSERT(config.inbound.chksums.all_chksum == 0); + CU_ASSERT(config.outbound.all_chksum == 0); + CU_ASSERT(!config.stats_en); + + odp_ipsec_sa_param_init(&sa_param); + CU_ASSERT(sa_param.proto == ODP_IPSEC_ESP); + CU_ASSERT(sa_param.crypto.cipher_alg == ODP_CIPHER_ALG_NULL); + CU_ASSERT(sa_param.crypto.auth_alg == ODP_AUTH_ALG_NULL); + CU_ASSERT(sa_param.opt.esn == 0); + CU_ASSERT(sa_param.opt.udp_encap == 0); + CU_ASSERT(sa_param.opt.copy_dscp == 0); + CU_ASSERT(sa_param.opt.copy_flabel == 0); + CU_ASSERT(sa_param.opt.copy_df == 0); + CU_ASSERT(sa_param.opt.dec_ttl == 0); + CU_ASSERT(sa_param.lifetime.soft_limit.bytes == 0); + CU_ASSERT(sa_param.lifetime.soft_limit.packets == 0); + CU_ASSERT(sa_param.lifetime.hard_limit.bytes == 0); + CU_ASSERT(sa_param.lifetime.hard_limit.packets == 0); + CU_ASSERT(sa_param.context == NULL); + CU_ASSERT(sa_param.context_len == 0); + CU_ASSERT(sa_param.inbound.lookup_mode == ODP_IPSEC_LOOKUP_DISABLED); + CU_ASSERT(sa_param.inbound.antireplay_ws == 0); + CU_ASSERT(sa_param.inbound.pipeline == ODP_IPSEC_PIPELINE_NONE); + CU_ASSERT(sa_param.outbound.tunnel.type == ODP_IPSEC_TUNNEL_IPV4); + CU_ASSERT(sa_param.outbound.tunnel.ipv4.dscp == 0); + CU_ASSERT(sa_param.outbound.tunnel.ipv4.df == 0); + CU_ASSERT(sa_param.outbound.tunnel.ipv4.ttl == 255); + CU_ASSERT(sa_param.outbound.tunnel.ipv6.flabel == 0); + CU_ASSERT(sa_param.outbound.tunnel.ipv6.dscp == 0); + CU_ASSERT(sa_param.outbound.tunnel.ipv6.hlimit == 255); + CU_ASSERT(sa_param.outbound.frag_mode == ODP_IPSEC_FRAG_DISABLED); +} + static void test_ipsec_stats(void) { ipsec_test_flags flags; @@ -1394,6 +1497,7 @@ static void test_ipsec_stats(void) odp_testinfo_t ipsec_out_suite[] = { ODP_TEST_INFO(ipsec_test_capability), + ODP_TEST_INFO(ipsec_test_default_values), ODP_TEST_INFO_CONDITIONAL(test_out_ipv4_ah_sha256, ipsec_check_ah_sha256), ODP_TEST_INFO_CONDITIONAL(test_out_ipv4_ah_sha256_tun_ipv4, @@ -1444,6 +1548,8 @@ odp_testinfo_t ipsec_out_suite[] = { ipsec_check_esp_null_sha256), ODP_TEST_INFO_CONDITIONAL(test_sa_info, ipsec_check_esp_aes_cbc_128_sha1), + ODP_TEST_INFO_CONDITIONAL(test_test_sa_update_seq_num, + ipsec_check_test_sa_update_seq_num), ODP_TEST_INFO(test_esp_out_in_all_basic), ODP_TEST_INFO_CONDITIONAL(test_esp_out_in_all_hdr_in_packet, is_out_mode_inline), diff --git a/test/validation/api/pktio/lso.c b/test/validation/api/pktio/lso.c index 5d0596861..e3ef57bf5 100644 --- a/test/validation/api/pktio/lso.c +++ b/test/validation/api/pktio/lso.c @@ -772,6 +772,7 @@ static void lso_send_ipv4(int use_opt) ODPH_DBG(" LSO segment[%i] payload: %u bytes\n", i, payload_len); + CU_ASSERT(odp_packet_has_ipv4(packet[i])); CU_ASSERT(odp_packet_has_ipfrag(packet[i])); CU_ASSERT(odp_packet_has_error(packet[i]) == 0); CU_ASSERT(payload_len <= IPV4_MAX_PAYLOAD); diff --git a/test/validation/api/pktio/pktio.c b/test/validation/api/pktio/pktio.c index 47320e2e8..3f8df07f3 100644 --- a/test/validation/api/pktio/pktio.c +++ b/test/validation/api/pktio/pktio.c @@ -1135,7 +1135,7 @@ static void pktio_test_recv_multi_event(void) static void pktio_test_recv_queue(void) { odp_pktio_t pktio_tx, pktio_rx; - odp_pktio_t pktio[MAX_NUM_IFACES]; + odp_pktio_t pktio[MAX_NUM_IFACES] = {0}; odp_pktio_capability_t capa; odp_pktin_queue_param_t in_queue_param; odp_pktout_queue_param_t out_queue_param; @@ -1247,7 +1247,7 @@ static void pktio_test_recv_queue(void) static void test_recv_tmo(recv_tmo_mode_e mode) { odp_pktio_t pktio_tx, pktio_rx; - odp_pktio_t pktio[MAX_NUM_IFACES]; + odp_pktio_t pktio[MAX_NUM_IFACES] = {0}; odp_pktio_capability_t capa; odp_pktin_queue_param_t in_queue_param; odp_pktout_queue_t pktout_queue; @@ -2302,7 +2302,7 @@ static int pktio_check_pktin_ts(void) static void pktio_test_pktin_ts(void) { odp_pktio_t pktio_tx, pktio_rx; - odp_pktio_t pktio[MAX_NUM_IFACES]; + odp_pktio_t pktio[MAX_NUM_IFACES] = {0}; pktio_info_t pktio_rx_info; odp_pktio_capability_t capa; odp_pktio_config_t config; @@ -2417,7 +2417,7 @@ static int pktio_check_pktout_ts(void) static void pktio_test_pktout_ts(void) { odp_packet_t pkt_tbl[TX_BATCH_LEN]; - odp_pktio_t pktio[MAX_NUM_IFACES]; + odp_pktio_t pktio[MAX_NUM_IFACES] = {0}; odp_pktout_queue_t pktout_queue; odp_pktio_t pktio_tx, pktio_rx; uint32_t pkt_seq[TX_BATCH_LEN]; @@ -3381,7 +3381,7 @@ static void pktio_test_pktv_pktin_queue_config_sched(void) static void pktio_test_recv_maxlen_set(void) { odp_pktio_t pktio_tx, pktio_rx; - odp_pktio_t pktio[MAX_NUM_IFACES]; + odp_pktio_t pktio[MAX_NUM_IFACES] = {0}; pktio_info_t pktio_rx_info; odp_pktio_capability_t capa; odp_pktio_config_t config; diff --git a/test/validation/api/queue/queue.c b/test/validation/api/queue/queue.c index 256194f81..b5d594a9a 100644 --- a/test/validation/api/queue/queue.c +++ b/test/validation/api/queue/queue.c @@ -759,6 +759,8 @@ static void queue_test_info(void) CU_ASSERT(info.param.sched.lock_count == lock_count); odp_queue_print(q_order); + odp_queue_print_all(); + CU_ASSERT(odp_queue_destroy(q_plain) == 0); CU_ASSERT(odp_queue_destroy(q_order) == 0); } diff --git a/test/validation/api/timer/timer.c b/test/validation/api/timer/timer.c index 8943d4d97..f2cc93cb8 100644 --- a/test/validation/api/timer/timer.c +++ b/test/validation/api/timer/timer.c @@ -534,6 +534,8 @@ static void timer_test_event_type(odp_queue_type_t queue_type, odp_time_t t1, t2; uint64_t period_ns, period_tick, duration_ns; int i, ret, num_tmo; + const char *user_ctx = "User context"; + int test_print = 0; int num = 5; odp_timer_t timer[num]; @@ -562,6 +564,7 @@ static void timer_test_event_type(odp_queue_type_t queue_type, } else if (event_type == ODP_EVENT_TIMEOUT) { pool_param.type = ODP_POOL_TIMEOUT; pool_param.tmo.num = num; + test_print = 1; } else { CU_FAIL("Bad event_type"); return; @@ -589,7 +592,8 @@ static void timer_test_event_type(odp_queue_type_t queue_type, ODPH_DBG(" max_tmo %" PRIu64 "\n", timer_param.max_tmo); ODPH_DBG(" period_ns %" PRIu64 "\n", period_ns); ODPH_DBG(" period_tick %" PRIu64 "\n", period_tick); - ODPH_DBG(" duration_ns %" PRIu64 "\n\n", duration_ns); + ODPH_DBG(" duration_ns %" PRIu64 "\n", duration_ns); + ODPH_DBG(" user_ptr %p\n\n", user_ctx); for (i = 0; i < num; i++) { if (event_type == ODP_EVENT_BUFFER) { @@ -605,7 +609,7 @@ static void timer_test_event_type(odp_queue_type_t queue_type, CU_ASSERT(ev != ODP_EVENT_INVALID); - timer[i] = odp_timer_alloc(timer_pool, queue, NULL); + timer[i] = odp_timer_alloc(timer_pool, queue, user_ctx); CU_ASSERT_FATAL(timer[i] != ODP_TIMER_INVALID); ret = odp_timer_set_rel(timer[i], (i + 1) * period_tick, &ev); @@ -620,6 +624,12 @@ static void timer_test_event_type(odp_queue_type_t queue_type, CU_ASSERT(ret == ODP_TIMER_SUCCESS); } + if (test_print) { + printf("\n"); + odp_timer_pool_print(timer_pool); + odp_timer_print(timer[0]); + } + ev = ODP_EVENT_INVALID; num_tmo = 0; t1 = odp_time_local(); @@ -642,6 +652,13 @@ static void timer_test_event_type(odp_queue_type_t queue_type, CU_ASSERT(odp_event_type(ev) == event_type); + if (test_print) { + test_print = 0; + tmo = odp_timeout_from_event(ev); + odp_timeout_print(tmo); + printf("\n"); + } + odp_event_free(ev); num_tmo++; |