diff options
| author | Matias Elo <matias.elo@nokia.com> | 2024-03-11 16:39:27 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-03-11 16:39:27 +0200 |
| commit | 8063101c4fac56e16c5a2bb9843f2fd9c5acbfd7 (patch) | |
| tree | 63e2500c41ea7d1e4714236561be641b1639f287 | |
| parent | c00ef7d6bce1e483c4cf1bb3cdf6cd629530d795 (diff) | |
| parent | 9ff786ed3d9d553f8e108eff4ee4ceec4adb585e (diff) | |
Merge ODP v1.44.0.0HEADv1.44.0.0_DPDK_22.11master
Merge ODP linux-generic v1.44.0.0 into linux-dpdk.
425 files changed, 13588 insertions, 2217 deletions
diff --git a/.checkpatch.conf b/.checkpatch.conf index 16235c39b..abb822996 100644 --- a/.checkpatch.conf +++ b/.checkpatch.conf @@ -19,4 +19,3 @@ --ignore=EMAIL_SUBJECT --ignore=C99_COMMENT_TOLERANCE --codespell ---codespellfile=/usr/share/codespell/dictionary.txt diff --git a/.github/workflows/ci-pipeline-arm64.yml b/.github/workflows/ci-pipeline-arm64.yml index 5b4f7e13c..391ebfc3b 100644 --- a/.github/workflows/ci-pipeline-arm64.yml +++ b/.github/workflows/ci-pipeline-arm64.yml @@ -28,7 +28,7 @@ jobs: '--enable-pcapng-support'] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC=gcc -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -51,7 +51,7 @@ jobs: '--enable-wfe-locks'] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC=clang -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -70,7 +70,7 @@ jobs: conf: ['', '--enable-lto'] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="gcc-${{matrix.cc_ver}}" -e CXX="g++-${{matrix.cc_ver}}" -e CONF="${CONF} ${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/build_static.sh - if: ${{ failure() }} @@ -86,7 +86,7 @@ jobs: os: ['rocky_linux_8'] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${{matrix.os}}-${ARCH}-native /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -104,7 +104,7 @@ jobs: conf: ['', '--enable-abi-compat'] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="gcc-${{matrix.cc_ver}}" -e CXX="g++-${{matrix.cc_ver}}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -115,7 +115,7 @@ jobs: runs-on: [self-hosted, ARM64] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/out_of_tree.sh - if: ${{ failure() }} @@ -130,7 +130,7 @@ jobs: conf: ['--enable-user-guides', '--enable-user-guides --enable-abi-compat'] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # Ignore distcheck failure (caused by the first 'make check' run unmounting huge pages) - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/distcheck.sh || true @@ -153,7 +153,7 @@ jobs: '--enable-wfe-locks'] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC=gcc -e ARCH="${ARCH}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -175,7 +175,7 @@ jobs: '--disable-host-optimization --enable-abi-compat'] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC=clang -e ARCH="${ARCH}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -191,7 +191,7 @@ jobs: cflags: ['-march=armv8.2-a -O2', '-march=armv8.2-a+crypto -O2'] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e ARCH="${ARCH}" -e CXX=g++-10 -e CFLAGS="${{matrix.cflags}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -206,7 +206,7 @@ jobs: os: ['ubuntu_22.04'] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${{matrix.os}}-${ARCH}-native /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -217,7 +217,7 @@ jobs: runs-on: [self-hosted, ARM64] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" -e ODP_CONFIG_FILE=/odp/platform/linux-dpdk/test/sched-basic.conf $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -228,7 +228,7 @@ jobs: runs-on: [self-hosted, ARM64] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" -e ODP_CONFIG_FILE=/odp/platform/linux-dpdk/test/stash-custom.conf $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -239,7 +239,7 @@ jobs: runs-on: [self-hosted, ARM64] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" -e ODP_SCHEDULER=sp $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -250,7 +250,7 @@ jobs: runs-on: [self-hosted, ARM64] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" -e ODP_CONFIG_FILE=/odp/platform/linux-dpdk/test/process-mode.conf -e ODPH_PROC_MODE=1 $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native /odp/scripts/ci/check.sh @@ -262,8 +262,19 @@ jobs: runs-on: [self-hosted, ARM64] steps: - uses: OpenDataPlane/action-clean-up@main - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native-dpdk_21.11 /odp/scripts/ci/check.sh - if: ${{ failure() }} uses: ./.github/actions/run-failure-log + + Run_dpdk-23_11: + if: ${{ github.repository == 'OpenDataPlane/odp-dpdk' }} + runs-on: [self-hosted, ARM64] + steps: + - uses: OpenDataPlane/action-clean-up@main + - uses: actions/checkout@v4 + - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" + -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-native-dpdk_23.11 /odp/scripts/ci/check.sh + - if: ${{ failure() }} + uses: ./.github/actions/run-failure-log diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index d0b6a0647..832d91ee1 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -11,7 +11,7 @@ jobs: Checkpatch: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Install dependencies @@ -22,13 +22,13 @@ jobs: if: github.event_name == 'pull_request' env: CHECKPATCH_COMMAND: ./scripts/checkpatch.pl - uses: webispy/checkpatch-action@v8 + uses: webispy/checkpatch-action@v9 - name: Check push if: github.event_name == 'push' && github.ref != 'refs/heads/master' run: | AFTER=${{ github.event.after }} BEFORE=${{ github.event.before }} - if [ -z "${BEFORE//0}" ] || [ -z "${AFTER//0}" ]; then + if [ -z "${BEFORE//0}" ] || [ -z "${AFTER//0}" ] || ${{ github.event.forced }}; then COMMIT_RANGE="" else COMMIT_RANGE="${BEFORE}..${AFTER}" @@ -38,7 +38,7 @@ jobs: Documentation: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install dependencies run: | sudo apt update @@ -72,7 +72,7 @@ jobs: '--enable-lto --enable-abi-compat', '--enable-pcapng-support'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC=gcc -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -92,7 +92,7 @@ jobs: '--enable-pcapng-support', '--without-openssl --without-pcap'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC=clang -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -109,7 +109,7 @@ jobs: cc_ver: [9, 10, 11, 12] conf: ['', '--enable-lto'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="gcc-${{matrix.cc_ver}}" -e CXX="g++-${{matrix.cc_ver}}" -e CONF="${CONF} ${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-static /odp/scripts/ci/build_static.sh - if: ${{ failure() }} @@ -127,7 +127,7 @@ jobs: conf: ['', '--enable-abi-compat', 'CFLAGS=-march=armv8.2-a', 'CFLAGS=-march=armv8-a+lse', '--enable-wfe-locks'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${CONF} ${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -143,7 +143,7 @@ jobs: matrix: conf: ['', '--enable-abi-compat'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e CONF="${CONF} ${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -160,7 +160,7 @@ jobs: cc: [gcc, clang] conf: ['', '--enable-abi-compat'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -175,7 +175,7 @@ jobs: os: ['centos_7', 'rocky_linux_8'] conf: ['--enable-abi-compat'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${{matrix.os}}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -191,7 +191,7 @@ jobs: cc_ver: [7, 8] conf: ['', '--enable-abi-compat'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="gcc-${{matrix.cc_ver}}" -e CXX="g++-${{matrix.cc_ver}}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -207,7 +207,7 @@ jobs: cc_ver: [10, 11, 12, 13] conf: ['', '--enable-abi-compat'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="gcc-${{matrix.cc_ver}}" -e CXX="g++-${{matrix.cc_ver}}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -216,7 +216,7 @@ jobs: Build_out-of-tree: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/out_of_tree.sh - if: ${{ failure() }} @@ -227,7 +227,7 @@ jobs: env: CONF: "--with-platform=linux-generic" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ODP_LIB_NAME="libodp-linux" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/build_${ARCH}.sh - if: ${{ failure() }} @@ -240,7 +240,7 @@ jobs: matrix: conf: ['--enable-user-guides', '--enable-user-guides --enable-abi-compat'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # Ignore distcheck failure (caused by the first 'make check' run unmounting huge pages) - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/distcheck.sh || true @@ -260,7 +260,7 @@ jobs: '--disable-host-optimization --enable-abi-compat', '--without-openssl --without-pcap'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC=gcc -e ARCH="${ARCH}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -280,7 +280,7 @@ jobs: '--disable-host-optimization --enable-event-validation=warn', '--disable-host-optimization --enable-abi-compat'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC=clang -e ARCH="${ARCH}" -e CONF="${{matrix.conf}}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -294,7 +294,7 @@ jobs: cc: [gcc, clang] os: ['ubuntu_22.04'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${{matrix.cc}}" -e ARCH="${ARCH}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${{matrix.os}}-${ARCH} /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -303,7 +303,7 @@ jobs: Run_sched_config: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" -e ODP_CONFIG_FILE=/odp/platform/linux-dpdk/test/sched-basic.conf $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -312,7 +312,7 @@ jobs: Run_stash_config: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" -e ODP_CONFIG_FILE=/odp/platform/linux-dpdk/test/stash-custom.conf $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -321,7 +321,7 @@ jobs: Run_scheduler_sp: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" -e ODP_SCHEDULER=sp $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/check.sh - if: ${{ failure() }} @@ -330,7 +330,7 @@ jobs: Run_process_mode: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" -e ODP_CONFIG_FILE=/odp/platform/linux-dpdk/test/process-mode.conf -e ODPH_PROC_MODE=1 $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/check.sh @@ -340,7 +340,7 @@ jobs: Run_default_timer: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e ODP_CONFIG_FILE=/odp/platform/linux-dpdk/test/default-timer.conf -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH} /odp/scripts/ci/check_validation.sh @@ -350,12 +350,21 @@ jobs: Run_dpdk-21_11: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-dpdk_21.11 /odp/scripts/ci/check.sh - if: ${{ failure() }} uses: ./.github/actions/run-failure-log + Run_dpdk-23_11: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" + -e CONF="${CONF}" $CONTAINER_NAMESPACE/odp-ci-${OS}-${ARCH}-dpdk_23.11 /odp/scripts/ci/check.sh + - if: ${{ failure() }} + uses: ./.github/actions/run-failure-log + Run_crypto: runs-on: ubuntu-20.04 strategy: @@ -363,7 +372,7 @@ jobs: matrix: driver: [crypto_aesni_mb, crypto_aesni_gcm] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e ARCH="${ARCH}" -e CONF="${CONF}" -e ODP_PLATFORM_PARAMS="--vdev=${{matrix.driver}}" -e ODP_CONFIG_FILE=/odp/platform/linux-dpdk/test/crypto.conf diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index db25671fd..44aebc805 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -15,7 +15,7 @@ jobs: Coverity-analysis: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: sudo docker run -i -v `pwd`:/odp --privileged --shm-size 8g -e CC="${CC}" -e GITHUB_SHA="${GITHUB_SHA}" -e COVERITY_TOKEN="${{ secrets.COVERITY_TOKEN }}" diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 325c3b30a..e04f2bb88 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -7,9 +7,9 @@ on: jobs: Documentation: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install dependencies run: | sudo apt update @@ -44,7 +44,7 @@ jobs: - name: Deploy env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: crazy-max/ghaction-github-pages@v3 + uses: crazy-max/ghaction-github-pages@v4 with: allow_empty_commit: false build_dir: ./doc/gh-pages @@ -1,3 +1,40 @@ +== OpenDataPlane (1.44.0.0) + +=== Backward compatible API changes +==== ML +* Add new API module for Machine Learning offload. + +=== Implementation +==== ML +* Implement the API using ONNX Runtime. A non-standard ONNX install path can be +passed with `--with-ort-path` configure option. See `DEPENDENCIES` for +additional information. + +=== Example Applications +==== debug +* Print packet IO link information. +* Add packet IO interface name option (`-i`). + +==== packet_dump +* Add support for measuring packet input delay. + +=== Performance Tests +==== l2fwd +* Add packet input timestamping option (`-T`). +* Add packet data prefetch option (`-F`). +* Add packet data read option (`-R`). +* Add packet print option (`-V`). + +==== pool_latency +* New pool latency tester application for profiling pool allocation and free +latencies under different load patterns. + +==== sched_perf +* Add options to read (`-u`) and modify (`-U`) event user area contents per +event reception. +* Run application indefinitely when the number of events to schedule option +(`-s`) is set to zero. + == OpenDataPlane (1.43.0.0) === Backward incompatible API changes diff --git a/DEPENDENCIES b/DEPENDENCIES index 7dbe86489..5af0d4ac3 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -305,6 +305,35 @@ Prerequisites for building the OpenDataPlane (ODP) API which would be the first actual queue in case 5 regular combined queues were configured (zero-indexing). +3.7 Machine Learning API support (optional) + Use ML API for model inferencing. ML implementation uses ONNX Runtime library + (https://github.com/microsoft/onnxruntime). ODP has been tested with ONNX + Runtime version 1.16.3. + +3.7.1 Prebuilt onnxruntime download + Download a default CPU version onnxruntime-linux-x64-*.tgz and unzip it to + any folder. + + $ wget -P ~ https://github.com/microsoft/onnxruntime/releases/download/v<version>/onnxruntime-linux-x64-<version>.tgz + $ mkdir <onnxruntime path> + $ cd <onnxruntime path>/ + $ tar --strip=1 -zxvf ~/onnxruntime-linux-x64-<version>.tgz + +3.7.1 Build onnxruntime from source + $ git clone --recursive https://github.com/Microsoft/onnxruntime.git + $ cd onnxruntime + + # Configure + $ ./build.sh --config RelWithDebInfo --build_shared_lib --parallel + $ tools/ci_build/github/linux/copy_strip_binary.sh -r build/Linux/ -a onnxruntime -l libonnxruntime.so.1.14.0 -c RelWithDebInfo -s . -t <commit id> + $ cp -r build/Linux/onnxruntime/ <onnxruntime path> + +3.7.2 Build ODP with ML support + After installing onnxruntime and example dependencies, ODP can be configured to be + built with ML support by giving onnxruntime path with --with-ort-path. + + $ ../configure --with-ort-path=<onnxruntime path> + 4.0 Packages needed to build API tests CUnit test framework version 2.1-3 is required diff --git a/Makefile.am b/Makefile.am index 663cacf6f..5fbe3681f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -6,11 +6,13 @@ if PLATFORM_IS_LINUX_GENERIC PLATFORM_DIR = platform/linux-generic PLATFORM_DUMPCONF_DIR = platform/linux-generic/dumpconfig PLATFORM_TEST_DIR = platform/linux-generic/test +PLATFORM_EXAMPLE_DIR = platform/linux-generic/example endif if PLATFORM_IS_LINUX_DPDK PLATFORM_DIR = platform/linux-dpdk PLATFORM_DUMPCONF_DIR = platform/linux-dpdk/dumpconfig PLATFORM_TEST_DIR = platform/linux-dpdk/test +PLATFORM_EXAMPLE_DIR = platform/linux-dpdk/example endif SUBDIRS = \ @@ -30,6 +32,7 @@ endif if WITH_EXAMPLES SUBDIRS += example +SUBDIRS += $(PLATFORM_EXAMPLE_DIR) endif if WITH_TESTS diff --git a/config/odp-linux-dpdk.conf b/config/odp-linux-dpdk.conf index 7e1192fca..e82626bcd 100644 --- a/config/odp-linux-dpdk.conf +++ b/config/odp-linux-dpdk.conf @@ -16,7 +16,7 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.25" +config_file_version = "0.1.26" # System options system: { @@ -335,3 +335,34 @@ dma: { # Maximum number of inflight transfers per session max_inflight = 16 } + +ml: { + # Enable onnxruntime profiling, when enabled, a json file will be + # generated after inference. chrome://tracing/ can be used to check + # the profiling. Use 0 to disable and 1 to enable profiling. + enable_profiling = 0 + + # Choose onnxruntime execution mode, which can be "SEQUENTIAL" or + # "PARALLEL" + execution_mode = "SEQUENTIAL" + + # Set the number of threads used to parallelize the execution of the + # graph across nodes. A value of 0 means onnxruntime will pick a default. + inter_op_num_threads = 0 + + # Set the number of threads used to parallelize the execution within + # a node. A value of 0 means onnxruntime will pick a default. + intra_op_num_threads = 0 + + # Set graph optimization level. Valid values are: + # DISABLE_ALL: disables all optimizations + # ENABLE_BASIC: enables basic optimizations + # ENABLE_EXTENDED: enables basic and extended optimizations + # ENABLE_ALL: enables all available optimizations including layout optimization + graph_optimization_level = "ENABLE_ALL" + + # Serialize the optimized model to disk. When initializing a session + # with the same model, no need to apply optimization anymore, thus + # reducing model startup time. + optimized_model_filepath = "" +} diff --git a/config/odp-linux-generic.conf b/config/odp-linux-generic.conf index 63ac31fe2..93997ecb3 100644 --- a/config/odp-linux-generic.conf +++ b/config/odp-linux-generic.conf @@ -16,7 +16,7 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.27" +config_file_version = "0.1.28" # System options system: { @@ -240,8 +240,9 @@ sched_basic: { # When waiting for events during a schedule call, save power by # sleeping in the poll loop. First, run schedule loop normally for # poll_time_nsec nanoseconds. If there are no events to schedule in that - # time, continue polling, but sleep for sleep_time_nsec nanoseconds on - # each round. + # time, continue polling, but sleep on each round. Sleep time is + # sleep_time_nsec nanoseconds, or the time to the next timer expiration, + # whichever is smaller. Timer pools are scanned just before sleep. # # During sleep, the thread is not polling for packet input or timers. # Each thread measures time and sleeps independently of other threads. @@ -257,7 +258,7 @@ sched_basic: { # Time in nsec to sleep # - # Actual sleep time may vary. + # Must be less than one second. Actual sleep time may vary. sleep_time_nsec = 0 } } @@ -364,3 +365,34 @@ ipsec: { async_outbound = 0 } } + +ml: { + # Enable onnxruntime profiling, when enabled, a json file will be + # generated after inference. chrome://tracing/ can be used to check + # the profiling. Use 0 to disable and 1 to enable profiling. + enable_profiling = 0 + + # Choose onnxruntime execution mode, which can be "SEQUENTIAL" or + # "PARALLEL" + execution_mode = "SEQUENTIAL" + + # Set the number of threads used to parallelize the execution of the + # graph across nodes. A value of 0 means onnxruntime will pick a default. + inter_op_num_threads = 0 + + # Set the number of threads used to parallelize the execution within + # a node. A value of 0 means onnxruntime will pick a default. + intra_op_num_threads = 0 + + # Set graph optimization level. Valid values are: + # DISABLE_ALL: disables all optimizations + # ENABLE_BASIC: enables basic optimizations + # ENABLE_EXTENDED: enables basic and extended optimizations + # ENABLE_ALL: enables all available optimizations including layout optimization + graph_optimization_level = "ENABLE_ALL" + + # Serialize the optimized model to disk. When initializing a session + # with the same model, no need to apply optimization anymore, thus + # reducing model startup time. + optimized_model_filepath = "" +} diff --git a/configure.ac b/configure.ac index fc1f17403..d71c5a4ae 100644 --- a/configure.ac +++ b/configure.ac @@ -3,7 +3,7 @@ AC_PREREQ([2.5]) # ODP API version ########################################################################## m4_define([odp_version_generation], [1]) -m4_define([odp_version_major], [43]) +m4_define([odp_version_major], [44]) m4_define([odp_version_minor], [0]) m4_define([odp_version_patch], [0]) @@ -113,6 +113,7 @@ ODP_CHECK_CFLAG([-Wundef]) ODP_CHECK_CFLAG([-Wwrite-strings]) ODP_CHECK_CFLAG([-Wformat-truncation=0]) ODP_CHECK_CFLAG([-Wformat-overflow=0]) +ODP_CHECK_CFLAG([-Wshadow=local]) # GCC 10 sometimes gets confused about object sizes and gives bogus warnings. # Make the affected warnings generate only warnings, not errors. diff --git a/doc/Doxyfile_common b/doc/Doxyfile_common index 2aa54ac48..6c169b4e7 100644 --- a/doc/Doxyfile_common +++ b/doc/Doxyfile_common @@ -30,6 +30,7 @@ MACRO_EXPANSION = YES EXPAND_ONLY_PREDEF = YES INTERNAL_DOCS = YES DOT_IMAGE_FORMAT = svg +DOT_GRAPH_MAX_NODES = 60 PREDEFINED = __GNUC__ \ __attribute__(x)= \ ODP_ALIGNED(x)= \ diff --git a/doc/application-api-guide/Doxyfile b/doc/application-api-guide/Doxyfile index 7f8c70c8e..9c718f203 100644 --- a/doc/application-api-guide/Doxyfile +++ b/doc/application-api-guide/Doxyfile @@ -5,6 +5,9 @@ PROJECT_NUMBER = $(ODP_VERSION_API) PROJECT_LOGO = $(SRCDIR)/doc/images/ODP-Logo-HQ.svg INPUT = $(SRCDIR)/doc/application-api-guide \ include \ - $(SRCDIR)/include -EXAMPLE_PATH = $(SRCDIR)/example $(SRCDIR)/CONTRIBUTING $(SRCDIR)/include/README + $(SRCDIR)/include \ + $(SRCDIR)/example \ + $(SRCDIR)/test/performance +EXAMPLE_PATH = $(SRCDIR)/example $(SRCDIR)/CONTRIBUTING $(SRCDIR)/include/README \ + $(SRCDIR)/test/performance WARNINGS = NO diff --git a/doc/application-api-guide/Makefile.am b/doc/application-api-guide/Makefile.am index 7d9003976..29599597d 100644 --- a/doc/application-api-guide/Makefile.am +++ b/doc/application-api-guide/Makefile.am @@ -1,7 +1,6 @@ EXTRA_DIST = \ Doxyfile \ api_guide_lines.dox \ - examples.dox \ odp.dox \ release.dox diff --git a/doc/application-api-guide/examples.dox b/doc/application-api-guide/examples.dox deleted file mode 100644 index 18817cd63..000000000 --- a/doc/application-api-guide/examples.dox +++ /dev/null @@ -1,100 +0,0 @@ -/* Copyright (c) 2015-2019, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -/** - * @example odp_classifier.c - * Classifier example application - */ - -/** - * @example odp_cli.c - * CLI example application - */ - -/** - * @example odp_debug.c - * Debug example application - */ - -/** - * @example odp_generator.c - * Traffic generator and loopback demo application - */ - -/** - * @example odp_hello.c - * Minimal example application - */ - -/** - * @example odp_ipfragreass.c - * IPv4 lock-free fragmentation and reassembly example application - */ - -/** - * @example ipsec_api/odp_ipsec.c - * IPsec example application using IPsec API - */ - - /** - * @example ipsec_crypto/odp_ipsec.c - * IPsec example application using crypto API - */ - -/** - * @example odp_l2fwd.c - * L2 forwarding example application - */ - -/** - * @example odp_l2fwd_simple.c - * Minimal L2 forwarding example application - */ - -/** - * @example odp_l3fwd.c - * L3 forwarding example application - */ - -/** - *@example odp_ping.c - * Example application which replies to ICMPv4 echo requests (ping) - */ - -/** - *@example odp_pktio.c - * Basic packet IO loopback test application - */ - - /** - *@example odp_simple_pipeline.c - * Simple pipeline example application - */ - -/** - * @example odp_switch.c - * Minimal learning Ethernet switch example application - */ - - /** - * @example odp_sysinfo.c - * System information example application - */ - -/** - * @example time_global_test.c - * Time example application - */ - -/** - * @example odp_timer_test.c - * Timer example application - */ - - /** - * @example odp_traffic_mgmt.c - * Traffic manager example application - */ diff --git a/doc/implementers-guide/implementers-guide.adoc b/doc/implementers-guide/implementers-guide.adoc index 7b234ecea..c20f04906 100644 --- a/doc/implementers-guide/implementers-guide.adoc +++ b/doc/implementers-guide/implementers-guide.adoc @@ -208,7 +208,7 @@ Within the platform agnostic area, the validation tests for a given interface are also grouped by modules, matching the ODP interface modules: for instance, `test/common_plat/validation/api` mainly contains a list of directories matching each module name (as defined by the doxygen `@defgroup` -or `@ingroup` statement present in each API `.h` file). +or `@addtogroup` statement present in each API `.h` file). Within each of these directories, a library (called `libtest<module>.la`) and its associated `.h` file (called `<module>.h`) defines all the test functions diff --git a/example/classifier/odp_classifier.c b/example/classifier/odp_classifier.c index 4f71c835b..f4218fd9c 100644 --- a/example/classifier/odp_classifier.c +++ b/example/classifier/odp_classifier.c @@ -1,9 +1,15 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia - * Copyright (c) 2020, Marvell - * All rights reserved. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2018 Linaro Limited + * Copyright (c) 2019-2022 Nokia + * Copyright (c) 2020 Marvell + */ + +/** + * @example odp_classifier.c + * + * Classifier API example application * - * SPDX-License-Identifier: BSD-3-Clause + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include <stdlib.h> diff --git a/example/classifier/odp_classifier_run.sh b/example/classifier/odp_classifier_run.sh index 2b5fe7ebb..ad0c3a76a 100755 --- a/example/classifier/odp_classifier_run.sh +++ b/example/classifier/odp_classifier_run.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Copyright (c) 2020, Marvell -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2020 Marvell # if [ -f ./pktio_env ]; then diff --git a/example/cli/odp_cli.c b/example/cli/odp_cli.c index 5f87b2a78..381fc8a59 100644 --- a/example/cli/odp_cli.c +++ b/example/cli/odp_cli.c @@ -1,15 +1,15 @@ -/* Copyright (c) 2021, Nokia - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021 Nokia */ -/* - * ODP CLI Helper Example +/** + * @example odp_cli.c * * This example shows how to start and stop ODP CLI using the CLI helper * API functions. This example application can also be used to try out * the CLI by connecting to a running application with a telnet client. + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include <odp_api.h> diff --git a/example/cli/odp_cli_run.sh b/example/cli/odp_cli_run.sh index 0dc00b793..bb212fffb 100755 --- a/example/cli/odp_cli_run.sh +++ b/example/cli/odp_cli_run.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Copyright (c) 2021, Nokia -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2021 Nokia # ./odp_cli${EXEEXT} -t 2 diff --git a/example/debug/odp_debug.c b/example/debug/odp_debug.c index a2f322e09..79107ca8f 100644 --- a/example/debug/odp_debug.c +++ b/example/debug/odp_debug.c @@ -1,7 +1,13 @@ -/* Copyright (c) 2020-2022, Nokia - * All rights reserved. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2020-2022 Nokia + */ + +/** + * @example odp_debug.c * - * SPDX-License-Identifier: BSD-3-Clause + * This example application demonstrates the usage of various debug print functions of ODP API. + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include <stdlib.h> @@ -14,6 +20,8 @@ #include <odp_api.h> #include <odp/helper/odph_api.h> +#define MAX_NAME_LEN 128 + typedef struct test_global_t { int system; int shm; @@ -23,6 +31,7 @@ typedef struct test_global_t { int ipsec; int timer; int stash; + char pktio_name[MAX_NAME_LEN]; } test_global_t; @@ -35,35 +44,37 @@ static void print_usage(void) "are called when no options are given.\n" "\n" "OPTIONS:\n" - " -S, --system Call odp_sys_info_print() and odp_sys_config_print()\n" - " -s, --shm Create a SHM and call odp_shm_print()\n" - " -p, --pool Create various types of pools and call odp_pool_print()\n" - " -q, --queue Create various types of queues and call odp_queue_print()\n" - " -i, --interface Create packet IO interface (loop), and call both odp_pktio_print()\n" - " and odp_pktio_extra_stats_print()\n" - " -I, --ipsec Call odp_ipsec_print()\n" - " -t, --timer Call timer pool, timer and timeout print functions\n" - " -a, --stash Create stash and call odp_stash_print()\n" - " -h, --help Display help and exit.\n\n"); + " -S, --system Call odp_sys_info_print() and odp_sys_config_print()\n" + " -s, --shm Create a SHM and call odp_shm_print()\n" + " -p, --pool Create various types of pools and call odp_pool_print()\n" + " -q, --queue Create various types of queues and call odp_queue_print()\n" + " -i, --interface <name> Start a packet IO interface, and call odp_pktio_print(),\n" + " odp_pktio_extra_stats_print(), etc. Uses loop interface by default.\n" + " -I, --ipsec Call odp_ipsec_print()\n" + " -t, --timer Call timer pool, timer and timeout print functions\n" + " -a, --stash Create stash and call odp_stash_print()\n" + " -h, --help Display help and exit.\n\n"); } static int parse_options(int argc, char *argv[], test_global_t *global) { int opt, long_index; + char *str; + uint32_t str_len = 0; const struct option longopts[] = { {"system", no_argument, NULL, 'S'}, {"shm", no_argument, NULL, 's'}, {"pool", no_argument, NULL, 'p'}, {"queue", no_argument, NULL, 'q'}, - {"interface", no_argument, NULL, 'i'}, + {"interface", required_argument, NULL, 'i'}, {"ipsec", no_argument, NULL, 'I'}, {"timer", no_argument, NULL, 't'}, {"stash", no_argument, NULL, 'a'}, {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} }; - const char *shortopts = "+SspqiItah"; + const char *shortopts = "+Sspqi:Itah"; int ret = 0; while (1) { @@ -87,6 +98,11 @@ static int parse_options(int argc, char *argv[], test_global_t *global) break; case 'i': global->pktio = 1; + str = optarg; + str_len = strlen(str); + + if (str_len && str_len < MAX_NAME_LEN) + strcpy(global->pktio_name, str); break; case 'I': global->ipsec = 1; @@ -104,6 +120,11 @@ static int parse_options(int argc, char *argv[], test_global_t *global) } } + if (global->pktio && (str_len == 0 || str_len >= MAX_NAME_LEN)) { + ODPH_ERR("Bad interface name length: %u\n", str_len); + ret = -1; + } + return ret; } @@ -314,11 +335,13 @@ static int queue_debug(void) return 0; } -static int pktio_debug(void) +static int pktio_debug(test_global_t *global) { odp_pool_t pool; odp_pool_param_t pool_param; odp_pktio_t pktio; + odp_pktio_link_info_t info; + uint8_t mac[ODPH_ETHADDR_LEN]; int pkt_len = 100; odp_pool_param_init(&pool_param); @@ -333,19 +356,79 @@ static int pktio_debug(void) return -1; } - pktio = odp_pktio_open("loop", pool, NULL); + pktio = odp_pktio_open(global->pktio_name, pool, NULL); if (pktio == ODP_PKTIO_INVALID) { ODPH_ERR("Pktio open failed\n"); return -1; } + /* Start interface with default config */ + if (odp_pktin_queue_config(pktio, NULL)) { + ODPH_ERR("Packet input queue config failed\n"); + return -1; + } + + if (odp_pktout_queue_config(pktio, NULL)) { + ODPH_ERR("Packet output queue config failed\n"); + return -1; + } + + if (odp_pktio_start(pktio)) { + ODPH_ERR("Pktio start failed\n"); + return -1; + } + + printf("\nWaiting link up"); + + /* Wait max 5 seconds for link up */ + for (int i = 0; i < 25; i++) { + if (odp_pktio_link_status(pktio) == ODP_PKTIO_LINK_STATUS_UP) + break; + + odp_time_wait_ns(200 * ODP_TIME_MSEC_IN_NS); + printf("."); + fflush(NULL); + } + + printf("\n\n"); + + printf("Packet IO\n---------\n"); + printf(" index: %i\n", odp_pktio_index(pktio)); + printf(" handle: 0x%" PRIx64 "\n", odp_pktio_to_u64(pktio)); + + if (odp_pktio_mac_addr(pktio, mac, ODPH_ETHADDR_LEN) == ODPH_ETHADDR_LEN) { + printf(" mac address: %02x:%02x:%02x:%02x:%02x:%02x\n", mac[0], mac[1], mac[2], + mac[3], mac[4], mac[5]); + } + + printf(" input maxlen: %u\n", odp_pktin_maxlen(pktio)); + printf(" output maxlen: %u\n", odp_pktout_maxlen(pktio)); + printf(" promisc mode: %i\n", odp_pktio_promisc_mode(pktio)); + printf(" timestamp res: %" PRIu64 " Hz\n\n", odp_pktio_ts_res(pktio)); + + if (odp_pktio_link_info(pktio, &info) == 0) { + printf("Link info\n---------\n"); + printf(" auto neg: %i\n", info.autoneg); + printf(" duplex: %i\n", info.duplex); + printf(" media: %s\n", info.media); + printf(" pause_rx: %i\n", info.pause_rx); + printf(" pause_tx: %i\n", info.pause_tx); + printf(" speed: %u Mbit/s\n", info.speed); + printf(" status: %i\n", info.status); + } + printf("\n"); odp_pktio_print(pktio); printf("\n"); odp_pktio_extra_stats_print(pktio); + if (odp_pktio_stop(pktio)) { + ODPH_ERR("Pktio stop failed\n"); + return -1; + } + if (odp_pktio_close(pktio)) { ODPH_ERR("Pktio close failed\n"); return -1; @@ -562,6 +645,7 @@ int main(int argc, char *argv[]) global->ipsec = 1; global->timer = 1; global->stash = 1; + strcpy(global->pktio_name, "loop"); } else { if (parse_options(argc, argv, global)) exit(EXIT_FAILURE); @@ -606,7 +690,7 @@ int main(int argc, char *argv[]) exit(EXIT_FAILURE); } - if (global->pktio && pktio_debug()) { + if (global->pktio && pktio_debug(global)) { ODPH_ERR("Packet debug failed.\n"); exit(EXIT_FAILURE); } diff --git a/example/generator/generator_run.sh b/example/generator/generator_run.sh index 528c1b595..ca69b53bd 100755 --- a/example/generator/generator_run.sh +++ b/example/generator/generator_run.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Copyright (c) 2020, Marvell -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2020 Marvell # if [ -f ./pktio_env ]; then diff --git a/example/generator/odp_generator.c b/example/generator/odp_generator.c index 95ba04c1e..6ec19d0ea 100644 --- a/example/generator/odp_generator.c +++ b/example/generator/odp_generator.c @@ -1,7 +1,13 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited + */ + +/** + * @example odp_generator.c + * + * Traffic generator and loopback demo application * - * SPDX-License-Identifier: BSD-3-Clause + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ /** enable strtok */ diff --git a/example/hello/odp_hello.c b/example/hello/odp_hello.c index 391406946..58cc35502 100644 --- a/example/hello/odp_hello.c +++ b/example/hello/odp_hello.c @@ -1,13 +1,16 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited */ -/* This is a minimal application which demonstrates the startup and shutdown +/** + * @example odp_hello.c + * + * This is a minimal application which demonstrates the startup and shutdown * steps of an ODP application. It can be also used to debug API related * build problems, etc. It does not use helpers to minimize dependency to * anything else than the ODP API header file. + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include <stdio.h> diff --git a/example/ipfragreass/odp_ipfragreass.c b/example/ipfragreass/odp_ipfragreass.c index 55c76d7cc..2b7df861e 100644 --- a/example/ipfragreass/odp_ipfragreass.c +++ b/example/ipfragreass/odp_ipfragreass.c @@ -1,13 +1,13 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017-2018 Linaro Limited */ /** - * @file + * @example odp_ipfragreass.c + * + * IPv4 lock-free fragmentation and reassembly example application * - * @example odp_ipfragreass.c ODP IPv4 lock-free fragmentation and reassembly + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include <stdio.h> diff --git a/example/ipfragreass/odp_ipfragreass_fragment.c b/example/ipfragreass/odp_ipfragreass_fragment.c index 2cce75e91..2e9d9ad64 100644 --- a/example/ipfragreass/odp_ipfragreass_fragment.c +++ b/example/ipfragreass/odp_ipfragreass_fragment.c @@ -1,9 +1,9 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #include <stdio.h> #include <assert.h> diff --git a/example/ipfragreass/odp_ipfragreass_fragment.h b/example/ipfragreass/odp_ipfragreass_fragment.h index e0e8cfea7..748f84186 100644 --- a/example/ipfragreass/odp_ipfragreass_fragment.h +++ b/example/ipfragreass/odp_ipfragreass_fragment.h @@ -1,9 +1,9 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_FRAGREASS_PP_FRAG_H_ #define ODP_FRAGREASS_PP_FRAG_H_ diff --git a/example/ipfragreass/odp_ipfragreass_helpers.c b/example/ipfragreass/odp_ipfragreass_helpers.c index ac737fe44..a48157fcf 100644 --- a/example/ipfragreass/odp_ipfragreass_helpers.c +++ b/example/ipfragreass/odp_ipfragreass_helpers.c @@ -1,9 +1,9 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #include <stdio.h> #include <stdlib.h> diff --git a/example/ipfragreass/odp_ipfragreass_helpers.h b/example/ipfragreass/odp_ipfragreass_helpers.h index af47e326c..9f89ca4c4 100644 --- a/example/ipfragreass/odp_ipfragreass_helpers.h +++ b/example/ipfragreass/odp_ipfragreass_helpers.h @@ -1,9 +1,9 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_FRAGREASS_PP_HELPERS_H_ #define ODP_FRAGREASS_PP_HELPERS_H_ diff --git a/example/ipfragreass/odp_ipfragreass_ip.h b/example/ipfragreass/odp_ipfragreass_ip.h index 9c8f2a5f7..97e68324c 100644 --- a/example/ipfragreass/odp_ipfragreass_ip.h +++ b/example/ipfragreass/odp_ipfragreass_ip.h @@ -1,9 +1,9 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_FRAGREASS_PP_IP_H_ #define ODP_FRAGREASS_PP_IP_H_ diff --git a/example/ipfragreass/odp_ipfragreass_reassemble.c b/example/ipfragreass/odp_ipfragreass_reassemble.c index 2542a3b97..55ee42504 100644 --- a/example/ipfragreass/odp_ipfragreass_reassemble.c +++ b/example/ipfragreass/odp_ipfragreass_reassemble.c @@ -1,9 +1,9 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #include <odp_api.h> #include <stdio.h> diff --git a/example/ipfragreass/odp_ipfragreass_reassemble.h b/example/ipfragreass/odp_ipfragreass_reassemble.h index 1fb71284e..b00f3825e 100644 --- a/example/ipfragreass/odp_ipfragreass_reassemble.h +++ b/example/ipfragreass/odp_ipfragreass_reassemble.h @@ -1,9 +1,9 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_FRAGREASS_PP_REASSEMBLE_H_ #define ODP_FRAGREASS_PP_REASSEMBLE_H_ diff --git a/example/ipsec_api/README b/example/ipsec_api/README index f7ea93e59..7a392a728 100644 --- a/example/ipsec_api/README +++ b/example/ipsec_api/README @@ -1,8 +1,6 @@ -Copyright (c) 2014-2018, Linaro Limited -Copyright (c) 2020, Nokia -All rights reserved. - -SPDX-License-Identifier: BSD-3-Clause +SPDX-License-Identifier: BSD-3-Clause +Copyright (c) 2014-2018 Linaro Limited +Copyright (c) 2020 Nokia 1. Intro diff --git a/example/ipsec_api/odp_ipsec.c b/example/ipsec_api/odp_ipsec.c index 524b1ae76..a67a3a1e8 100644 --- a/example/ipsec_api/odp_ipsec.c +++ b/example/ipsec_api/odp_ipsec.c @@ -1,13 +1,13 @@ -/* Copyright (c) 2013-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2013-2018 Linaro Limited */ /** - * @file + * @example ipsec_api/odp_ipsec.c + * + * IPsec example application using ODP IPsec API * - * @example odp_example_ipsec.c ODP basic packet IO cross connect with IPsec test application + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ /* enable strtok */ diff --git a/example/ipsec_api/odp_ipsec_cache.c b/example/ipsec_api/odp_ipsec_cache.c index c16c71115..827c9dce2 100644 --- a/example/ipsec_api/odp_ipsec_cache.c +++ b/example/ipsec_api/odp_ipsec_cache.c @@ -1,9 +1,9 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #include <stdlib.h> #include <string.h> diff --git a/example/ipsec_api/odp_ipsec_cache.h b/example/ipsec_api/odp_ipsec_cache.h index b0e1fa3f0..5dd0c80b3 100644 --- a/example/ipsec_api/odp_ipsec_cache.h +++ b/example/ipsec_api/odp_ipsec_cache.h @@ -1,9 +1,9 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_IPSEC_CACHE_H_ #define ODP_IPSEC_CACHE_H_ diff --git a/example/ipsec_crypto/README b/example/ipsec_crypto/README index 8e66372b5..0411396a1 100644 --- a/example/ipsec_crypto/README +++ b/example/ipsec_crypto/README @@ -1,7 +1,5 @@ -Copyright (c) 2014-2018, Linaro Limited -All rights reserved. - -SPDX-License-Identifier: BSD-3-Clause +SPDX-License-Identifier: BSD-3-Clause +Copyright (c) 2014-2018 Linaro Limited 1. Intro diff --git a/example/ipsec_crypto/odp_ipsec.c b/example/ipsec_crypto/odp_ipsec.c index 06d52c311..92e32d301 100644 --- a/example/ipsec_crypto/odp_ipsec.c +++ b/example/ipsec_crypto/odp_ipsec.c @@ -1,14 +1,14 @@ -/* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2021-2022, Nokia - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2013-2018 Linaro Limited + * Copyright (c) 2021-2022 Nokia */ /** - * @file + * @example ipsec_crypto/odp_ipsec.c + * + * IPsec example application using ODP crypto API * - * @example odp_example_ipsec.c ODP basic packet IO cross connect with IPsec test application + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ /* enable strtok */ @@ -975,10 +975,10 @@ pkt_disposition_e do_ipsec_out_seq(odp_packet_t *pkt, esp->seq_no = odp_cpu_to_be_32((*ctx->ipsec.esp_seq)++); } if (ctx->ipsec.tun_hdr_offset) { - odph_ipv4hdr_t *ip; + odph_ipv4hdr_t *ip_tun; - ip = (odph_ipv4hdr_t *)(ctx->ipsec.tun_hdr_offset + buf); - ip->id = odp_cpu_to_be_16((*ctx->ipsec.tun_hdr_id)++); + ip_tun = (odph_ipv4hdr_t *)(ctx->ipsec.tun_hdr_offset + buf); + ip_tun->id = odp_cpu_to_be_16((*ctx->ipsec.tun_hdr_id)++); } /* Issue crypto request */ diff --git a/example/ipsec_crypto/odp_ipsec_cache.c b/example/ipsec_crypto/odp_ipsec_cache.c index 65a51bd1d..0461d4fbf 100644 --- a/example/ipsec_crypto/odp_ipsec_cache.c +++ b/example/ipsec_crypto/odp_ipsec_cache.c @@ -1,9 +1,9 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #include <stdlib.h> #include <string.h> diff --git a/example/ipsec_crypto/odp_ipsec_cache.h b/example/ipsec_crypto/odp_ipsec_cache.h index b1e3e7ac1..29c1b983a 100644 --- a/example/ipsec_crypto/odp_ipsec_cache.h +++ b/example/ipsec_crypto/odp_ipsec_cache.h @@ -1,9 +1,9 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_IPSEC_CACHE_H_ #define ODP_IPSEC_CACHE_H_ diff --git a/example/ipsec_crypto/odp_ipsec_fwd_db.c b/example/ipsec_crypto/odp_ipsec_fwd_db.c index 9bd399ca9..292d9c7c6 100644 --- a/example/ipsec_crypto/odp_ipsec_fwd_db.c +++ b/example/ipsec_crypto/odp_ipsec_fwd_db.c @@ -1,9 +1,9 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + /* enable strtok */ #ifndef _GNU_SOURCE #define _GNU_SOURCE diff --git a/example/ipsec_crypto/odp_ipsec_fwd_db.h b/example/ipsec_crypto/odp_ipsec_fwd_db.h index c6d9c6c99..d9c84b29b 100644 --- a/example/ipsec_crypto/odp_ipsec_fwd_db.h +++ b/example/ipsec_crypto/odp_ipsec_fwd_db.h @@ -1,9 +1,9 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_IPSEC_FWD_DB_H_ #define ODP_IPSEC_FWD_DB_H_ diff --git a/example/ipsec_crypto/odp_ipsec_misc.h b/example/ipsec_crypto/odp_ipsec_misc.h index 69788d115..921c4e3c0 100644 --- a/example/ipsec_crypto/odp_ipsec_misc.h +++ b/example/ipsec_crypto/odp_ipsec_misc.h @@ -1,9 +1,9 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_IPSEC_MISC_H_ #define ODP_IPSEC_MISC_H_ diff --git a/example/ipsec_crypto/odp_ipsec_sa_db.c b/example/ipsec_crypto/odp_ipsec_sa_db.c index 9d3418681..9ddbc1152 100644 --- a/example/ipsec_crypto/odp_ipsec_sa_db.c +++ b/example/ipsec_crypto/odp_ipsec_sa_db.c @@ -1,9 +1,9 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + /* enable strtok */ #ifndef _GNU_SOURCE #define _GNU_SOURCE diff --git a/example/ipsec_crypto/odp_ipsec_sa_db.h b/example/ipsec_crypto/odp_ipsec_sa_db.h index 72231369f..83f103af3 100644 --- a/example/ipsec_crypto/odp_ipsec_sa_db.h +++ b/example/ipsec_crypto/odp_ipsec_sa_db.h @@ -1,9 +1,9 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_IPSEC_SA_DB_H_ #define ODP_IPSEC_SA_DB_H_ diff --git a/example/ipsec_crypto/odp_ipsec_sp_db.c b/example/ipsec_crypto/odp_ipsec_sp_db.c index 3ead3b0a3..956fb212d 100644 --- a/example/ipsec_crypto/odp_ipsec_sp_db.c +++ b/example/ipsec_crypto/odp_ipsec_sp_db.c @@ -1,9 +1,9 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + /* enable strtok */ #ifndef _GNU_SOURCE #define _GNU_SOURCE diff --git a/example/ipsec_crypto/odp_ipsec_sp_db.h b/example/ipsec_crypto/odp_ipsec_sp_db.h index b71ea9377..ac6c0c896 100644 --- a/example/ipsec_crypto/odp_ipsec_sp_db.h +++ b/example/ipsec_crypto/odp_ipsec_sp_db.h @@ -1,9 +1,9 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_IPSEC_SP_DB_H_ #define ODP_IPSEC_SP_DB_H_ diff --git a/example/ipsec_crypto/odp_ipsec_stream.c b/example/ipsec_crypto/odp_ipsec_stream.c index f8bc64eae..330b4cc54 100644 --- a/example/ipsec_crypto/odp_ipsec_stream.c +++ b/example/ipsec_crypto/odp_ipsec_stream.c @@ -1,9 +1,9 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + /* enable strtok */ #ifndef _GNU_SOURCE #define _GNU_SOURCE diff --git a/example/ipsec_crypto/odp_ipsec_stream.h b/example/ipsec_crypto/odp_ipsec_stream.h index 2055d3f00..8e1e936dd 100644 --- a/example/ipsec_crypto/odp_ipsec_stream.h +++ b/example/ipsec_crypto/odp_ipsec_stream.h @@ -1,9 +1,9 @@ -/* Copyright (c) 2014-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2014-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef ODP_IPSEC_STREAM_H_ #define ODP_IPSEC_STREAM_H_ diff --git a/example/l2fwd_simple/l2fwd_simple_run.sh b/example/l2fwd_simple/l2fwd_simple_run.sh index b44e10ae1..741240e27 100755 --- a/example/l2fwd_simple/l2fwd_simple_run.sh +++ b/example/l2fwd_simple/l2fwd_simple_run.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Copyright (c) 2016-2018, Linaro Limited -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2016-2018 Linaro Limited # if [ -f ./pktio_env ]; then diff --git a/example/l2fwd_simple/odp_l2fwd_simple.c b/example/l2fwd_simple/odp_l2fwd_simple.c index c9ae94c12..5d9aaec87 100644 --- a/example/l2fwd_simple/odp_l2fwd_simple.c +++ b/example/l2fwd_simple/odp_l2fwd_simple.c @@ -1,7 +1,13 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited + */ + +/** + * @example odp_l2fwd_simple.c + * + * Minimal L2 forwarding example application * - * SPDX-License-Identifier: BSD-3-Clause + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include <stdlib.h> diff --git a/example/l3fwd/odp_l3fwd.c b/example/l3fwd/odp_l3fwd.c index 7d6b15af0..c74c7d4d6 100644 --- a/example/l3fwd/odp_l3fwd.c +++ b/example/l3fwd/odp_l3fwd.c @@ -1,7 +1,13 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited + */ + +/** + * @example odp_l3fwd.c + * + * L3 forwarding example application * - * SPDX-License-Identifier: BSD-3-Clause + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include <stdlib.h> diff --git a/example/l3fwd/odp_l3fwd_db.c b/example/l3fwd/odp_l3fwd_db.c index a4e4681f6..7d82695be 100644 --- a/example/l3fwd/odp_l3fwd_db.c +++ b/example/l3fwd/odp_l3fwd_db.c @@ -1,9 +1,9 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif diff --git a/example/l3fwd/odp_l3fwd_db.h b/example/l3fwd/odp_l3fwd_db.h index b26c67e44..5c9d63e6e 100644 --- a/example/l3fwd/odp_l3fwd_db.h +++ b/example/l3fwd/odp_l3fwd_db.h @@ -1,9 +1,9 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef _ODP_L3FWD_DB_H_ #define _ODP_L3FWD_DB_H_ diff --git a/example/l3fwd/odp_l3fwd_lpm.c b/example/l3fwd/odp_l3fwd_lpm.c index 7eb32b5d0..41572d179 100644 --- a/example/l3fwd/odp_l3fwd_lpm.c +++ b/example/l3fwd/odp_l3fwd_lpm.c @@ -1,8 +1,9 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited */ + +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif diff --git a/example/l3fwd/odp_l3fwd_lpm.h b/example/l3fwd/odp_l3fwd_lpm.h index 4f2b9ad7f..bc3382532 100644 --- a/example/l3fwd/odp_l3fwd_lpm.h +++ b/example/l3fwd/odp_l3fwd_lpm.h @@ -1,9 +1,9 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef _ODP_L3FWD_LPM_H_ #define _ODP_L3FWD_LPM_H_ diff --git a/example/l3fwd/odp_l3fwd_run.sh b/example/l3fwd/odp_l3fwd_run.sh index 89be1510f..0b39d5a52 100755 --- a/example/l3fwd/odp_l3fwd_run.sh +++ b/example/l3fwd/odp_l3fwd_run.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Copyright (c) 2016-2018, Linaro Limited -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2016-2018 Linaro Limited # if [ -f ./pktio_env ]; then diff --git a/example/packet/Makefile.am b/example/packet/Makefile.am index cf33b2ef7..5e4d9f5ea 100644 --- a/example/packet/Makefile.am +++ b/example/packet/Makefile.am @@ -35,5 +35,3 @@ clean-local: rm -f $(builddir)/$$f; \ done \ fi - -.NOTPARALLEL: diff --git a/example/packet/odp_packet_dump.c b/example/packet/odp_packet_dump.c index 6d00c237e..2e16861d9 100644 --- a/example/packet/odp_packet_dump.c +++ b/example/packet/odp_packet_dump.c @@ -1,7 +1,14 @@ -/* Copyright (c) 2018, Linaro Limited - * All rights reserved. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018 Linaro Limited + * Copyright (c) 2024 Nokia + */ + +/** + * @example odp_packet_dump.c + * + * Packet dump example application which prints received packets to terminal * - * SPDX-License-Identifier: BSD-3-Clause + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include <stdio.h> @@ -13,6 +20,7 @@ #include <getopt.h> #include <odp_api.h> +#include <odp/helper/odph_api.h> #define MAX_PKTIOS 32 #define MAX_PKTIO_NAME 255 @@ -85,8 +93,7 @@ static int parse_int_list(char *str, int integer[], int max_num) str[len] = 0; if (i == max_num) { - printf("Error: maximum number of options is %i\n", - max_num); + ODPH_ERR("Maximum number of options is %i\n", max_num); return -1; } @@ -155,14 +162,13 @@ static int parse_options(int argc, char *argv[], test_global_t *global) str_len -= len + 1; if (i == MAX_PKTIOS) { - printf("Error: Too many interfaces\n"); + ODPH_ERR("Too many interfaces\n"); ret = -1; break; } if (len > MAX_PKTIO_NAME) { - printf("Error: Too long interface name %s\n", - str); + ODPH_ERR("Too long interface name %s\n", str); ret = -1; break; } @@ -196,7 +202,7 @@ static int parse_options(int argc, char *argv[], test_global_t *global) } if (global->opt.num_pktio == 0) { - printf("Error: At least one pktio interface needed.\n"); + ODPH_ERR("At least one pktio interface needed\n"); ret = -1; } @@ -208,6 +214,7 @@ static int open_pktios(test_global_t *global) odp_pool_param_t pool_param; odp_pktio_param_t pktio_param; odp_pool_t pool; + odp_pktio_capability_t pktio_capa; odp_pool_capability_t pool_capa; odp_pktio_t pktio; odp_pktio_config_t pktio_config; @@ -219,7 +226,7 @@ static int open_pktios(test_global_t *global) num_pktio = global->opt.num_pktio; if (odp_pool_capability(&pool_capa)) { - printf("Error: Pool capability failed.\n"); + ODPH_ERR("Pool capability failed\n"); return -1; } @@ -235,7 +242,7 @@ static int open_pktios(test_global_t *global) global->pool = pool; if (pool == ODP_POOL_INVALID) { - printf("Error: Pool create.\n"); + ODPH_ERR("Pool create failed\n"); return -1; } @@ -252,16 +259,21 @@ static int open_pktios(test_global_t *global) pktio = odp_pktio_open(name, pool, &pktio_param); if (pktio == ODP_PKTIO_INVALID) { - printf("Error (%s): Pktio open failed.\n", name); + ODPH_ERR("Pktio open failed for %s\n", name); return -1; } global->pktio[i].pktio = pktio; + if (odp_pktio_capability(pktio, &pktio_capa)) { + ODPH_ERR("Pktio capability failed for %s\n", name); + return -1; + } + odp_pktio_print(pktio); odp_pktio_config_init(&pktio_config); - pktio_config.pktin.bit.ts_all = 1; + pktio_config.pktin.bit.ts_all = pktio_capa.config.pktin.bit.ts_all; pktio_config.parser.layer = ODP_PROTO_LAYER_ALL; odp_pktio_config(pktio, &pktio_config); @@ -274,7 +286,7 @@ static int open_pktios(test_global_t *global) pktin_param.num_queues = 1; if (odp_pktin_queue_config(pktio, &pktin_param)) { - printf("Error (%s): Pktin config failed.\n", name); + ODPH_ERR("Pktin config failed for %s\n", name); return -1; } } @@ -288,8 +300,7 @@ static int start_pktios(test_global_t *global) for (i = 0; i < global->opt.num_pktio; i++) { if (odp_pktio_start(global->pktio[i].pktio)) { - printf("Error (%s): Pktio start failed.\n", - global->opt.pktio_name[i]); + ODPH_ERR("Pktio start failed for %s\n", global->opt.pktio_name[i]); return -1; } @@ -312,8 +323,7 @@ static int stop_pktios(test_global_t *global) continue; if (odp_pktio_stop(pktio)) { - printf("Error (%s): Pktio stop failed.\n", - global->opt.pktio_name[i]); + ODPH_ERR("Pktio stop failed for %s\n", global->opt.pktio_name[i]); ret = -1; } } @@ -350,8 +360,7 @@ static int close_pktios(test_global_t *global) continue; if (odp_pktio_close(pktio)) { - printf("Error (%s): Pktio close failed.\n", - global->opt.pktio_name[i]); + ODPH_ERR("Pktio close failed for %s\n", global->opt.pktio_name[i]); ret = -1; } } @@ -362,7 +371,7 @@ static int close_pktios(test_global_t *global) return ret; if (odp_pool_destroy(pool)) { - printf("Error: Pool destroy failed.\n"); + ODPH_ERR("Pool destroy failed\n"); ret = -1; } @@ -429,9 +438,9 @@ static void print_data(odp_packet_t pkt, uint32_t offset, uint32_t len) static int print_packet(test_global_t *global, odp_packet_t pkt, uint64_t num_packet) { - odp_pktio_t pktio; + odp_pktio_t pktio = odp_packet_input(pkt); odp_pktio_info_t pktio_info; - odp_time_t time; + odp_time_t pktio_time, time; uint64_t sec, nsec; uint32_t offset; int i, type, match; @@ -448,10 +457,13 @@ static int print_packet(test_global_t *global, odp_packet_t pkt, int icmp = odp_packet_has_icmp(pkt); int ipv4 = odp_packet_has_ipv4(pkt); - if (odp_packet_has_ts(pkt)) + if (odp_packet_has_ts(pkt)) { + pktio_time = odp_pktio_time(pktio, NULL); time = odp_packet_ts(pkt); - else + } else { time = odp_time_local(); + pktio_time = ODP_TIME_NULL; + } /* Filter based on L3 type */ if (num_filter_l3) { @@ -490,15 +502,23 @@ static int print_packet(test_global_t *global, odp_packet_t pkt, nsec = odp_time_to_ns(time); sec = nsec / ODP_TIME_SEC_IN_NS; nsec = nsec - (sec * ODP_TIME_SEC_IN_NS); - pktio = odp_packet_input(pkt); if (odp_pktio_info(pktio, &pktio_info)) { - printf("Error: odp_pktio_info() failed\n"); + ODPH_ERR("Pktio info failed\n"); return -1; } printf("PACKET [%" PRIu64 "]\n", num_packet); printf(" time: %" PRIu64 ".%09" PRIu64 " sec\n", sec, nsec); + + if (odp_time_cmp(pktio_time, ODP_TIME_NULL)) { + nsec = odp_time_to_ns(pktio_time); + sec = nsec / ODP_TIME_SEC_IN_NS; + nsec = nsec - (sec * ODP_TIME_SEC_IN_NS); + printf(" pktio time: %" PRIu64 ".%09" PRIu64 " sec\n", sec, nsec); + printf(" input delay: %" PRIu64 " nsec\n", odp_time_diff_ns(pktio_time, time)); + } + printf(" interface name: %s\n", pktio_info.name); printf(" packet length: %u bytes\n", odp_packet_len(pkt)); @@ -645,7 +665,7 @@ static int receive_packets(test_global_t *global) continue; if (odp_event_type(ev) != ODP_EVENT_PACKET) { - printf("Bad event type: %i\n", odp_event_type(ev)); + ODPH_ERR("Bad event type: %i\n", odp_event_type(ev)); odp_event_free(ev); continue; } @@ -688,13 +708,13 @@ int main(int argc, char *argv[]) /* Init ODP before calling anything else */ if (odp_init_global(&instance, NULL, NULL)) { - printf("Error: Global init failed.\n"); + ODPH_ERR("Global init failed\n"); return -1; } /* Init this thread */ if (odp_init_local(instance, ODP_THREAD_CONTROL)) { - printf("Error: Local init failed.\n"); + ODPH_ERR("Local init failed\n"); return -1; } @@ -705,39 +725,39 @@ int main(int argc, char *argv[]) odp_sys_info_print(); if (open_pktios(global)) { - printf("Error: pktio open failed\n"); + ODPH_ERR("Pktio open failed\n"); return -1; } if (start_pktios(global)) { - printf("Error: pktio start failed\n"); + ODPH_ERR("Pktio start failed\n"); return -1; } if (receive_packets(global)) { - printf("Error: packet receive failed\n"); + ODPH_ERR("Packet receive failed\n"); return -1; } if (stop_pktios(global)) { - printf("Error: pktio stop failed\n"); + ODPH_ERR("Pktio stop failed\n"); return -1; } empty_queues(); if (close_pktios(global)) { - printf("Error: pktio close failed\n"); + ODPH_ERR("Pktio close failed\n"); return -1; } if (odp_term_local()) { - printf("Error: term local failed.\n"); + ODPH_ERR("Term local failed\n"); return -1; } if (odp_term_global(instance)) { - printf("Error: term global failed.\n"); + ODPH_ERR("Term global failed\n"); return -1; } diff --git a/example/packet/odp_pktio.c b/example/packet/odp_pktio.c index 191d9fdb6..a82537fe7 100644 --- a/example/packet/odp_pktio.c +++ b/example/packet/odp_pktio.c @@ -1,7 +1,13 @@ -/* Copyright (c) 2013-2018, Linaro Limited - * All rights reserved. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2013-2018 Linaro Limited + */ + +/** + * @example odp_pktio.c + * + * Basic packet IO loopback example application * - * SPDX-License-Identifier: BSD-3-Clause + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include <stdlib.h> @@ -66,7 +72,7 @@ typedef struct { char **if_names; /**< Array of pointers to interface names */ int mode; /**< Packet IO mode */ char *if_str; /**< Storage for interface names */ - int time; /**< Time to run app */ + double time; /**< Time to run app */ } appl_args_t; /** @@ -473,7 +479,7 @@ int main(int argc, char *argv[]) odp_pktio_stop(pktio); } /* use delay to let workers clean up queues */ - odp_time_wait_ns(ODP_TIME_SEC_IN_NS); + odp_time_wait_ns(100 * ODP_TIME_MSEC_IN_NS); odp_atomic_store_u32(&args->exit_threads, 1); } @@ -609,7 +615,7 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) appl_args->cpu_count = atoi(optarg); break; case 't': - appl_args->time = atoi(optarg); + appl_args->time = atof(optarg); break; /* parse packet-io interface names */ case 'i': @@ -736,7 +742,7 @@ static void usage(char *progname) "\n" "Optional OPTIONS\n" " -c, --count <number> CPU count, 0=all available, default=1\n" - " -t, --time <seconds> Number of seconds to run.\n" + " -t, --time <seconds> Number of seconds to run (e.g. 0.1).\n" " -m, --mode 0: Receive and send directly (no queues)\n" " 1: Receive and send via queues.\n" " 2: Receive via scheduler, send via queues.\n" diff --git a/example/packet/packet_dump_run.sh b/example/packet/packet_dump_run.sh index 4e7861b1c..fea5d8b3c 100755 --- a/example/packet/packet_dump_run.sh +++ b/example/packet/packet_dump_run.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Copyright (c) 2018, Linaro Limited -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2018 Linaro Limited # if [ -f ./pktio_env ]; then diff --git a/example/packet/pktio_run.sh b/example/packet/pktio_run.sh index 33115b41f..f08c45936 100755 --- a/example/packet/pktio_run.sh +++ b/example/packet/pktio_run.sh @@ -1,11 +1,11 @@ #!/bin/bash # -# Copyright (c) 2016-2018, Linaro Limited -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2016-2018 Linaro Limited # +TEST_TIME=0.1 + if [ -f ./pktio_env ]; then . ./pktio_env else @@ -17,7 +17,7 @@ fi setup_interfaces # burst mode -./odp_pktio${EXEEXT} -i $IF1 -t 1 -m 0 +./odp_pktio${EXEEXT} -i $IF1 -t $TEST_TIME -m 0 STATUS=$? if [ ${STATUS} -ne 0 ]; then echo "Error: status ${STATUS}" @@ -28,7 +28,7 @@ validate_result echo "Pass -m 0: status ${STATUS}" # queue mode -./odp_pktio${EXEEXT} -i $IF1 -t 1 -m 1 +./odp_pktio${EXEEXT} -i $IF1 -t $TEST_TIME -m 1 STATUS=$? if [ ${STATUS} -ne 0 ]; then @@ -40,7 +40,7 @@ validate_result echo "Pass -m 1: status ${STATUS}" # sched/queue mode -./odp_pktio${EXEEXT} -i $IF1 -t 1 -m 2 +./odp_pktio${EXEEXT} -i $IF1 -t $TEST_TIME -m 2 STATUS=$? if [ ${STATUS} -ne 0 ]; then @@ -52,7 +52,7 @@ validate_result echo "Pass -m 2: status ${STATUS}" # cpu number option test 1 -./odp_pktio${EXEEXT} -i $IF1 -t 1 -m 0 -c 1 +./odp_pktio${EXEEXT} -i $IF1 -t $TEST_TIME -m 0 -c 1 STATUS=$? if [ ${STATUS} -ne 0 ]; then diff --git a/example/ping/odp_ping.c b/example/ping/odp_ping.c index dbe453319..97b856895 100644 --- a/example/ping/odp_ping.c +++ b/example/ping/odp_ping.c @@ -2,6 +2,16 @@ * Copyright (c) 2019-2023 Nokia */ +/** + * @example odp_ping.c + * + * This application replies to IPv4 ping requests. It can be used to test + * connectivity with standard ping utility. ARP table needs to be setup manually + * on the sender side as the application does not reply to ARP requests. + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include <stdio.h> #include <string.h> #include <signal.h> diff --git a/example/ping/ping_run.sh b/example/ping/ping_run.sh index 0017c4ce0..9598db70a 100755 --- a/example/ping/ping_run.sh +++ b/example/ping/ping_run.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Copyright (c) 2019, Nokia -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2019 Nokia # if [ -f ./pktio_env ]; then diff --git a/example/simple_pipeline/odp_simple_pipeline.c b/example/simple_pipeline/odp_simple_pipeline.c index b8fb2d0be..ad80bc394 100644 --- a/example/simple_pipeline/odp_simple_pipeline.c +++ b/example/simple_pipeline/odp_simple_pipeline.c @@ -1,9 +1,18 @@ -/* Copyright (c) 2019, Nokia - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2019 Nokia */ + /** + * @example odp_simple_pipeline.c + * + * Simple pipeline example application which receives packets from one + * interface and passes them through 0-N worker stages before outputting them + * from a second network interface. The RX, worker, and TX stages are connected + * using plain queues and each stage is run on a separate CPU thread. + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include <stdlib.h> #include <stdio.h> #include <getopt.h> diff --git a/example/simple_pipeline/simple_pipeline_run.sh b/example/simple_pipeline/simple_pipeline_run.sh index f9248c42d..ba66e506e 100755 --- a/example/simple_pipeline/simple_pipeline_run.sh +++ b/example/simple_pipeline/simple_pipeline_run.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Copyright (c) 2019, Nokia -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2019 Nokia # # Exit code expected by automake for skipped tests diff --git a/example/switch/odp_switch.c b/example/switch/odp_switch.c index ae12b0b11..0160c2d0a 100644 --- a/example/switch/odp_switch.c +++ b/example/switch/odp_switch.c @@ -1,8 +1,14 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * Copyright (c) 2020, Nokia - * All rights reserved. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited + * Copyright (c) 2020 Nokia + */ + +/** + * @example odp_switch.c + * + * Minimal learning Ethernet switch example application * - * SPDX-License-Identifier: BSD-3-Clause + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include <stdio.h> diff --git a/example/switch/switch_run.sh b/example/switch/switch_run.sh index 489b17de5..c01c505ba 100755 --- a/example/switch/switch_run.sh +++ b/example/switch/switch_run.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Copyright (c) 2016-2018, Linaro Limited -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2016-2018 Linaro Limited # RETVAL=0 diff --git a/example/sysinfo/odp_sysinfo.c b/example/sysinfo/odp_sysinfo.c index 151388263..df33f45cf 100644 --- a/example/sysinfo/odp_sysinfo.c +++ b/example/sysinfo/odp_sysinfo.c @@ -1,10 +1,17 @@ -/* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2022-2023, Nokia - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018 Linaro Limited + * Copyright (c) 2022-2023 Nokia */ + /** + * @example odp_sysinfo.c + * + * Example application which queries and prints out various system information + * and capabilities which are available through ODP APIs. + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif @@ -739,6 +746,7 @@ int main(int argc, char **argv) odp_ipsec_capability_t ipsec_capa; odp_schedule_capability_t schedule_capa; odp_stash_capability_t stash_capa; + odp_ml_capability_t ml_capa; appl_args_t appl_args; uint64_t huge_page[MAX_HUGE_PAGES]; char ava_mask_str[ODP_CPUMASK_STR_SIZE]; @@ -854,6 +862,11 @@ int main(int argc, char **argv) exit(EXIT_FAILURE); } + if (odp_ml_capability(&ml_capa)) { + ODPH_ERR("ml capability failed\n"); + exit(EXIT_FAILURE); + } + crypto_ret = odp_crypto_capability(&crypto_capa); if (crypto_ret < 0) ODPH_ERR("crypto capability failed\n"); @@ -1066,6 +1079,33 @@ int main(int argc, char **argv) printf(" max_put_batch: %u\n", stash_capa.max_put_batch); printf(" stats: 0x%" PRIx64 "\n", stash_capa.stats.all); + printf("\n"); + printf(" ML\n"); + printf(" max_models: %u\n", ml_capa.max_models); + printf(" max_models_loaded: %u\n", ml_capa.max_models_loaded); + printf(" max_model_size: %" PRIu64 "B\n", ml_capa.max_model_size); + printf(" max_compl_id: %u\n", ml_capa.max_compl_id); + printf(" max_inputs: %u\n", ml_capa.max_inputs); + printf(" max_outputs: %u\n", ml_capa.max_outputs); + printf(" max_segs_per_input: %u\n", ml_capa.max_segs_per_input); + printf(" max_segs_per_output: %u\n", ml_capa.max_segs_per_output); + printf(" min_input_align: %u\n", ml_capa.min_input_align); + printf(" min_output_align: %u\n", ml_capa.min_output_align); + printf(" packed_input_data: %u\n", ml_capa.packed_input_data); + printf(" packed_output_data: %u\n", ml_capa.packed_output_data); + printf(" load.compl_mode_mask: 0x%x\n", ml_capa.load.compl_mode_mask); + printf(" load.compl_queue_plain: %i\n", ml_capa.load.compl_queue_plain); + printf(" load.compl_queue_sched: %i\n", ml_capa.load.compl_queue_sched); + printf(" run.compl_mode_mask: 0x%x\n", ml_capa.run.compl_mode_mask); + printf(" run.compl_queue_plain: %i\n", ml_capa.run.compl_queue_plain); + printf(" run.compl_queue_sched: %i\n", ml_capa.run.compl_queue_sched); + printf(" pool.max_pools: %u\n", ml_capa.pool.max_pools); + printf(" pool.max_num: %u\n", ml_capa.pool.max_num); + printf(" pool.max_uarea_size: %u B\n", ml_capa.pool.max_uarea_size); + printf(" pool.uarea_persistence: %u\n", ml_capa.pool.uarea_persistence); + printf(" pool.min_cache_size: %u\n", ml_capa.pool.min_cache_size); + printf(" pool.max_cache_size: %u\n", ml_capa.pool.max_cache_size); + print_timer_capa(&appl_args); if (crypto_ret == 0) { diff --git a/example/time/Makefile.am b/example/time/Makefile.am index 5621ef99e..b5a1ad35d 100644 --- a/example/time/Makefile.am +++ b/example/time/Makefile.am @@ -6,4 +6,4 @@ if test_example TESTS = odp_time_global endif -odp_time_global_SOURCES = time_global_test.c +odp_time_global_SOURCES = odp_time_global_test.c diff --git a/example/time/time_global_test.c b/example/time/odp_time_global_test.c index 2357f25d6..7c1409abf 100644 --- a/example/time/time_global_test.c +++ b/example/time/odp_time_global_test.c @@ -1,7 +1,13 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2018 Linaro Limited + */ + +/** + * @example odp_time_global_test.c + * + * Time API test application * - * SPDX-License-Identifier: BSD-3-Clause + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include <inttypes.h> diff --git a/example/timer/Makefile.am b/example/timer/Makefile.am index da85be233..66cf0edfb 100644 --- a/example/timer/Makefile.am +++ b/example/timer/Makefile.am @@ -16,5 +16,3 @@ TESTS = odp_timer_accuracy_run.sh \ endif EXTRA_DIST = odp_timer_accuracy_run.sh - -.NOTPARALLEL: diff --git a/example/timer/odp_timer_accuracy.c b/example/timer/odp_timer_accuracy.c index 23c40e66f..e5df1c24e 100644 --- a/example/timer/odp_timer_accuracy.c +++ b/example/timer/odp_timer_accuracy.c @@ -1,8 +1,14 @@ -/* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2019-2023, Nokia - * All rights reserved. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2018 Linaro Limited + * Copyright (c) 2019-2023 Nokia + */ + +/** + * @example odp_timer_accuracy.c * - * SPDX-License-Identifier: BSD-3-Clause + * ODP timer accuracy test application + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include <errno.h> @@ -761,7 +767,7 @@ static int start_timers(test_global_t *test_global) odp_timer_start_t start_param; if (mode == MODE_PERIODIC) { - odp_timer_periodic_start_t start_param; + odp_timer_periodic_start_t periodic_start; nsec = offset_ns + (j * burst_gap); @@ -775,13 +781,13 @@ static int start_timers(test_global_t *test_global) ctx->first_period = start_tick + odp_timer_ns_to_tick(timer_pool, test_global->period_dbl + 0.5); - start_param.freq_multiplier = test_global->opt.multiplier; - start_param.first_tick = 0; + periodic_start.freq_multiplier = test_global->opt.multiplier; + periodic_start.first_tick = 0; if (nsec) - start_param.first_tick = + periodic_start.first_tick = start_tick + odp_timer_ns_to_tick(timer_pool, nsec); - start_param.tmo_ev = ctx->event; - retval = odp_timer_periodic_start(ctx->timer, &start_param); + periodic_start.tmo_ev = ctx->event; + retval = odp_timer_periodic_start(ctx->timer, &periodic_start); } else { nsec = offset_ns + (i * period_ns) + (j * burst_gap); ctx->nsec = start_ns + nsec; @@ -869,7 +875,6 @@ static void print_nsec_error(const char *str, int64_t nsec, double res_ns, static void print_stat(test_global_t *test_global) { - uint64_t i; test_stat_t test_stat; test_stat_t *stat = &test_stat; uint64_t tot_timers; @@ -937,7 +942,7 @@ static void print_stat(test_global_t *test_global) fprintf(file, " Timer thread tmo(ns) diff(ns)\n"); - for (i = 0; i < tot_timers; i++) { + for (uint64_t i = 0; i < tot_timers; i++) { fprintf(file, "%8" PRIu64 " %7u %12" PRIu64 " %10" PRIi64 "\n", i, log[i].tid, log[i].tmo_ns, log[i].diff_ns); } diff --git a/example/timer/odp_timer_accuracy_run.sh b/example/timer/odp_timer_accuracy_run.sh index 3b29191c4..1c879f4e6 100755 --- a/example/timer/odp_timer_accuracy_run.sh +++ b/example/timer/odp_timer_accuracy_run.sh @@ -1,9 +1,7 @@ #!/bin/bash # -# Copyright (c) 2022, Nokia -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2022 Nokia # ./odp_timer_accuracy${EXEEXT} -p 100000000 -n 10 diff --git a/example/timer/odp_timer_simple.c b/example/timer/odp_timer_simple.c index 681f95714..ceba66c62 100644 --- a/example/timer/odp_timer_simple.c +++ b/example/timer/odp_timer_simple.c @@ -1,13 +1,13 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2016-2018 Linaro Limited */ + /** - * @file + * @example odp_timer_simple.c + * + * Minimal example application demonstrating ODP timer API usage * - * @example odp_timer_simple.c ODP simple example to schedule timer - * action for 1 second. + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include <string.h> @@ -121,9 +121,8 @@ int main(int argc ODP_UNUSED, char *argv[] ODP_UNUSED) ev = odp_timeout_to_event(tmo); - /* Calculate period for timer in uint64_t value, in current case - * we will schedule timer for 1 second */ - period = odp_timer_ns_to_tick(timer_pool, 1 * ODP_TIME_SEC_IN_NS); + /* Calculate timer period in ticks */ + period = odp_timer_ns_to_tick(timer_pool, 100 * ODP_TIME_MSEC_IN_NS); /* Wait time to return from odp_schedule() if there are no * events @@ -159,7 +158,7 @@ int main(int argc ODP_UNUSED, char *argv[] ODP_UNUSED) i, odp_time_to_ns(time)); /* Do not free current event, just go back to loop and program - * timeout to next second. + * the next timeout. */ } diff --git a/example/timer/odp_timer_test.c b/example/timer/odp_timer_test.c index a397a6ac9..674d4f4ce 100644 --- a/example/timer/odp_timer_test.c +++ b/example/timer/odp_timer_test.c @@ -1,7 +1,13 @@ -/* Copyright (c) 2013-2018, Linaro Limited - * All rights reserved. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2013-2018 Linaro Limited + */ + +/** + * @example odp_timer_test.c * - * SPDX-License-Identifier: BSD-3-Clause + * Timer test application + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include <string.h> @@ -161,7 +167,7 @@ static void test_abs_timeouts(int thr, test_globals_t *gbls) ODPH_ABORT("Unexpected event type (%u) received\n", odp_event_type(ev)); } - odp_timeout_t tmo = odp_timeout_from_event(ev); + tmo = odp_timeout_from_event(ev); tick = odp_timeout_tick(tmo); ttp = odp_timeout_user_ptr(tmo); ttp->ev = ev; @@ -178,7 +184,10 @@ static void test_abs_timeouts(int thr, test_globals_t *gbls) continue; odp_event_free(ttp->ev); - odp_timer_free(ttp->tim); + + if (odp_timer_free(ttp->tim)) + ODPH_ABORT("Timer free failed (%" PRIu64 ")\n", odp_timer_to_u64(ttp->tim)); + ttp = NULL; } diff --git a/example/traffic_mgmt/odp_traffic_mgmt.c b/example/traffic_mgmt/odp_traffic_mgmt.c index 5c97c5433..4ed4f2044 100644 --- a/example/traffic_mgmt/odp_traffic_mgmt.c +++ b/example/traffic_mgmt/odp_traffic_mgmt.c @@ -1,12 +1,17 @@ -/* Copyright 2015 EZchip Semiconductor Ltd. All Rights Reserved. - * - * Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2022, Marvell - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015 EZchip Semiconductor Ltd. + * Copyright (c) 2015-2018 Linaro Limited + * Copyright (c) 2022 Marvell */ + /** + * @example odp_traffic_mgmt.c + * + * Traffic manager API example application + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif @@ -287,7 +292,7 @@ static uint32_t next_rand_byte; static odp_atomic_u32_t atomic_pkts_into_tm; static odp_atomic_u32_t atomic_pkts_from_tm; -static uint32_t g_num_pkts_to_send = 1000; +static uint32_t g_num_pkts_to_send = 100; static uint8_t g_print_tm_stats = TRUE; static void tester_egress_fcn(odp_packet_t odp_pkt); @@ -632,20 +637,19 @@ static int create_and_config_tm(void) tm_shaper_max_burst = tm_capa.per_level[0].max_burst; for (level = 1; level < tm_capa.max_levels; level++) { - odp_tm_level_capabilities_t *per_level = - &tm_capa.per_level[level]; + odp_tm_level_capabilities_t *level_capa = &tm_capa.per_level[level]; - if (per_level->min_rate > tm_shaper_min_rate) - tm_shaper_min_rate = per_level->min_rate; + if (level_capa->min_rate > tm_shaper_min_rate) + tm_shaper_min_rate = level_capa->min_rate; - if (per_level->min_burst > tm_shaper_min_burst) - tm_shaper_min_burst = per_level->min_burst; + if (level_capa->min_burst > tm_shaper_min_burst) + tm_shaper_min_burst = level_capa->min_burst; - if (per_level->max_rate < tm_shaper_max_rate) - tm_shaper_max_rate = per_level->max_rate; + if (level_capa->max_rate < tm_shaper_max_rate) + tm_shaper_max_rate = level_capa->max_rate; - if (per_level->max_burst < tm_shaper_max_burst) - tm_shaper_max_burst = per_level->max_burst; + if (level_capa->max_burst < tm_shaper_max_burst) + tm_shaper_max_burst = level_capa->max_burst; } if (tm_shaper_min_rate > tm_shaper_max_rate || diff --git a/helper/include/odp/helper/macros.h b/helper/include/odp/helper/macros.h index e30db7bab..1623f17e7 100644 --- a/helper/include/odp/helper/macros.h +++ b/helper/include/odp/helper/macros.h @@ -32,9 +32,9 @@ extern "C" { */ #define ODPH_MIN(a, b) \ __extension__ ({ \ - __typeof__(a) tmp_a = (a); \ - __typeof__(b) tmp_b = (b); \ - tmp_a < tmp_b ? tmp_a : tmp_b; \ + __typeof__(a) min_a = (a); \ + __typeof__(b) min_b = (b); \ + min_a < min_b ? min_a : min_b; \ }) /** @@ -42,9 +42,9 @@ extern "C" { */ #define ODPH_MAX(a, b) \ __extension__ ({ \ - __typeof__(a) tmp_a = (a); \ - __typeof__(b) tmp_b = (b); \ - tmp_a > tmp_b ? tmp_a : tmp_b; \ + __typeof__(a) max_a = (a); \ + __typeof__(b) max_b = (b); \ + max_a > max_b ? max_a : max_b; \ }) /** @@ -52,8 +52,8 @@ extern "C" { */ #define ODPH_ABS(v) \ __extension__ ({ \ - __typeof__(v) tmp_v = (v); \ - tmp_v < 0 ? -tmp_v : tmp_v; \ + __typeof__(v) abs_v = (v); \ + abs_v < 0 ? -abs_v : abs_v; \ }) /** diff --git a/helper/test/Makefile.am b/helper/test/Makefile.am index 80aac1083..9cf48d7d9 100644 --- a/helper/test/Makefile.am +++ b/helper/test/Makefile.am @@ -67,5 +67,3 @@ clean-local: rm -f $(builddir)/$$f; \ done \ fi - -.NOTPARALLEL: diff --git a/include/Makefile.am b/include/Makefile.am index bc28a5cba..9e23c4fa3 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -34,6 +34,9 @@ odpapiinclude_HEADERS = \ odp/api/init.h \ odp/api/ipsec.h \ odp/api/ipsec_types.h \ + odp/api/ml.h \ + odp/api/ml_quantize.h \ + odp/api/ml_types.h \ odp/api/packet.h \ odp/api/packet_types.h \ odp/api/packet_flags.h \ @@ -101,6 +104,9 @@ odpapispecinclude_HEADERS = \ odp/api/spec/init.h \ odp/api/spec/ipsec.h \ odp/api/spec/ipsec_types.h \ + odp/api/spec/ml.h \ + odp/api/spec/ml_quantize.h \ + odp/api/spec/ml_types.h \ odp/api/spec/packet.h \ odp/api/spec/packet_types.h \ odp/api/spec/packet_flags.h \ @@ -169,6 +175,7 @@ odpapiabidefaultinclude_HEADERS = \ odp/api/abi-default/init.h \ odp/api/abi-default/ipsec.h \ odp/api/abi-default/ipsec_types.h \ + odp/api/abi-default/ml_types.h \ odp/api/abi-default/packet.h \ odp/api/abi-default/packet_types.h \ odp/api/abi-default/packet_flags.h \ @@ -232,6 +239,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/arm32-linux/odp/api/abi/init.h \ odp/arch/arm32-linux/odp/api/abi/ipsec.h \ odp/arch/arm32-linux/odp/api/abi/ipsec_types.h \ + odp/arch/arm32-linux/odp/api/abi/ml_types.h \ odp/arch/arm32-linux/odp/api/abi/packet.h \ odp/arch/arm32-linux/odp/api/abi/packet_types.h \ odp/arch/arm32-linux/odp/api/abi/packet_flags.h \ @@ -291,6 +299,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/arm64-linux/odp/api/abi/init.h \ odp/arch/arm64-linux/odp/api/abi/ipsec.h \ odp/arch/arm64-linux/odp/api/abi/ipsec_types.h \ + odp/arch/arm64-linux/odp/api/abi/ml_types.h \ odp/arch/arm64-linux/odp/api/abi/packet.h \ odp/arch/arm64-linux/odp/api/abi/packet_types.h \ odp/arch/arm64-linux/odp/api/abi/packet_flags.h \ @@ -350,6 +359,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/default-linux/odp/api/abi/init.h \ odp/arch/default-linux/odp/api/abi/ipsec.h \ odp/arch/default-linux/odp/api/abi/ipsec_types.h \ + odp/arch/default-linux/odp/api/abi/ml_types.h \ odp/arch/default-linux/odp/api/abi/packet.h \ odp/arch/default-linux/odp/api/abi/packet_types.h \ odp/arch/default-linux/odp/api/abi/packet_flags.h \ @@ -409,6 +419,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/power64-linux/odp/api/abi/init.h \ odp/arch/power64-linux/odp/api/abi/ipsec.h \ odp/arch/power64-linux/odp/api/abi/ipsec_types.h \ + odp/arch/power64-linux/odp/api/abi/ml_types.h \ odp/arch/power64-linux/odp/api/abi/packet.h \ odp/arch/power64-linux/odp/api/abi/packet_types.h \ odp/arch/power64-linux/odp/api/abi/packet_flags.h \ @@ -468,6 +479,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/x86_32-linux/odp/api/abi/init.h \ odp/arch/x86_32-linux/odp/api/abi/ipsec.h \ odp/arch/x86_32-linux/odp/api/abi/ipsec_types.h \ + odp/arch/x86_32-linux/odp/api/abi/ml_types.h \ odp/arch/x86_32-linux/odp/api/abi/packet.h \ odp/arch/x86_32-linux/odp/api/abi/packet_types.h \ odp/arch/x86_32-linux/odp/api/abi/packet_flags.h \ @@ -527,6 +539,7 @@ odpapiabiarchinclude_HEADERS = \ odp/arch/x86_64-linux/odp/api/abi/init.h \ odp/arch/x86_64-linux/odp/api/abi/ipsec.h \ odp/arch/x86_64-linux/odp/api/abi/ipsec_types.h \ + odp/arch/x86_64-linux/odp/api/abi/ml_types.h \ odp/arch/x86_64-linux/odp/api/abi/packet.h \ odp/arch/x86_64-linux/odp/api/abi/packet_types.h \ odp/arch/x86_64-linux/odp/api/abi/packet_flags.h \ diff --git a/include/odp/api/abi-default/align.h b/include/odp/api/abi-default/align.h index 0fa058549..fa95d728b 100644 --- a/include/odp/api/abi-default/align.h +++ b/include/odp/api/abi-default/align.h @@ -17,7 +17,7 @@ extern "C" { #include <odp/api/abi/cpu.h> -/** @ingroup odp_compiler_optim +/** @addtogroup odp_compiler_optim * @{ */ diff --git a/include/odp/api/abi-default/buffer_types.h b/include/odp/api/abi-default/buffer_types.h index 59588224f..9179ae321 100644 --- a/include/odp/api/abi-default/buffer_types.h +++ b/include/odp/api/abi-default/buffer_types.h @@ -13,7 +13,7 @@ extern "C" { /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_buffer_t; -/** @ingroup odp_buffer +/** @addtogroup odp_buffer * @{ */ diff --git a/include/odp/api/abi-default/classification.h b/include/odp/api/abi-default/classification.h index e7519329a..fdc98f252 100644 --- a/include/odp/api/abi-default/classification.h +++ b/include/odp/api/abi-default/classification.h @@ -15,7 +15,7 @@ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_cos_t; /** Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_pmr_t; -/** @ingroup odp_classification +/** @addtogroup odp_classification * @{ */ diff --git a/include/odp/api/abi-default/comp.h b/include/odp/api/abi-default/comp.h index b5638eba8..3f936aa20 100644 --- a/include/odp/api/abi-default/comp.h +++ b/include/odp/api/abi-default/comp.h @@ -14,7 +14,7 @@ extern "C" { /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_comp_session_t; -/** @ingroup odp_compression +/** @addtogroup odp_compression * @{ */ diff --git a/include/odp/api/abi-default/crypto_types.h b/include/odp/api/abi-default/crypto_types.h index 58898dfea..8d860b6ef 100644 --- a/include/odp/api/abi-default/crypto_types.h +++ b/include/odp/api/abi-default/crypto_types.h @@ -12,7 +12,7 @@ extern "C" { #include <stdint.h> -/** @ingroup odp_crypto +/** @addtogroup odp_crypto * @{ */ diff --git a/include/odp/api/abi-default/dma_types.h b/include/odp/api/abi-default/dma_types.h index 1d27a11aa..005ba3d16 100644 --- a/include/odp/api/abi-default/dma_types.h +++ b/include/odp/api/abi-default/dma_types.h @@ -17,7 +17,7 @@ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_dma_t; /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_dma_compl_t; -/** @ingroup odp_dma +/** @addtogroup odp_dma * @{ */ diff --git a/include/odp/api/abi-default/event_types.h b/include/odp/api/abi-default/event_types.h index d6231a98f..e5b50d9c0 100644 --- a/include/odp/api/abi-default/event_types.h +++ b/include/odp/api/abi-default/event_types.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2017-2018 Linaro Limited - * Copyright (c) 2022 Nokia + * Copyright (c) 2022-2023 Nokia */ #ifndef ODP_ABI_EVENT_TYPES_H_ @@ -15,7 +15,7 @@ extern "C" { /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_event_t; -/** @ingroup odp_event +/** @addtogroup odp_event * @{ */ @@ -31,6 +31,7 @@ typedef enum { ODP_EVENT_PACKET_VECTOR = 6, ODP_EVENT_PACKET_TX_COMPL = 7, ODP_EVENT_DMA_COMPL = 8, + ODP_EVENT_ML_COMPL = 9 } odp_event_type_t; typedef enum { @@ -38,7 +39,9 @@ typedef enum { ODP_EVENT_PACKET_BASIC = 1, ODP_EVENT_PACKET_CRYPTO = 2, ODP_EVENT_PACKET_IPSEC = 3, - ODP_EVENT_PACKET_COMP = 4 + ODP_EVENT_PACKET_COMP = 4, + ODP_EVENT_ML_COMPL_LOAD = 5, + ODP_EVENT_ML_COMPL_RUN = 6 } odp_event_subtype_t; /** diff --git a/include/odp/api/abi-default/ipsec_types.h b/include/odp/api/abi-default/ipsec_types.h index 737f67153..9d099b80d 100644 --- a/include/odp/api/abi-default/ipsec_types.h +++ b/include/odp/api/abi-default/ipsec_types.h @@ -15,7 +15,7 @@ extern "C" { /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_ipsec_sa_t; -/** @ingroup odp_ipsec +/** @addtogroup odp_ipsec * @{ */ diff --git a/include/odp/api/abi-default/ml_types.h b/include/odp/api/abi-default/ml_types.h new file mode 100644 index 000000000..723beb1bc --- /dev/null +++ b/include/odp/api/abi-default/ml_types.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021-2023 Nokia + */ + +#ifndef ODP_ABI_ML_TYPES_H_ +#define ODP_ABI_ML_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/** @internal Dummy type for strong typing */ +typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_ml_model_t; + +/** @internal Dummy type for strong typing */ +typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_ml_compl_t; + +/** @internal Implementation specific ML parameters */ +struct _odp_ml_model_extra_param_t { + /** @internal Dummy field to avoid empty struct */ + char dummy; +}; + +/** @addtogroup odp_ml + * @{ + */ + +typedef _odp_abi_ml_model_t *odp_ml_model_t; +typedef _odp_abi_ml_compl_t *odp_ml_compl_t; +typedef struct _odp_ml_model_extra_param_t odp_ml_model_extra_param_t; + +#define ODP_ML_MODEL_INVALID ((odp_ml_model_t)0) +#define ODP_ML_COMPL_INVALID ((odp_ml_compl_t)0) + +#define ODP_ML_MODEL_NAME_LEN 64 +#define ODP_ML_MODEL_IO_NAME_LEN 64 +#define ODP_ML_SHAPE_NAME_LEN 16 +#define ODP_ML_EXTRA_STAT_NAME_LEN 64 + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/odp/api/abi-default/packet_io_types.h b/include/odp/api/abi-default/packet_io_types.h index ddf8c3a3f..1aa1cf816 100644 --- a/include/odp/api/abi-default/packet_io_types.h +++ b/include/odp/api/abi-default/packet_io_types.h @@ -25,7 +25,6 @@ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_pktio_t; typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_lso_profile_t; /** @addtogroup odp_packet_io - * Operations on a packet. * @{ */ diff --git a/include/odp/api/abi-default/packet_types.h b/include/odp/api/abi-default/packet_types.h index 300eaf177..e8b2c8484 100644 --- a/include/odp/api/abi-default/packet_types.h +++ b/include/odp/api/abi-default/packet_types.h @@ -27,7 +27,7 @@ typedef struct { char dummy; /**< *internal Dummy */ } _odp_abi_packet_vector_t; /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< *internal Dummy */ } _odp_abi_packet_tx_compl_t; -/** @ingroup odp_packet +/** @addtogroup odp_packet * @{ */ diff --git a/include/odp/api/abi-default/pool_types.h b/include/odp/api/abi-default/pool_types.h index e4ca40422..ce1042c12 100644 --- a/include/odp/api/abi-default/pool_types.h +++ b/include/odp/api/abi-default/pool_types.h @@ -13,7 +13,7 @@ extern "C" { /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_pool_t; -/** @ingroup odp_pool +/** @addtogroup odp_pool * @{ */ diff --git a/include/odp/api/abi-default/proto_stats_types.h b/include/odp/api/abi-default/proto_stats_types.h index 0d87012f3..e17adf886 100644 --- a/include/odp/api/abi-default/proto_stats_types.h +++ b/include/odp/api/abi-default/proto_stats_types.h @@ -15,8 +15,7 @@ extern "C" { /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_proto_stats_t; -/** @ingroup odp_proto_stats - * Operations on a proto stats object. +/** @addtogroup odp_proto_stats * @{ */ diff --git a/include/odp/api/abi-default/queue_types.h b/include/odp/api/abi-default/queue_types.h index 5a1dc40d9..677348c18 100644 --- a/include/odp/api/abi-default/queue_types.h +++ b/include/odp/api/abi-default/queue_types.h @@ -13,7 +13,7 @@ extern "C" { /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_queue_t; -/** @ingroup odp_queue +/** @addtogroup odp_queue * @{ */ diff --git a/include/odp/api/abi-default/shared_memory.h b/include/odp/api/abi-default/shared_memory.h index f4930da27..70d6e906f 100644 --- a/include/odp/api/abi-default/shared_memory.h +++ b/include/odp/api/abi-default/shared_memory.h @@ -12,7 +12,7 @@ extern "C" { /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_shm_t; -/** @ingroup odp_shared_memory +/** @addtogroup odp_shared_memory * @{ */ diff --git a/include/odp/api/abi-default/stash_types.h b/include/odp/api/abi-default/stash_types.h index 10db242d3..6779f3af6 100644 --- a/include/odp/api/abi-default/stash_types.h +++ b/include/odp/api/abi-default/stash_types.h @@ -12,7 +12,7 @@ extern "C" { /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_stash_t; -/** @ingroup odp_stash +/** @addtogroup odp_stash * @{ */ diff --git a/include/odp/api/abi-default/thread_types.h b/include/odp/api/abi-default/thread_types.h index 1511f488d..d8c27fb98 100644 --- a/include/odp/api/abi-default/thread_types.h +++ b/include/odp/api/abi-default/thread_types.h @@ -9,7 +9,7 @@ extern "C" { #endif -/** @ingroup odp_thread +/** @addtogroup odp_thread * @{ */ diff --git a/include/odp/api/abi-default/time_types.h b/include/odp/api/abi-default/time_types.h index 32d9384dd..afbe6d188 100644 --- a/include/odp/api/abi-default/time_types.h +++ b/include/odp/api/abi-default/time_types.h @@ -11,7 +11,7 @@ extern "C" { #include <stdint.h> -/** @ingroup odp_time +/** @addtogroup odp_time * @{ **/ diff --git a/include/odp/api/ml.h b/include/odp/api/ml.h new file mode 100644 index 000000000..55213dd52 --- /dev/null +++ b/include/odp/api/ml.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021 Nokia + */ + +/** + * @file + * + * ODP Machine Learning (ML) offload + */ + +#ifndef ODP_API_ML_H_ +#define ODP_API_ML_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/spec/ml.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/odp/api/ml_quantize.h b/include/odp/api/ml_quantize.h new file mode 100644 index 000000000..cab2a3f22 --- /dev/null +++ b/include/odp/api/ml_quantize.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +/** + * @file + * + * ODP Machine Learning (ML) offload + */ + +#ifndef ODP_API_ML_QUANTIZE_H_ +#define ODP_API_ML_QUANTIZE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/spec/ml_quantize.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/odp/api/ml_types.h b/include/odp/api/ml_types.h new file mode 100644 index 000000000..3c3f8a416 --- /dev/null +++ b/include/odp/api/ml_types.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021 Nokia + */ + +/** + * @file + * + * ODP Machine Learning (ML) types + */ + +#ifndef ODP_API_ML_TYPES_H_ +#define ODP_API_ML_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/abi/ml_types.h> + +#include <odp/api/spec/ml_types.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/odp/api/spec/buffer.h b/include/odp/api/spec/buffer.h index 11750136d..5ce2355b8 100644 --- a/include/odp/api/spec/buffer.h +++ b/include/odp/api/spec/buffer.h @@ -22,7 +22,7 @@ extern "C" { #include <odp/api/pool_types.h> #include <odp/api/std_types.h> -/** @defgroup odp_buffer ODP BUFFER +/** @addtogroup odp_buffer * Buffer event metadata and operations. * @{ */ diff --git a/include/odp/api/spec/buffer_types.h b/include/odp/api/spec/buffer_types.h index 7b0e80584..7307e72f7 100644 --- a/include/odp/api/spec/buffer_types.h +++ b/include/odp/api/spec/buffer_types.h @@ -17,7 +17,7 @@ extern "C" { #endif -/** @addtogroup odp_buffer +/** @defgroup odp_buffer ODP BUFFER * @{ */ diff --git a/include/odp/api/spec/crypto.h b/include/odp/api/spec/crypto.h index 49bc29ee4..8b7d1df54 100644 --- a/include/odp/api/spec/crypto.h +++ b/include/odp/api/spec/crypto.h @@ -21,7 +21,7 @@ extern "C" { #endif -/** @defgroup odp_crypto ODP CRYPTO +/** @addtogroup odp_crypto * Data ciphering and authentication. * @{ */ diff --git a/include/odp/api/spec/crypto_types.h b/include/odp/api/spec/crypto_types.h index 31214c0e9..579022762 100644 --- a/include/odp/api/spec/crypto_types.h +++ b/include/odp/api/spec/crypto_types.h @@ -20,7 +20,7 @@ extern "C" { #endif -/** @addtogroup odp_crypto +/** @defgroup odp_crypto ODP CRYPTO * @{ */ diff --git a/include/odp/api/spec/debug.h b/include/odp/api/spec/debug.h index 0cf5179a8..4a3365a3e 100644 --- a/include/odp/api/spec/debug.h +++ b/include/odp/api/spec/debug.h @@ -15,6 +15,10 @@ extern "C" { #endif +/** @addtogroup odp_initialization + * @{ + */ + /** * @def ODP_STATIC_ASSERT * Compile time assertion macro. Fails compilation and outputs message 'msg' @@ -26,6 +30,10 @@ extern "C" { * @param msg Compile time error message to be displayed if cond is false */ +/** + * @} + */ + #ifdef __cplusplus } #endif diff --git a/include/odp/api/spec/event.h b/include/odp/api/spec/event.h index 220c955c3..69464125b 100644 --- a/include/odp/api/spec/event.h +++ b/include/odp/api/spec/event.h @@ -21,7 +21,7 @@ extern "C" { #include <odp/api/packet_types.h> #include <odp/api/pool_types.h> -/** @defgroup odp_event ODP EVENT +/** @addtogroup odp_event * Generic event metadata and operations. * @{ */ diff --git a/include/odp/api/spec/event_types.h b/include/odp/api/spec/event_types.h index 9df5e03ef..b967e2871 100644 --- a/include/odp/api/spec/event_types.h +++ b/include/odp/api/spec/event_types.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2015-2018 Linaro Limited - * Copyright (c) 2022 Nokia + * Copyright (c) 2022-2023 Nokia */ /** @@ -17,7 +17,7 @@ extern "C" { #endif -/** @addtogroup odp_event +/** @defgroup odp_event ODP EVENT * @{ */ @@ -59,6 +59,8 @@ extern "C" { * completion. * - ODP_EVENT_DMA_COMPL * - DMA completion event (odp_dma_compl_t) indicates that a DMA transfer has finished + * - ODP_EVENT_ML_COMPL + * - ML completion event (odp_ml_compl_t) indicates that an ML operation has finished */ /** @@ -92,6 +94,12 @@ extern "C" { * packet metadata. * - ODP_EVENT_NO_SUBTYPE * - An event type does not have any subtypes defined + * - ODP_EVENT_ML_COMPL_LOAD + * - ML completion event (odp_ml_compl_t) that contains results from a completed model load or + * unload operation. + * - ODP_EVENT_ML_COMPL_RUN + * - ML completion event (odp_ml_compl_t) that contains results from a completed model run + * operation. */ /** diff --git a/include/odp/api/spec/ipsec.h b/include/odp/api/spec/ipsec.h index b66e9a1ca..cdb18116b 100644 --- a/include/odp/api/spec/ipsec.h +++ b/include/odp/api/spec/ipsec.h @@ -23,7 +23,7 @@ extern "C" { #include <odp/api/packet_types.h> #include <odp/api/std_types.h> -/** @defgroup odp_ipsec ODP IPSEC +/** @addtogroup odp_ipsec * IPSEC protocol offload. * @{ */ diff --git a/include/odp/api/spec/ipsec_types.h b/include/odp/api/spec/ipsec_types.h index 00fd944f5..71b53a770 100644 --- a/include/odp/api/spec/ipsec_types.h +++ b/include/odp/api/spec/ipsec_types.h @@ -24,7 +24,7 @@ extern "C" { #include <odp/api/std_types.h> #include <odp/api/traffic_mngr.h> -/** @addtogroup odp_ipsec +/** @defgroup odp_ipsec ODP IPSEC * @{ */ diff --git a/include/odp/api/spec/ml.h b/include/odp/api/spec/ml.h new file mode 100644 index 000000000..1a7710ab3 --- /dev/null +++ b/include/odp/api/spec/ml.h @@ -0,0 +1,699 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021-2023 Nokia + * Copyright (c) 2021 Marvell + */ + +/** + * @file + * + * ODP Machine Learning (ML) offload + */ + +#ifndef ODP_API_SPEC_ML_H_ +#define ODP_API_SPEC_ML_H_ +#include <odp/visibility_begin.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/event_types.h> +#include <odp/api/ml_types.h> +#include <odp/api/pool_types.h> +#include <odp/api/std_types.h> + +/** + * @addtogroup odp_ml + * Machine Learning (ML) offload + * @{ + * + * <b> ML API call sequence </b> + * + * Before ML offload can be used, it must be configured with an odp_ml_config() call. An application + * fills in configuration parameters to describe its intended ML offload usage. The parameter + * values may help ODP implementation to optimize memory and other HW resource usage. The + * application may use odp_ml_capability() to check ML capabilities both before and after the + * configuration step. + * + * After configuration, an ML model binary is passed with other parameters to odp_ml_model_create() + * call which checks and prepares the model for usage. The application may use odp_ml_model_info(), + * odp_ml_model_input_info() and odp_ml_model_output_info() calls to check model input and output + * data formats. Before the application can use the model for inference, it loads the model with + * an odp_ml_model_load() or odp_ml_model_load_start() call. After a successful load, the + * application may use e.g. odp_ml_run() or odp_ml_run_start() to perform inferences. + * + * When all previously started inference operations are complete, application uses + * odp_ml_model_unload() or odp_ml_model_unload_start() to unload the model. After a successful + * unload, the model may be destroyed with an odp_ml_model_destroy() call, or loaded again. + * + * <b> Completion identifiers </b> + * + * Completion identifiers are used with ML operations in asynchronous poll mode + * (#ODP_ML_COMPL_MODE_POLL). Application declares the maximum identifier value it will + * use per model with odp_ml_model_param_t.max_compl_id parameter. It cannot exceed + * the implementation capability of odp_ml_capability_t.max_compl_id. Completion identifier + * values are model specific. The same value can be used simultaneously with two different + * models, but cannot be used simultaneously in two ML operations on the same model. A value may be + * reused for the next ML operation (on the same model) only after the previous operation is + * complete. Within those limitations, application may use/reuse completion identifier + * values from 0 to max_compl_id range freely. + */ + +/** + * Query ML capabilities + * + * Outputs ML capabilities on success. Use this capability call to check ML offload implementation + * limits and its support of various ML API features. When ML offload is not available, + * odp_ml_capability_t.max_models is zero. + * + * @param[out] capa Pointer to a capability structure for output + * + * @retval 0 on success + * @retval <0 on failure + */ +int odp_ml_capability(odp_ml_capability_t *capa); + +/** + * Initialize ML configuration parameters + * + * Initialize an odp_ml_config_t to its default values. + * + * @param[out] config Configuration structure to be initialized + */ +void odp_ml_config_init(odp_ml_config_t *config); + +/** + * Configure ML offload + * + * Initializes and configures ML offload according to the configuration parameters. This function + * must be called only once and before any ML resources are created. Use odp_ml_capability() to + * query configuration capabilities and odp_ml_config_init() to initialize configuration + * parameters into their default values. + * + * @param config ML configuration parameters + * + * @retval 0 on success + * @retval <0 on failure + */ +int odp_ml_config(const odp_ml_config_t *config); + +/** + * Initialize ML model parameters + * + * Initialize an odp_ml_model_param_t to its default values. + * + * @param[out] param Parameters structure to be initialized + */ +void odp_ml_model_param_init(odp_ml_model_param_t *param); + +/** + * Create an ML model + * + * Creates an ML model according to the parameters. Use odp_ml_model_param_init() to initialize + * parameters into their default values. The use of model name is optional. Unique names are not + * required. However, odp_ml_model_lookup() returns only a single matching model. Maximum name + * string length is #ODP_ML_MODEL_NAME_LEN. + * + * The call copies the model binary and prepares it for loading. Application may free memory + * buffers pointed by the parameters when the call returns. Use odp_ml_model_load() + * or odp_ml_model_load_start() to load the model. A model is ready for inference runs + * (see e.g. odp_ml_run()) after it has been loaded successfully. + * + * When model metadata misses some details of model input / output data format, user can + * pass those with odp_ml_model_param_t.extra_info. Some ODP implementations may define + * implementation specific extra parameters (e.g. hints about HW resource usage), user can pass + * those with odp_ml_model_param_t.extra_param when applicable. + * + * @param name Name of the model, or NULL + * @param param ML model parameters + * + * @return ML model handle on success + * @retval ODP_ML_MODEL_INVALID on failure + */ +odp_ml_model_t odp_ml_model_create(const char *name, const odp_ml_model_param_t *param); + +/** + * Destroy an ML model + * + * Destroys a model and releases the resources reserved for it. If the model has been loaded, it + * must be unloaded (see odp_ml_model_unload() or odp_ml_model_unload_start()) prior to calling + * this function. + * + * @param model ML model to be destroyed + * + * @retval 0 on success + * @retval <0 on failure + */ +int odp_ml_model_destroy(odp_ml_model_t model); + +/** + * Find a model by name + * + * @param name Name of the model + * + * @return Handle of the first matching ML model + * @retval ODP_ML_MODEL_INVALID Model could not be found + */ +odp_ml_model_t odp_ml_model_lookup(const char *name); + +/** + * Load ML model + * + * Loads ML model in synchronous mode. When the call returns, load is complete and the model is + * ready for inference requests. A loaded model must be unloaded before it can be destroyed. + * The same model can be loaded and unloaded multiple times before being destroyed. + * + * The call optionally outputs results. Use NULL as 'result' pointer if results are not required. + * + * Application should not try to keep loaded more than configured number of models + * (odp_ml_config_t.max_models_loaded). Check ML capabilities for maximum number of loaded + * models (odp_ml_capability_t.max_models_loaded) and support of load completion modes + * (odp_ml_capability_t.load). + * + * @param model ML model to be loaded + * @param[out] result Pointer to load result structure for output, or NULL + * + * @retval 0 Model load was successful + * @retval <0 on failure + */ +int odp_ml_model_load(odp_ml_model_t model, odp_ml_load_result_t *result); + +/** + * Start asynchronous model load + * + * Otherwise like odp_ml_model_load(), but loads the model asynchronously. A successful call + * requests the model to be loaded, but does not wait for load completion. Completion parameters + * are used to select if load completion is reported in poll (#ODP_ML_COMPL_MODE_POLL) or event + * (#ODP_ML_COMPL_MODE_EVENT) mode. For poll mode, odp_ml_model_load_status() is called to check + * for completion. For event mode, ML offload sends the completion event into the completion + * queue when the load is complete. Use odp_ml_compl_param_init() to initialize completion + * parameters into their default values. + * + * @param model ML model to be loaded + * @param compl_param Completion parameters for load + * + * @retval 0 Model load started successfully + * @retval <0 on failure + */ +int odp_ml_model_load_start(odp_ml_model_t model, const odp_ml_compl_param_t *compl_param); + +/** + * Check model load completion + * + * Checks if a previously started model load (in #ODP_ML_COMPL_MODE_POLL mode) has completed. + * The completion identifier value from load operation completion parameters + * (odp_ml_compl_param_t.compl_id) is passed as a parameter. It specifies the load operation to be + * checked. Initially 0 is returned for all configured (but unused) completion identifier values. + * An odp_ml_model_load_start() call clears the previous completion status of an identifier, and + * this function returns 0 while the load is in progress. When the load is successfully + * complete, >0 is returned. If the load completed with a failure, -1 is returned. The same + * value is returned until the next start operation that reuses the identifier (with the same + * model). The completion identifier may be reused only after >0 or -1 is returned. + * + * Optionally, outputs more detailed operation results into odp_ml_load_result_t structure. + * Use NULL as 'result' pointer if these results are not required. + * + * @param model ML model being loaded + * @param compl_id Completion identifier that was used in load start + * @param[out] result Pointer to load result structure for output, or NULL + * + * @retval >0 Model load was successful + * @retval 0 Model load has not finished + * @retval -1 Model load failed + * @retval <-1 Failed to read completion status (e.g. bad handle) + */ +int odp_ml_model_load_status(odp_ml_model_t model, uint32_t compl_id, odp_ml_load_result_t *result); + +/** + * Unload ML model + * + * Unloads ML model in synchronous mode. All previously started inference operations must have been + * completed before model unload is attempted. When the call returns, unload is complete and the + * model is ready to be destroyed or loaded again. + * + * The call optionally outputs results. Use NULL as 'result' pointer if results are not required. + * + * @param model ML model to be unloaded + * @param[out] result Pointer to load result structure for output, or NULL + * + * @retval 0 Model unload was successful + * @retval <0 on failure + */ +int odp_ml_model_unload(odp_ml_model_t model, odp_ml_load_result_t *result); + +/** + * Start asynchronous model unload + * + * Otherwise like odp_ml_model_unload(), but unloads the model asynchronously. A successful call + * requests the model to be unloaded, but does not wait for unload completion. Completion + * parameters are used to select if unload completion is reported in poll (#ODP_ML_COMPL_MODE_POLL) + * or event (#ODP_ML_COMPL_MODE_EVENT) mode. For poll mode, odp_ml_model_unload_status() is called + * to check for completion. For event mode, ML offload sends the completion event into the + * completion queue when the unload is complete. Use odp_ml_compl_param_init() to initialize + * completion parameters into their default values. + * + * @param model ML model to be unloaded + * @param compl_param Completion parameters for unload + * + * @retval 0 Model unload started successfully + * @retval <0 on failure + */ +int odp_ml_model_unload_start(odp_ml_model_t model, const odp_ml_compl_param_t *compl_param); + +/** + * Check model unload completion + * + * Checks if a previously started model unload (in #ODP_ML_COMPL_MODE_POLL mode) has completed. + * The completion identifier value from unload operation completion parameters + * (odp_ml_compl_param_t.compl_id) is passed as a parameter. It specifies the unload operation to be + * checked. Initially 0 is returned for all configured (but unused) completion identifier values. + * An odp_ml_model_unload_start() call clears the previous completion status of an identifier, and + * this function returns 0 while the unload is in progress. When the unload is successfully + * complete, >0 is returned. If the unload completed with a failure, -1 is returned. The same + * value is returned until the next start operation that reuses the identifier (with the same + * model). The completion identifier may be reused only after >0 or -1 is returned. + * + * Optionally, outputs more detailed operation results into odp_ml_load_result_t structure. + * Use NULL as 'result' pointer if these results are not required. + * + * @param model ML model being unloaded + * @param compl_id Completion identifier that was used in unload start + * @param[out] result Pointer to load result structure for output, or NULL + * + * @retval >0 Model unload was successful + * @retval 0 Model unload has not finished + * @retval -1 Model unload failed + * @retval <-1 Failed to read completion status (e.g. bad handle) + */ +int odp_ml_model_unload_status(odp_ml_model_t model, uint32_t compl_id, + odp_ml_load_result_t *result); + +/** + * Initialize model run parameters + * + * Initialize an odp_ml_run_param_t to its default values. + * + * @param[out] param Model run parameters structure to be initialized + */ +void odp_ml_run_param_init(odp_ml_run_param_t *param); + +/** + * Run the model in synchronous mode + * + * Performs an ML inference operation using the model and input data pointed by the data descriptor. + * A successful operation writes inference output data into memory buffers pointed by the data + * descriptor. Input/output data buffers are described as an array of segment descriptors. Each + * segment descriptor specifies a memory buffer used with only one model input/output. Multiple + * subsequent descriptors may be used to specify segmented data for the same input/output. + * When the model has multiple inputs/outputs, descriptor order in the array follows the model + * input/output order reported by odp_ml_model_input_info() and odp_ml_model_output_info() calls. + * All memory buffers for the first input/output are specified before any buffers for the second + * input/output, and so on. + * + * When some model inputs/outputs have #ODP_ML_SHAPE_BATCH shape type, the batch size is specified + * in run parameters (odp_ml_run_param_t.batch_size). The same batch size is used for all such + * inputs/outputs. Application may request additional operation results by setting 'result' pointer + * in run parameters. Use odp_ml_run_param_init() to initialize run parameters into their default + * values. Default run parameter values are used when 'param' is NULL. + * + * Returns 1 when model run completed successfully. Returns 0 when the operation was not performed + * due to ML offload resources being temporarily busy. Returns <0 on failure. + * + * @param model ML model to be run + * @param data Model input/output data descriptor + * @param param Model run parameters, or NULL + * + * @retval 1 Model run completed successfully + * @retval 0 Resources are busy and model was not run + * @retval <0 on failure + */ +int odp_ml_run(odp_ml_model_t model, const odp_ml_data_t *data, const odp_ml_run_param_t *param); + +/** + * Run the model multiple times in synchronous mode + * + * Otherwise like odp_ml_run(), but runs the model 'num' times with different input/output + * data buffers. Output data buffers of one ML inference operation must not overlap with + * input/output data buffers of another one. + * + * Returns number of model runs successfully completed. When return value is less than 'num', + * the remaining runs were not performed due to ML offload resources being temporarily busy. + * Returns <0 on failure. + * + * @param model ML model to be run + * @param data Array of model input/output data descriptors. The array has 'num' elements. + * @param param Array of model run parameters, or NULL. The array has 'num' elements + * when used. + * @param num Number of model runs to perform + * + * @return Number of model runs completed successfully (1 ... num) + * @retval 0 Resources are busy and model was not run + * @retval <0 on failure + */ +int odp_ml_run_multi(odp_ml_model_t model, const odp_ml_data_t data[], + const odp_ml_run_param_t param[], int num); + +/** + * Start model run in asynchronous mode + * + * Otherwise like odp_ml_run(), but runs the model asynchronously. A successful call + * requests the model to be run, but does not wait for run completion. Completion parameters + * select if run completion is reported in poll (#ODP_ML_COMPL_MODE_POLL) or event + * (#ODP_ML_COMPL_MODE_EVENT) mode. For poll mode, odp_ml_run_status() is called to check + * for completion. For event mode, ML offload sends the completion event into the completion queue + * when the run is complete. Use odp_ml_compl_param_init() to initialize completion parameters + * into their default values. + * + * Additional operation results (odp_ml_run_result_t) are available through the status call + * (odp_ml_run_status()) or completion event (odp_ml_compl_run_result()). Results are + * not output through the run parameters structure (i.e. odp_ml_run_param_t.result is ignored). + * + * Returns 1 when model run was started successfully. Returns 0 when model run was not started + * due to ML offload resources being temporarily busy. Returns <0 on failure. + * + * @param model ML model to be run + * @param data Model input/output data descriptor + * @param compl_param Completion parameters + * @param run_param Model run parameters, or NULL + * + * @retval 1 Model run started successfully + * @retval 0 Resources are busy and model run was not started + * @retval <0 on failure + */ +int odp_ml_run_start(odp_ml_model_t model, const odp_ml_data_t *data, + const odp_ml_compl_param_t *compl_param, const odp_ml_run_param_t *run_param); + +/** + * Start multiple model runs in asynchronous mode + * + * Otherwise like odp_ml_run_start(), but starts 'num' model runs with different input/output + * data buffers. Output data buffers of one ML inference operation must not overlap with + * input/output data buffers of another one. + * + * Returns number of model runs started successfully. When return value is less than 'num', + * the remaining runs were not started due to ML offload resources being temporarily busy. + * Returns <0 on failure. + * + * @param model ML model to be run + * @param data Array of model input/output data descriptors. The array has 'num' elements. + * @param compl_param Array of completion parameters. The array has 'num' elements. + * @param run_param Array of model run parameters, or NULL. The array has 'num' elements + * when used. + * @param num Number of model runs to start + * + * @return Number of model runs started successfully (1 ... num) + * @retval 0 Resources are busy and model runs were not started + * @retval <0 on failure + */ +int odp_ml_run_start_multi(odp_ml_model_t model, const odp_ml_data_t data[], + const odp_ml_compl_param_t compl_param[], + const odp_ml_run_param_t run_param[], int num); + +/** + * Check model run completion + * + * Checks if a previously started model run (in #ODP_ML_COMPL_MODE_POLL mode) has completed. + * The completion identifier value from run operation completion parameters + * (odp_ml_compl_param_t.compl_id) is passed as a parameter. It specifies the run operation to be + * checked. Initially 0 is returned for all configured (but unused) completion identifier values. + * An odp_ml_run_start() call clears the previous completion status of an identifier, and + * this function returns 0 while the run is in progress. When the run is successfully + * complete, >0 is returned. If the run completed with a failure, -1 is returned. The same + * value is returned until the next start operation that reuses the identifier (with the same + * model). The completion identifier may be reused only after >0 or -1 is returned. + * + * Optionally, outputs more detailed operation results into odp_ml_run_result_t structure. + * Use NULL as 'result' pointer if these results are not required. + * + * @param model ML model running + * @param compl_id Completion identifier that was used in run start + * @param[out] result Pointer to run result structure for output, or NULL + * + * @retval >0 Model run was successful + * @retval 0 Model run has not finished + * @retval -1 Model run failed + * @retval <-1 Failed to read completion status (e.g. bad handle) + */ +int odp_ml_run_status(odp_ml_model_t model, uint32_t compl_id, odp_ml_run_result_t *result); + +/** + * Initialize ML completion event pool parameters + * + * Initialize an odp_ml_compl_pool_param_t to its default values. + * + * @param[out] param Parameter structure to be initialized + */ +void odp_ml_compl_pool_param_init(odp_ml_compl_pool_param_t *param); + +/** + * Create ML completion event pool + * + * Creates a pool of ML completion events (ODP_EVENT_ML_COMPL). Pool type is ODP_POOL_ML_COMPL. + * The use of pool name is optional. Unique names are not required. However, odp_pool_lookup() + * returns only a single matching pool. Use odp_ml_compl_pool_param_init() to initialize pool + * parameters into their default values. Parameters values must not exceed pool capabilities + * (see odp_ml_compl_pool_capability_t). + * + * @param name Name of the pool or NULL. Maximum string length is #ODP_POOL_NAME_LEN. + * @param param Pool parameters + * + * @return Pool handle on success + * @retval ODP_POOL_INVALID on failure + */ +odp_pool_t odp_ml_compl_pool_create(const char *name, const odp_ml_compl_pool_param_t *param); + +/** + * Allocate ML completion event + * + * Allocates an ML completion event from a pool. The pool must have been created with + * odp_ml_compl_pool_create() call. All completion event metadata are set to their default values. + * + * @param pool ML completion event pool + * + * @return ML completion event handle + * @retval ODP_ML_COMPL_INVALID Completion event could not be allocated + */ +odp_ml_compl_t odp_ml_compl_alloc(odp_pool_t pool); + +/** + * Free ML completion event + * + * Frees an ML completion event into the pool it was allocated from. + * + * @param ml_compl ML completion event handle + */ +void odp_ml_compl_free(odp_ml_compl_t ml_compl); + +/** + * Check ML model run results from completion event + * + * Reads model run results from an ML completion event (ODP_EVENT_ML_COMPL). The event indicates + * completion of a previously started inference operation. Subtype of the completion event must be + * ODP_EVENT_ML_COMPL_RUN. Function return value indicates if the model run succeeded or failed. + * Additionally, outputs more detailed results into the provided odp_ml_run_result_t + * structure. Use NULL as 'result' pointer if those results are not required. + * + * @param ml_compl ML completion event (subtype ODP_EVENT_ML_COMPL_RUN) + * @param[out] result Pointer to ML run result structure for output, or NULL. + * + * @retval 0 Model run was successful + * @retval -1 Model run failed + * @retval <-1 Failed to read results from the event (e.g. bad handle) + */ +int odp_ml_compl_run_result(odp_ml_compl_t ml_compl, odp_ml_run_result_t *result); + +/** + * Check ML model load / unload results from completion event + * + * Reads model load / unload results from an ML completion event (ODP_EVENT_ML_COMPL). The event + * indicates completion of a previously started model load / unload operation. Subtype of the + * completion event must be ODP_EVENT_ML_COMPL_LOAD. Function return value indicates if the model + * load / unload succeeded or failed. Additionally, outputs more detailed results into the provided + * odp_ml_load_result_t structure. Use NULL as 'result' pointer if those results are not required. + * + * @param ml_compl ML completion event (subtype ODP_EVENT_ML_COMPL_LOAD) + * @param[out] result Pointer to model load / unload result structure for output, or NULL. + * + * @retval 0 Model load / unload was successful + * @retval -1 Model load / unload failed + * @retval <-1 Failed to read results from the event (e.g. bad handle) + */ +int odp_ml_compl_load_result(odp_ml_compl_t ml_compl, odp_ml_load_result_t *result); + +/** + * ML completion event user area + * + * Returns pointer to the user area associated with the completion event. Size of the area is + * fixed and defined in pool parameters. + * + * @param ml_compl ML completion event + * + * @return Pointer to the user area of the completion event + * @retval NULL The completion event does not have user area + */ +void *odp_ml_compl_user_area(odp_ml_compl_t ml_compl); + +/** + * Convert event to ML completion event + * + * Converts an ODP_EVENT_ML_COMPL type event to an ML completion event. + * + * @param event Event handle + * + * @return ML completion event handle + */ +odp_ml_compl_t odp_ml_compl_from_event(odp_event_t event); + +/** + * Convert ML completion event to event + * + * @param ml_compl ML completion event handle + * + * @return Event handle + */ +odp_event_t odp_ml_compl_to_event(odp_ml_compl_t ml_compl); + +/** + * Convert ML completion event handle to a uint64_t value for debugging + * + * @param ml_compl ML completion event handle to be converted + * + * @return uint64_t value that can be used for debugging (e.g. printed) + */ +uint64_t odp_ml_compl_to_u64(odp_ml_compl_t ml_compl); + +/** + * Initialize ML completion parameters + * + * Initialize an odp_ml_compl_param_t to its default values. + * + * @param[out] param Address of parameters structure to be initialized + */ +void odp_ml_compl_param_init(odp_ml_compl_param_t *param); + +/** + * Retrieve model information + * + * Retrieve information about the model. Model information includes e.g. version numbers and + * number of model inputs/outputs. Information about each input and output can be retrieved with + * odp_ml_model_input_info() and odp_ml_model_output_info() calls. + * + * @param model ML model handle + * @param[out] info Pointer to model information structure for output + * + * @retval 0 on success + * @retval <0 on failure + */ +int odp_ml_model_info(odp_ml_model_t model, odp_ml_model_info_t *info); + +/** + * Retrieve model input information + * + * Writes information about each model input into the array. If there are more inputs than array + * elements, writes only 'num' elements. Returns the number of model inputs on success, and zero on + * failure. When 'num' is zero, ignores value of 'info' and returns normally. + * + * @param model ML model handle + * @param[out] info Pointer to model input information array for output + * @param num Number of elements in the array + * + * @return Number of model inputs + * @retval 0 on failure + */ +uint32_t odp_ml_model_input_info(odp_ml_model_t model, odp_ml_input_info_t info[], uint32_t num); + +/** + * Retrieve model output information + * + * Writes information about each model output into the array. If there are more outputs than array + * elements, writes only 'num' elements. Returns the number of model outputs on success, and zero on + * failure. When 'num' is zero, ignores value of 'info' and returns normally. + * + * @param model ML model handle + * @param[out] info Pointer to model output information array for output + * @param num Number of elements in the array + * + * @return Number of model outputs + * @retval 0 on failure + */ +uint32_t odp_ml_model_output_info(odp_ml_model_t model, odp_ml_output_info_t info[], uint32_t num); + +/** + * Convert ML model handle to a uint64_t value for debugging + * + * @param model ML model handle + * + * @return uint64_t value that can be used for debugging (e.g. printed) + */ +uint64_t odp_ml_model_to_u64(odp_ml_model_t model); + +/** + * Print debug information about the model. + * + * Print implementation defined information about ML model to the ODP log. The information is + * intended to be used for debugging. + + * @param model ML model handle + */ +void odp_ml_model_print(odp_ml_model_t model); + +/** + * Print ML debug information + * + * Print implementation defined information about ML offload to the ODP log. The information is + * intended to be used for debugging. + */ +void odp_ml_print(void); + +/** + * Extra statistics counter information + * + * Returns the number of extra statistics counters supported by the ML offload, and outputs + * information (e.g. name) about those. Counters are implementation specific and maintained + * per model. Statistics counting is enabled through model create parameters. + * + * When 'info' pointer is not NULL, fills in up to 'num' counter info structures. If the return + * value is larger than 'num', there are more counters than the function was allowed to output. + * If the return value N is less than 'num' (on success), only first N structures have been written. + * + * Info array elements are filled in the same order than odp_ml_model_extra_stats() outputs + * counter values. + * + * @param model ML model + * @param[out] info Pointer to extra statistics counter information array for output. + * NULL may be used to query only the number of counters. + * @param num Number of elements in the array + * + * @return Number of extra statistics counters + * @retval <0 on failure + */ +int odp_ml_model_extra_stat_info(odp_ml_model_t model, odp_ml_extra_stat_info_t info[], int num); + +/** + * Read extra statistics counter values + * + * Reads extra statistics counter values and returns the number of supported counters. Outputs + * up to 'num' counter values into 'stats' array. If the return value is larger than 'num', + * there are more counters than the function was allowed to output. If the return value N is less + * than 'num' (on success), only first N counters have been written. The order of counters in + * the array matches the counter information array order on odp_ml_model_extra_stat_info() output. + * + * @param model ML model + * @param[out] stats Pointer to extra statistics counter array for output + * @param num Number of elements in the array + * + * @return Number of extra statistics counters + * @retval <0 on failure + */ +int odp_ml_model_extra_stats(odp_ml_model_t model, uint64_t stats[], int num); + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#include <odp/visibility_end.h> +#endif diff --git a/include/odp/api/spec/ml_quantize.h b/include/odp/api/spec/ml_quantize.h new file mode 100644 index 000000000..25565ef27 --- /dev/null +++ b/include/odp/api/spec/ml_quantize.h @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +/** + * @file + * + * ODP Machine Learning (ML) quantization functions + */ + +#ifndef ODP_API_SPEC_ML_QUANTIZE_H_ +#define ODP_API_SPEC_ML_QUANTIZE_H_ +#include <odp/visibility_begin.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/std_types.h> + +/** @addtogroup odp_ml + * @{ + */ + +/** + * Quantize 32-bit float to uint8_t + * + * Quantizes 'num' 32-bit floating point values to uint8_t values using the provided scale and + * zero point. + * + * dst_u8 = (src_fp32 / scale) + zerop + * + * @param[out] dst_u8 Destination address for quantized values + * @param src_fp32 Source address of values to be quantized + * @param num Number of values + * @param scale Scale for quantization + * @param zerop Zero point for quantization + */ +void odp_ml_fp32_to_uint8(uint8_t *dst_u8, const float *src_fp32, uint32_t num, + float scale, uint8_t zerop); + +/** + * De-quantize 32-bit float from uint8_t + * + * De-quantizes 'num' 32-bit floating point values from uint8_t values using the provided scale and + * zero point. + * + * dst_fp32 = (src_u8 - zerop) * scale + * + * @param[out] dst_fp32 Destination address for de-quantized values + * @param src_u8 Source address of values to be de-quantized + * @param num Number of values + * @param scale Scale for de-quantization + * @param zerop Zero point for de-quantization + */ +void odp_ml_fp32_from_uint8(float *dst_fp32, const uint8_t *src_u8, uint32_t num, + float scale, uint8_t zerop); + +/** + * Quantize 32-bit float to int8_t + * + * Quantizes 'num' 32-bit floating point values to int8_t values using the provided scale and + * zero point. + * + * dst_i8 = (src_fp32 / scale) + zerop + * + * @param[out] dst_i8 Destination address for quantized values + * @param src_fp32 Source address of values to be quantized + * @param num Number of values + * @param scale Scale for quantization + * @param zerop Zero point for quantization + */ +void odp_ml_fp32_to_int8(int8_t *dst_i8, const float *src_fp32, uint32_t num, float scale, + int8_t zerop); + +/** + * De-quantize 32-bit float from int8_t + * + * De-quantizes 'num' 32-bit floating point values from int8_t values using the provided scale and + * zero point. + * + * dst_fp32 = (src_i8 - zerop) * scale + * + * @param[out] dst_fp32 Destination address for de-quantized values + * @param src_i8 Source address of values to be de-quantized + * @param num Number of values + * @param scale Scale for de-quantization + * @param zerop Zero point for de-quantization + */ +void odp_ml_fp32_from_int8(float *dst_fp32, const int8_t *src_i8, uint32_t num, float scale, + int8_t zerop); + +/** + * Quantize 32-bit float to 16-bit float + * + * Quantizes 'num' 32-bit floating point values to 16-bit floating point values. + * + * @param[out] dst_fp16 Destination address for quantized values + * @param src_fp32 Source address of values to be quantized + * @param num Number of values + */ +void odp_ml_fp32_to_fp16(uint16_t *dst_fp16, const float *src_fp32, uint32_t num); + +/** + * De-quantize 32-bit float from 16-bit float + * + * De-quantizes 'num' 32-bit floating point values from 16-bit floating point values. + * + * @param[out] dst_fp32 Destination address for de-quantized values + * @param src_fp16 Source address of values to be de-quantized + * @param num Number of values + */ +void odp_ml_fp32_from_fp16(float *dst_fp32, const uint16_t *src_fp16, uint32_t num); + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#include <odp/visibility_end.h> +#endif diff --git a/include/odp/api/spec/ml_types.h b/include/odp/api/spec/ml_types.h new file mode 100644 index 000000000..2b8f9d6c8 --- /dev/null +++ b/include/odp/api/spec/ml_types.h @@ -0,0 +1,873 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021-2023 Nokia + * Copyright (c) 2021 Marvell + */ + +/** + * @file + * + * ODP Machine Learning (ML) types + */ + +#ifndef ODP_API_SPEC_ML_TYPES_H_ +#define ODP_API_SPEC_ML_TYPES_H_ +#include <odp/visibility_begin.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/event_types.h> +#include <odp/api/queue_types.h> +#include <odp/api/std_types.h> + +/** @defgroup odp_ml ODP ML + * @{ + */ + +/** + * @typedef odp_ml_model_t + * ODP ML model handle + */ + +/** + * @def ODP_ML_MODEL_INVALID + * Invalid ML model + */ + +/** + * @typedef odp_ml_compl_t + * ML completion event + */ + +/** + * @def ODP_ML_COMPL_INVALID + * Invalid ML completion event + */ + +/** + * @def ODP_ML_MODEL_NAME_LEN + * Maximum length of model name in chars (including null char) + */ + +/** + * @def ODP_ML_MODEL_IO_NAME_LEN + * Maximum length of model input/output name in chars (including null char) + */ + +/** + * @def ODP_ML_SHAPE_NAME_LEN + * Maximum length of data dimension name in chars (including null char) + */ + +/** + * @def ODP_ML_EXTRA_STAT_NAME_LEN + * Maximum length of extra statistics counter name in chars (including null char) + */ + +/** + * @typedef odp_ml_model_extra_param_t + * ODP implementation specific extra parameters for model creation + */ + +/** Maximum number of dimensions in input / output data shape */ +#define ODP_ML_MAX_DIMS 8 + +/** Dimension size is dynamic */ +#define ODP_ML_DIM_DYNAMIC 0 + +/** Synchronous operation */ +#define ODP_ML_COMPL_MODE_SYNC 0x1u + +/** + * Asynchronous poll mode operation + * + * A function call starts an operation and a status function call indicates when + * the operation has finished. + */ +#define ODP_ML_COMPL_MODE_POLL 0x2u + +/** + * Asynchronous event mode operation + * + * A function call starts an operation and a completion event indicates when + * the operation has finished. + */ +#define ODP_ML_COMPL_MODE_EVENT 0x4u + +/** ML completion mode */ +typedef uint32_t odp_ml_compl_mode_t; + +/** + * ML completion event pool capabilities + * + * Pool statistics are not supported with ML completion event pools. + */ +typedef struct odp_ml_compl_pool_capability_t { + /** + * Maximum number of ML completion event pools + * + * See odp_pool_capability_t.max_pools for maximum number of pools of any type. It + * includes also ML completion event pools. + */ + uint32_t max_pools; + + /** Maximum number of ML completion events in a pool */ + uint32_t max_num; + + /** Maximum user area size in bytes */ + uint32_t max_uarea_size; + + /** User area persistence + * + * See buf.uarea_persistence of odp_pool_capability_t for details + * (odp_pool_capability_t.uarea_persistence). + */ + odp_bool_t uarea_persistence; + + /** Maximum size of local thread cache */ + uint32_t max_cache_size; + + /** Minimum size of local thread cache */ + uint32_t min_cache_size; + +} odp_ml_compl_pool_capability_t; + +/** + * ML completion event pool parameters + * + * Use odp_ml_compl_pool_param_init() to initialize the structure to its default values. + */ +typedef struct odp_ml_compl_pool_param_t { + /** + * Number of ML completion events in the pool + * + * The maximum supported value is defined by ML pool capability 'max_num' + * (odp_ml_compl_pool_capability_t.max_num). + */ + uint32_t num; + + /** + * User area size in bytes + * + * The maximum supported value is defined by ML pool capability 'max_uarea_size'. + * Specify as zero if no user area is needed. The default value is 0. + */ + uint32_t uarea_size; + + /** Parameters for user area initialization */ + struct { + /** See uarea_init.init_fn of odp_pool_param_t for details + * (odp_pool_param_t.init_fn). Function is called during + * odp_ml_compl_pool_create(). The default value is NULL. */ + void (*init_fn)(void *uarea, uint32_t size, void *args, uint32_t index); + + /** See uarea_init.args of odp_pool_param_t for details + * (odp_pool_param_t.args). The default value is NULL. */ + void *args; + + } uarea_init; + + /** + * Maximum number of events cached locally per thread + * + * See odp_pool_param_t.cache_size documentation for details. Valid values range from + * 'min_cache_size' to 'max_cache_size' ML pool capability. The default value is + * implementation specific and set by odp_ml_compl_pool_param_init(). + */ + uint32_t cache_size; + +} odp_ml_compl_pool_param_t; + +/** Machine learning capabilities */ +typedef struct odp_ml_capability_t { + /** Maximum number of models + * + * Maximum number of models that can be created simultaneously. The value is zero when + * ML offload is not available. */ + uint32_t max_models; + + /** Maximum number of models that can be loaded simultaneously */ + uint32_t max_models_loaded; + + /** Maximum model size in bytes */ + uint64_t max_model_size; + + /** Maximum completion identifier value */ + uint32_t max_compl_id; + + /** Maximum number of model inputs */ + uint32_t max_inputs; + + /** Maximum number of model outputs */ + uint32_t max_outputs; + + /** + * Maximum number of data segments per model input + * + * Segmented input data is not supported when 1. + */ + uint32_t max_segs_per_input; + + /** + * Maximum number of data segments per model output + * + * Segmented output data is not supported when 1. + */ + uint32_t max_segs_per_output; + + /** + * Minimum input data alignment in bytes + * + * For each model input, the first data segment must start at this or a higher power of two + * memory alignment in bytes. The value is 1 when there is no alignment requirement. + */ + uint32_t min_input_align; + + /** + * Minimum output data alignment in bytes + * + * For each model output, the first data segment must start at this or a higher power of two + * memory alignment in bytes. The value is 1 when there is no alignment requirement. + */ + uint32_t min_output_align; + + /** + * Input data packing + * + * 0: Data packing is not required. + * 1: Data for all model inputs must be continuous in memory. The memory block starts with + * data for the first input and continues through all inputs in-order and without gaps + * between inputs. The minimum alignment requirement (min_input_align) applies only for + * the first input. + */ + odp_bool_t packed_input_data; + + /** + * Output data packing + * + * 0: Data packing is not required. + * 1: Data buffer space for all model outputs must be continuous in memory. The memory + * block starts with buffer space for the first output and continues through all outputs + * in-order and without gaps between outputs. The minimum alignment requirement + * (min_output_align) applies only for the first output. + */ + odp_bool_t packed_output_data; + + /** Model load / unload capabilities */ + struct { + /** + * Supported completion modes for model load / unload operations + * + * Mask of supported completion modes. Each supported mode has the corresponding + * flag (e.g. #ODP_ML_COMPL_MODE_SYNC) set in the mask. + */ + odp_ml_compl_mode_t compl_mode_mask; + + /** + * Support of model load / unload completion into plain queues + * + * Specifies if plain queues are supported as destination queues for + * load / unload completion events (#ODP_ML_COMPL_MODE_EVENT). + * + * 0: Plain queues are not supported as completion queues + * 1: Plain queues are supported as completion queues + */ + odp_bool_t compl_queue_plain; + + /** + * Support of model load / unload completion into scheduled queues + * + * Specifies if scheduled queues are supported as destination queues for + * load / unload completion events (#ODP_ML_COMPL_MODE_EVENT). + * + * 0: Scheduled queues are not supported as completion queues + * 1: Scheduled queues are supported as completion queues + */ + odp_bool_t compl_queue_sched; + + } load; + + /** Model run capabilities */ + struct { + /** + * Supported completion modes for model run operations + * + * Mask of supported completion modes. Each supported mode has the corresponding + * flag (e.g. #ODP_ML_COMPL_MODE_SYNC) set in the mask. + */ + odp_ml_compl_mode_t compl_mode_mask; + + /** + * Support of model run completion into plain queues + * + * Specifies if plain queues are supported as destination queues for + * run completion events (#ODP_ML_COMPL_MODE_EVENT). + * + * 0: Plain queues are not supported as completion queues + * 1: Plain queues are supported as completion queues + */ + odp_bool_t compl_queue_plain; + + /** + * Support of model run completion into scheduled queues + * + * Specifies if scheduled queues are supported as destination queues for + * run completion events (#ODP_ML_COMPL_MODE_EVENT). + * + * 0: Scheduled queues are not supported as completion queues + * 1: Scheduled queues are supported as completion queues + */ + odp_bool_t compl_queue_sched; + + } run; + + /** ML completion event pool capabilities */ + odp_ml_compl_pool_capability_t pool; + +} odp_ml_capability_t; + +/** Machine learning configuration parameters */ +typedef struct odp_ml_config_t { + /** + * Maximum number of models + * + * Application may create and use this many models simultaneously. The default value is 1. + */ + uint32_t max_models_created; + + /** + * Maximum number of models loaded + * + * Maximum number of models that the application will keep loaded simultaneously. + * The default value is 1. + */ + uint32_t max_models_loaded; + + /** + * Maximum model binary size in bytes + * + * All model binaries application will pass to odp_ml_model_create() are this size or + * smaller. + */ + uint64_t max_model_size; + + /** + * Load / unload completion modes + * + * Mask of completion modes that application will use with model load/unload operations. + * Multiple modes may be selected, but it is implementation specific if some combinations + * are not supported. In case of an unsupported combination odp_ml_config() returns + * failure. Check odp_ml_capability_t.load for supported modes. The default value is 0. + */ + odp_ml_compl_mode_t load_mode_mask; + + /** + * Run completion modes + * + * Mask of completion modes that application will use with model run operations. + * Multiple modes may be selected, but it is implementation specific if some combinations + * are not supported. In case of an unsupported combination odp_ml_config() returns + * failure. Check odp_ml_capability_t.run for supported modes. The default value is 0. + */ + odp_ml_compl_mode_t run_mode_mask; + +} odp_ml_config_t; + +/** Model input / output data type enumeration */ +typedef enum { + /** Data type is not defined */ + ODP_ML_DATA_TYPE_NONE = 0, + + /** 8-bit integer */ + ODP_ML_DATA_TYPE_INT8, + + /** 8-bit unsigned integer */ + ODP_ML_DATA_TYPE_UINT8, + + /** 16-bit integer */ + ODP_ML_DATA_TYPE_INT16, + + /** 16-bit unsigned integer */ + ODP_ML_DATA_TYPE_UINT16, + + /** 24-bit integer */ + ODP_ML_DATA_TYPE_INT24, + + /** 24-bit unsigned integer */ + ODP_ML_DATA_TYPE_UINT24, + + /** 32-bit integer */ + ODP_ML_DATA_TYPE_INT32, + + /** 32-bit unsigned integer */ + ODP_ML_DATA_TYPE_UINT32, + + /** 64-bit integer */ + ODP_ML_DATA_TYPE_INT64, + + /** 64-bit unsigned integer */ + ODP_ML_DATA_TYPE_UINT64, + + /** 16-bit floating point number */ + ODP_ML_DATA_TYPE_FP16, + + /** 16-bit brain floating point (bfloat16) number */ + ODP_ML_DATA_TYPE_BFP16, + + /** 32-bit floating point number */ + ODP_ML_DATA_TYPE_FP32, + + /** 64-bit floating point number */ + ODP_ML_DATA_TYPE_FP64, + +} odp_ml_data_type_t; + +/** Model input / output data shape type */ +typedef enum { + /** Type of shape is not defined */ + ODP_ML_SHAPE_NONE = 0, + + /** Static shape of data + * + * Shape is static when all dimensions have fixed sizes. + */ + ODP_ML_SHAPE_STATIC, + + /** Dynamic batch size + * + * Shape that has only one dynamic dimension, and the dimension is used as batch size of + * input / output data. The same batch size is applied for all inputs and outputs of + * the model. + */ + ODP_ML_SHAPE_BATCH, + +} odp_ml_shape_type_t; + +/** Model input / output data shape information */ +typedef struct odp_ml_shape_info_t { + /** Shape type */ + odp_ml_shape_type_t type; + + /** Number of dimensions + * + * Number of input / output data dimensions. When zero, the model does not have + * dimension information available. ODP API supports in maximum #ODP_ML_MAX_DIMS + * dimensions. + */ + uint32_t num_dim; + + /** Dimension sizes + * + * Number of data values in each ('num_dim') dimension. Type of the data is defined by + * odp_ml_data_type_t enumeration. Depending on the shape type, some dimensions may have + * dynamic size which is denoted with #ODP_ML_DIM_DYNAMIC value. When shape type is + * #ODP_ML_SHAPE_BATCH, the shape has one dynamic dimension which is used as the batch + * size. + * + * For example, a static (#ODP_ML_SHAPE_STATIC) NCHW tensor could be presented as: + * + * num_dim = 4; + * dim[0] = 1; // no batching, N = 1 + * dim[1] = 3; // 3 color channels + * dim[2] = 720; // height 720 pixels + * dim[3] = 1280; // width 1280 pixels + * + * ... and with dynamic batch size (#ODP_ML_SHAPE_BATCH): + * + * num_dim = 4; + * dim[0] = ODP_ML_DIM_DYNAMIC; // dynamic in range: dim_min[0] ... dim_max[0] + * dim[1] = 3; + * dim[2] = 720; + * dim[3] = 1280; + */ + uint32_t dim[ODP_ML_MAX_DIMS]; + + /** Minimum dimension sizes + * + * Defines the minimum value for each dynamic size (#ODP_ML_DIM_DYNAMIC) in dim[] array. + * Zero is used when the minimum value is unknown. When dimension size is static, the + * value is equal to dim[] array value. + */ + uint32_t dim_min[ODP_ML_MAX_DIMS]; + + /** Maximum dimension sizes + * + * Defines the maximum value for each dynamic size (#ODP_ML_DIM_DYNAMIC) in dim[] array. + * Zero is used when the maximum value is unknown. When dimension size is static, the + * value is equal to dim[] array value. + */ + uint32_t dim_max[ODP_ML_MAX_DIMS]; + + /** Dimension name + * + * Name of each ('num_dim') dimension as a null terminated string. Null string is used if + * a dimension does not have a name. Maximum string length is #ODP_ML_SHAPE_NAME_LEN + * including the null character. + * + * For example, an NCHW tensor could have dimensions named as: + * dim_name = {"N", "C", "H", "W"} + */ + char dim_name[ODP_ML_MAX_DIMS][ODP_ML_SHAPE_NAME_LEN]; + +} odp_ml_shape_info_t; + +/** Model input information */ +typedef struct odp_ml_input_info_t { + /** Model input name */ + char name[ODP_ML_MODEL_IO_NAME_LEN]; + + /** Model input data type */ + odp_ml_data_type_t data_type; + + /** Size of model input data type in bytes */ + uint32_t data_type_size; + + /** Model input data shape */ + odp_ml_shape_info_t shape; + +} odp_ml_input_info_t; + +/** Model output information */ +typedef struct odp_ml_output_info_t { + /** Model output name */ + char name[ODP_ML_MODEL_IO_NAME_LEN]; + + /** Model output data type */ + odp_ml_data_type_t data_type; + + /** Size of model output data type in bytes */ + uint32_t data_type_size; + + /** Model output data shape */ + odp_ml_shape_info_t shape; + +} odp_ml_output_info_t; + +/** Model information */ +typedef struct odp_ml_model_info_t { + /** Model name */ + char name[ODP_ML_MODEL_NAME_LEN]; + + /** + * Model version number + * + * Version number of the model binary. The number changes when the model is modified + * in any way. + */ + uint64_t model_version; + + /** + * Model interface version number + * + * The model interface version number changes only when model input or output data + * format is modified. Data formats are the same for two model versions that have + * the same interface version number. + */ + uint64_t interface_version; + + /** Model index assigned by the implementation */ + uint32_t index; + + /** Number of model inputs */ + uint32_t num_inputs; + + /** Number of model outputs */ + uint32_t num_outputs; + +} odp_ml_model_info_t; + +/** + * Model input / output data format + */ +typedef struct odp_ml_data_format_t { + /** Model input / output data type */ + odp_ml_data_type_t data_type; + + /** Size of data type in bytes */ + uint32_t data_type_size; + + /** Model input / output data shape */ + odp_ml_shape_info_t shape; + +} odp_ml_data_format_t; + +/** + * Machine learning model parameters + * + * Use odp_ml_model_param_init() to initialize the structure to its default values. + */ +typedef struct odp_ml_model_param_t { + /** + * Model binary + * + * Points to model binary stored into a memory buffer. Model format is + * implementation specific. */ + void *model; + + /** Size of the model binary in bytes */ + uint64_t size; + + /** + * Maximum completion identifier value + * + * When application uses asynchronous poll mode (#ODP_ML_COMPL_MODE_POLL) operations with + * the model, it will choose completion identifier values between 0 and this value. Valid + * values range from 0 to max_compl_id capability. The default value is zero. + */ + uint32_t max_compl_id; + + /** + * Enable / disable extra statistics counters + * + * Extra statistics may be read with odp_ml_model_extra_stats() when enabled. Statistics + * are disabled by default. + */ + odp_bool_t extra_stat_enable; + + /** + * Extra model information + * + * When model metadata misses some details of model input / output data format, user can + * pass those with this structure. When 'num_inputs' / 'num_outputs' is non-zero, data + * format of all model inputs / outputs are overridden by the provided values. Values are + * ignored when 'num_inputs' / 'num_outputs' is zero. + */ + struct { + /** + * Number of model inputs + * + * Number of model inputs and elements in 'input_format' array. When non-zero, + * the value must match the number of inputs defined in model metadata. The default + * value is 0. + */ + uint32_t num_inputs; + + /** + * Number of model outputs + * + * Number of model outputs and elements in 'output_format' array. When non-zero, + * the value must match the number of outputs defined in model metadata. The default + * value is 0. + */ + uint32_t num_outputs; + + /** + * Model input data format array + * + * Points to an array of data formats. The array has 'num_inputs' elements. Inputs + * are defined in the same order they are listed in model metadata. + * An odp_ml_model_create() call copies these values. The default value is NULL. + */ + const odp_ml_data_format_t *input_format; + + /** + * Model output data format array + * + * Points to an array of data formats. The array has 'num_outputs' elements. Outputs + * are defined in the same order they are listed in model metadata. + * An odp_ml_model_create() call copies these values. The default value is NULL. + */ + const odp_ml_data_format_t *output_format; + + } extra_info; + + /** + * ODP implementation specific extra parameters + * + * See ODP implementation documentation for details about extra parameter usage. For + * example, extra parameters may give hints about HW resource usage with the model to be + * created. An odp_ml_model_create() call copies these parameter values. When NULL, all + * extra parameters are set to their default values. The default value is NULL. + */ + const odp_ml_model_extra_param_t *extra_param; + +} odp_ml_model_param_t; + +/** Results of model run operation */ +typedef struct odp_ml_run_result_t { + /** Model run error code + * + * Zero when model run completed successfully. Otherwise, error code contains + * an implementation specific value. + */ + uint64_t error_code; + + /** User context pointer value from odp_ml_compl_param_t */ + void *user_ptr; + +} odp_ml_run_result_t; + +/** Result of model load / unload operation */ +typedef struct odp_ml_load_result_t { + /** Model load / unload error code + * + * Zero when model load / unload completed successfully. Otherwise, error code contains + * an implementation specific value. + */ + uint64_t error_code; + + /** User context pointer value from odp_ml_compl_param_t */ + void *user_ptr; + +} odp_ml_load_result_t; + +/** + * ML completion parameters + * + * Use odp_ml_compl_param_init() to initialize the structure to its default values. + */ +typedef struct odp_ml_compl_param_t { + /** + * Completion mode + * + * The selected completion mode defines which other parameters are used. When + * #ODP_ML_COMPL_MODE_EVENT mode is selected, 'event' and 'queue' must have valid values + * but value of 'compl_id' is ignored, or vice versa when #ODP_ML_COMPL_MODE_POLL mode is + * selected. + */ + odp_ml_compl_mode_t mode; + + /** + * Completion event + * + * Event to be enqueued by ML offload to the completion queue when ML operation + * is complete. Event type must be ODP_EVENT_ML_COMPL. ML offload sets the subtype of + * the event to ODP_EVENT_ML_COMPL_LOAD or ODP_EVENT_ML_COMPL_RUN based on the completed + * operation. + */ + odp_event_t event; + + /** + * Completion queue + * + * Destination queue for the completion event. + */ + odp_queue_t queue; + + /** + * Completion identifier + * + * When completion mode is #ODP_ML_COMPL_MODE_POLL, ML operation completion status is + * reported through this completion identifier. The value passed here is used in + * a following status call to check model load, unload, or inference completion + * (see e.g. odp_ml_model_load_status()). + * + * Application selects a value between 0 and max_compl_id defined in model creation + * parameters (see odp_ml_model_param_t). Only single ML operation (per model) may be + * started with the same identifier value at a time. A value may be reused for the next + * ML operation only after the previous operation is complete. + */ + uint32_t compl_id; + + /** + * User defined context pointer + * + * ODP implementation does not refer to the pointer, but just copies it to the result. + * For example, application may use this pointer to link a received completion event + * to the originating model run request and its input / output data. The default value + * is NULL. + */ + void *user_ptr; + +} odp_ml_compl_param_t; + +/** Model input / output data segment */ +typedef struct odp_ml_data_seg_t { + /** Segment start address */ + void *addr; + + /** Segment size in bytes */ + uint64_t size; + +} odp_ml_data_seg_t; + +/** Model input / output data for a model inference run */ +typedef struct odp_ml_data_t { + /** + * Number of input data segments + * + * Number of elements in 'input_seg' array (at least one per input). + */ + uint32_t num_input_seg; + + /** + * Number of output data segments + * + * Number of elements in 'output_seg' array (at least one per output). + */ + uint32_t num_output_seg; + + /** + * Model input data segments + * + * Points to an array of data segment descriptors for model input data. Each segment + * (odp_ml_data_seg_t) specifies data for one input only. Multiple consecutive segments may + * be used to specify data for the same input. Sum of those segment sizes must match data + * size of the input. Inputs are defined in the same order which odp_ml_model_input_info() + * reports those. + * + * Input data segments may overlap in memory. + */ + odp_ml_data_seg_t *input_seg; + + /** + * Model output data segments + * + * Points to an array of data segment descriptors for model output data. Each segment + * (odp_ml_data_seg_t) specifies data buffer space for one output only. Multiple + * consecutive segments may be used to specify buffer space for the same output. + * Sum of those segment sizes must match data size of the output. Outputs are defined + * in the same order which odp_ml_model_output_info() reports those. + * + * An output data segment must not overlap with any other (input or output) segment + * in memory. + */ + odp_ml_data_seg_t *output_seg; + +} odp_ml_data_t; + +/** + * Parameters for model run + * + * Use odp_ml_run_param_init() to initialize the structure to its default values. + */ +typedef struct odp_ml_run_param_t { + /** + * Batch size + * + * Batch size for all model inputs and outputs that have #ODP_ML_SHAPE_BATCH shape type. + * The default value is 0. + */ + uint32_t batch_size; + + /** + * Model run results + * + * Points to a result structure for model run result output. Results are output only + * in synchronous mode (#ODP_ML_COMPL_MODE_SYNC). The pointer value is ignored in + * asynchronous modes. Use NULL when results are not required. The default value is NULL. + */ + odp_ml_run_result_t *result; + +} odp_ml_run_param_t; + +/** + * ML extra statistics counter information + */ +typedef struct odp_ml_extra_stat_info_t { + /** Name of the statistics counter */ + char name[ODP_ML_EXTRA_STAT_NAME_LEN]; + +} odp_ml_extra_stat_info_t; + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#include <odp/visibility_end.h> +#endif diff --git a/include/odp/api/spec/packet.h b/include/odp/api/spec/packet.h index ca22280ec..7f6c732ee 100644 --- a/include/odp/api/spec/packet.h +++ b/include/odp/api/spec/packet.h @@ -25,7 +25,7 @@ extern "C" { #include <odp/api/std_types.h> #include <odp/api/time_types.h> -/** @defgroup odp_packet ODP PACKET +/** @addtogroup odp_packet * Packet event metadata and operations. * @{ */ diff --git a/include/odp/api/spec/packet_io.h b/include/odp/api/spec/packet_io.h index cfb463c39..a83617f7c 100644 --- a/include/odp/api/spec/packet_io.h +++ b/include/odp/api/spec/packet_io.h @@ -25,7 +25,7 @@ extern "C" { #include <odp/api/reassembly.h> #include <odp/api/time_types.h> -/** @defgroup odp_packet_io ODP PACKET IO +/** @addtogroup odp_packet_io * Packet IO interfaces. * * Packet IO is the Ingress and Egress interface to ODP processing. It diff --git a/include/odp/api/spec/packet_io_types.h b/include/odp/api/spec/packet_io_types.h index 6b80611ec..9e56e087a 100644 --- a/include/odp/api/spec/packet_io_types.h +++ b/include/odp/api/spec/packet_io_types.h @@ -24,7 +24,7 @@ extern "C" { #include <odp/api/reassembly.h> #include <odp/api/std_types.h> -/** @addtogroup odp_packet_io +/** @defgroup odp_packet_io ODP PACKET IO * @{ */ diff --git a/include/odp/api/spec/packet_types.h b/include/odp/api/spec/packet_types.h index ee62de4ff..b9d55abde 100644 --- a/include/odp/api/spec/packet_types.h +++ b/include/odp/api/spec/packet_types.h @@ -20,7 +20,7 @@ extern "C" { #include <odp/api/proto_stats_types.h> #include <odp/api/queue_types.h> -/** @addtogroup odp_packet +/** @defgroup odp_packet ODP PACKET * @{ */ diff --git a/include/odp/api/spec/pool.h b/include/odp/api/spec/pool.h index b02c0d294..1b71a5a09 100644 --- a/include/odp/api/spec/pool.h +++ b/include/odp/api/spec/pool.h @@ -20,7 +20,7 @@ extern "C" { #include <odp/api/std_types.h> #include <odp/api/pool_types.h> -/** @defgroup odp_pool ODP POOL +/** @addtogroup odp_pool * Packet and buffer (event) pools. * @{ */ diff --git a/include/odp/api/spec/pool_types.h b/include/odp/api/spec/pool_types.h index 7820349ef..cb3db4737 100644 --- a/include/odp/api/spec/pool_types.h +++ b/include/odp/api/spec/pool_types.h @@ -18,8 +18,9 @@ extern "C" { #include <odp/api/std_types.h> #include <odp/api/dma_types.h> +#include <odp/api/ml_types.h> -/** @addtogroup odp_pool +/** @defgroup odp_pool ODP POOL * @{ */ @@ -417,7 +418,10 @@ typedef enum odp_pool_type_t { ODP_POOL_VECTOR, /** DMA completion event pool */ - ODP_POOL_DMA_COMPL + ODP_POOL_DMA_COMPL, + + /** ML completion event pool */ + ODP_POOL_ML_COMPL } odp_pool_type_t; @@ -891,6 +895,9 @@ typedef struct odp_pool_info_t { /** Copy of pool parameters when pool type is ODP_POOL_DMA_COMPL. */ odp_dma_pool_param_t dma_pool_param; + + /** Copy of pool parameters when pool type is ODP_POOL_ML_COMPL. */ + odp_ml_compl_pool_param_t ml_pool_param; }; /** Additional info for packet pools */ diff --git a/include/odp/api/spec/proto_stats.h b/include/odp/api/spec/proto_stats.h index 1a1f67886..7dd57ac0f 100644 --- a/include/odp/api/spec/proto_stats.h +++ b/include/odp/api/spec/proto_stats.h @@ -18,7 +18,7 @@ extern "C" { #include <odp/api/proto_stats_types.h> -/** @defgroup odp_proto_stats ODP PROTO STATS +/** @addtogroup odp_proto_stats * Flow specific packet statistics. * @{ */ diff --git a/include/odp/api/spec/proto_stats_types.h b/include/odp/api/spec/proto_stats_types.h index f3ca80426..4c08e60ab 100644 --- a/include/odp/api/spec/proto_stats_types.h +++ b/include/odp/api/spec/proto_stats_types.h @@ -19,11 +19,16 @@ extern "C" { #include <odp/api/std_types.h> -/** @addtogroup odp_proto_stats +/** @defgroup odp_proto_stats ODP PROTO STATS * @{ */ /** + * @typedef odp_proto_stats_t + * ODP proto stats handle + */ + +/** * @def ODP_PROTO_STATS_INVALID * Invalid proto stats handle */ diff --git a/include/odp/api/spec/queue.h b/include/odp/api/spec/queue.h index 87f6e0d19..9ce2ac73f 100644 --- a/include/odp/api/spec/queue.h +++ b/include/odp/api/spec/queue.h @@ -21,7 +21,7 @@ extern "C" { #include <odp/api/queue_types.h> #include <odp/api/std_types.h> -/** @defgroup odp_queue ODP QUEUE +/** @addtogroup odp_queue * Queues for event passing and scheduling. * @{ */ diff --git a/include/odp/api/spec/queue_types.h b/include/odp/api/spec/queue_types.h index 5f84a5f49..9edf7271d 100644 --- a/include/odp/api/spec/queue_types.h +++ b/include/odp/api/spec/queue_types.h @@ -18,7 +18,7 @@ extern "C" { #include <odp/api/schedule_types.h> -/** @addtogroup odp_queue +/** @defgroup odp_queue ODP QUEUE * @{ */ diff --git a/include/odp/api/spec/random.h b/include/odp/api/spec/random.h index dd30f9d48..10f1026b3 100644 --- a/include/odp/api/spec/random.h +++ b/include/odp/api/spec/random.h @@ -19,7 +19,7 @@ extern "C" { #include <odp/api/random_types.h> #include <odp/api/std_types.h> -/** @defgroup odp_random ODP RANDOM +/** @addtogroup odp_random * Random number generation. * @{ */ diff --git a/include/odp/api/spec/random_types.h b/include/odp/api/spec/random_types.h index cb7dccc7c..5d5ee9450 100644 --- a/include/odp/api/spec/random_types.h +++ b/include/odp/api/spec/random_types.h @@ -19,7 +19,7 @@ extern "C" { #include <odp/api/std_types.h> -/** @addtogroup odp_random +/** @defgroup odp_random ODP RANDOM * @{ */ diff --git a/include/odp/api/spec/schedule.h b/include/odp/api/spec/schedule.h index 31da38e4d..7226c198b 100644 --- a/include/odp/api/spec/schedule.h +++ b/include/odp/api/spec/schedule.h @@ -22,7 +22,7 @@ extern "C" { #include <odp/api/schedule_types.h> #include <odp/api/thrmask.h> -/** @defgroup odp_scheduler ODP SCHEDULER +/** @addtogroup odp_scheduler * Event scheduler for work load balancing and prioritization. * @{ */ diff --git a/include/odp/api/spec/schedule_types.h b/include/odp/api/spec/schedule_types.h index b15397b96..30cb939dc 100644 --- a/include/odp/api/spec/schedule_types.h +++ b/include/odp/api/spec/schedule_types.h @@ -20,7 +20,7 @@ extern "C" { #endif -/** @addtogroup odp_scheduler +/** @defgroup odp_scheduler ODP SCHEDULER * @{ */ diff --git a/include/odp/api/spec/stash.h b/include/odp/api/spec/stash.h index 756214abe..61ae58eba 100644 --- a/include/odp/api/spec/stash.h +++ b/include/odp/api/spec/stash.h @@ -18,7 +18,7 @@ extern "C" { #include <odp/api/stash_types.h> -/** @defgroup odp_stash ODP STASH +/** @addtogroup odp_stash * Stash for storing object handles * @{ */ diff --git a/include/odp/api/spec/stash_types.h b/include/odp/api/spec/stash_types.h index 96e136d78..48c4b9be8 100644 --- a/include/odp/api/spec/stash_types.h +++ b/include/odp/api/spec/stash_types.h @@ -18,7 +18,7 @@ extern "C" { #include <odp/api/std_types.h> -/** @addtogroup odp_stash +/** @defgroup odp_stash ODP STASH * @{ */ diff --git a/include/odp/api/spec/std.h b/include/odp/api/spec/std.h index fba1ee31d..7be543338 100644 --- a/include/odp/api/spec/std.h +++ b/include/odp/api/spec/std.h @@ -19,12 +19,10 @@ extern "C" { #include <odp/api/std_types.h> -/** - * @defgroup odp_std ODP STD - * Standard types and performance optimized versions of selected C library - * functions. - * - * @{ +/** @addtogroup odp_std + * Standard types and performance optimized versions of selected C library + * functions. + * @{ */ /** diff --git a/include/odp/api/spec/std_types.h b/include/odp/api/spec/std_types.h index e2630e044..5428631b6 100644 --- a/include/odp/api/spec/std_types.h +++ b/include/odp/api/spec/std_types.h @@ -22,7 +22,7 @@ extern "C" { #endif -/** @addtogroup odp_std ODP STD +/** @defgroup odp_std ODP STD * @{ */ @@ -138,6 +138,9 @@ typedef union odp_feature_t { /** IPsec APIs, e.g., odp_ipsec_xxx() */ uint32_t ipsec:1; + /** Machine Learning APIs, e.g., odp_ml_xxx() */ + uint32_t ml:1; + /** Scheduler APIs, e.g., odp_schedule_xxx() */ uint32_t schedule:1; diff --git a/include/odp/api/spec/thread.h b/include/odp/api/spec/thread.h index dbb033da1..34d3c2b82 100644 --- a/include/odp/api/spec/thread.h +++ b/include/odp/api/spec/thread.h @@ -19,7 +19,7 @@ extern "C" { #include <odp/api/thread_types.h> -/** @defgroup odp_thread ODP THREAD +/** @addtogroup odp_thread * Thread types, masks and IDs. * @{ */ diff --git a/include/odp/api/spec/thread_types.h b/include/odp/api/spec/thread_types.h index 204d28cad..13846cd8f 100644 --- a/include/odp/api/spec/thread_types.h +++ b/include/odp/api/spec/thread_types.h @@ -15,7 +15,7 @@ extern "C" { #endif -/** @ingroup odp_thread ODP THREAD +/** @defgroup odp_thread ODP THREAD * @{ */ diff --git a/include/odp/api/spec/time.h b/include/odp/api/spec/time.h index c3571f7fa..f7e20a6f4 100644 --- a/include/odp/api/spec/time.h +++ b/include/odp/api/spec/time.h @@ -20,7 +20,7 @@ extern "C" { #include <odp/api/std_types.h> #include <odp/api/time_types.h> -/** @defgroup odp_time ODP TIME +/** @addtogroup odp_time * SoC global and CPU local wall clock time * * @{ diff --git a/include/odp/api/spec/time_types.h b/include/odp/api/spec/time_types.h index cf0393ef0..bd8f324a3 100644 --- a/include/odp/api/spec/time_types.h +++ b/include/odp/api/spec/time_types.h @@ -17,7 +17,7 @@ extern "C" { #endif -/** @addtogroup odp_time +/** @defgroup odp_time ODP TIME * @{ */ diff --git a/include/odp/arch/arm32-linux/odp/api/abi/ml_types.h b/include/odp/arch/arm32-linux/odp/api/abi/ml_types.h new file mode 100644 index 000000000..4201f92e6 --- /dev/null +++ b/include/odp/arch/arm32-linux/odp/api/abi/ml_types.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021 Nokia + */ + +#include <odp/api/abi-default/ml_types.h> diff --git a/include/odp/arch/arm64-linux/odp/api/abi/ml_types.h b/include/odp/arch/arm64-linux/odp/api/abi/ml_types.h new file mode 100644 index 000000000..4201f92e6 --- /dev/null +++ b/include/odp/arch/arm64-linux/odp/api/abi/ml_types.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021 Nokia + */ + +#include <odp/api/abi-default/ml_types.h> diff --git a/include/odp/arch/default-linux/odp/api/abi/ml_types.h b/include/odp/arch/default-linux/odp/api/abi/ml_types.h new file mode 100644 index 000000000..4201f92e6 --- /dev/null +++ b/include/odp/arch/default-linux/odp/api/abi/ml_types.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021 Nokia + */ + +#include <odp/api/abi-default/ml_types.h> diff --git a/include/odp/arch/power64-linux/odp/api/abi/ml_types.h b/include/odp/arch/power64-linux/odp/api/abi/ml_types.h new file mode 100644 index 000000000..4201f92e6 --- /dev/null +++ b/include/odp/arch/power64-linux/odp/api/abi/ml_types.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021 Nokia + */ + +#include <odp/api/abi-default/ml_types.h> diff --git a/include/odp/arch/x86_32-linux/odp/api/abi/ml_types.h b/include/odp/arch/x86_32-linux/odp/api/abi/ml_types.h new file mode 100644 index 000000000..4201f92e6 --- /dev/null +++ b/include/odp/arch/x86_32-linux/odp/api/abi/ml_types.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021 Nokia + */ + +#include <odp/api/abi-default/ml_types.h> diff --git a/include/odp/arch/x86_64-linux/odp/api/abi/ml_types.h b/include/odp/arch/x86_64-linux/odp/api/abi/ml_types.h new file mode 100644 index 000000000..4201f92e6 --- /dev/null +++ b/include/odp/arch/x86_64-linux/odp/api/abi/ml_types.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021 Nokia + */ + +#include <odp/api/abi-default/ml_types.h> diff --git a/include/odp_api.h b/include/odp_api.h index e7cad266f..cbd1af691 100644 --- a/include/odp_api.h +++ b/include/odp_api.h @@ -55,6 +55,8 @@ extern "C" { #include <odp/api/errno.h> #include <odp/api/thrmask.h> #include <odp/api/traffic_mngr.h> +#include <odp/api/ml.h> +#include <odp/api/ml_quantize.h> #include <odp/api/spinlock_recursive.h> #include <odp/api/rwlock_recursive.h> #include <odp/api/std.h> diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am index 886fe8f48..0bfe35092 100644 --- a/platform/linux-dpdk/Makefile.am +++ b/platform/linux-dpdk/Makefile.am @@ -13,6 +13,7 @@ AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/@ARCH_DIR@ AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/default AM_CPPFLAGS += $(OPENSSL_CPPFLAGS) +AM_CPPFLAGS += $(ORT_CPPFLAGS) AM_CFLAGS += $(DPDK_CFLAGS) AM_CFLAGS += $(LIBCONFIG_CFLAGS) @@ -88,6 +89,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/init.h \ include-abi/odp/api/abi/ipsec.h \ include-abi/odp/api/abi/ipsec_types.h \ + include-abi/odp/api/abi/ml_types.h \ include-abi/odp/api/abi/packet.h \ include-abi/odp/api/abi/packet_types.h \ include-abi/odp/api/abi/packet_flags.h \ @@ -136,6 +138,7 @@ noinst_HEADERS = \ ${top_srcdir}/platform/linux-generic/include/odp_classification_internal.h \ include/odp_eventdev_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_forward_typedefs_internal.h \ + ${top_srcdir}/platform/linux-generic/include/odp_ml_fp16.h \ ${top_srcdir}/platform/linux-generic/include/odp_global_data.h \ ${top_srcdir}/platform/linux-generic/include/odp_init_internal.h \ ${top_srcdir}/platform/linux-generic/include/odp_ipsec_internal.h \ @@ -217,6 +220,8 @@ __LIB__libodp_dpdk_la_SOURCES = \ ../linux-generic/odp_ipsec_sad.c \ ../linux-generic/odp_name_table.c \ ../linux-generic/odp_libconfig.c \ + ../linux-generic/odp_ml_fp16.c \ + ../linux-generic/odp_ml_quantize.c \ odp_packet.c \ odp_packet_dpdk.c \ ../linux-generic/odp_packet_vector.c \ @@ -255,6 +260,14 @@ __LIB__libodp_dpdk_la_SOURCES = \ ../linux-generic/odp_version.c \ ../linux-generic/odp_weak.c +if WITH_ML +__LIB__libodp_dpdk_la_SOURCES += \ + ../linux-generic/odp_ml.c +else +__LIB__libodp_dpdk_la_SOURCES += \ + ../linux-generic/odp_ml_null.c +endif + if ODP_ABI_COMPAT __LIB__libodp_dpdk_la_SOURCES += \ ../linux-generic/odp_atomic_api.c \ @@ -301,15 +314,11 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/wait_until_generic.h \ arch/default/odp/api/abi/wait_until.h endif -noinst_HEADERS += arch/arm/odp_atomic.h \ - arch/arm/odp_cpu.h \ - arch/arm/odp_cpu_idling.h \ - arch/arm/odp_llsc.h \ +noinst_HEADERS += arch/arm/odp_cpu.h \ arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h \ - arch/default/odp_random.h - + arch/default/odp_random.h \ + arch/default/odp_wait_until.h endif if ARCH_IS_AARCH64 __LIB__libodp_dpdk_la_SOURCES += arch/aarch64/odp_atomic.c \ @@ -334,9 +343,8 @@ endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ arch/aarch64/cpu_flags.h \ - arch/aarch64/odp_cpu_idling.h \ - arch/aarch64/odp_llsc.h \ - arch/aarch64/odp_random.h + arch/aarch64/odp_random.h \ + arch/aarch64/odp_wait_until.h endif if ARCH_IS_DEFAULT __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ @@ -357,8 +365,8 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h \ - arch/default/odp_random.h + arch/default/odp_random.h \ + arch/default/odp_wait_until.h endif if ARCH_IS_POWERPC __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ @@ -379,8 +387,8 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h \ - arch/default/odp_random.h + arch/default/odp_random.h \ + arch/default/odp_wait_until.h endif if ARCH_IS_X86 __LIB__libodp_dpdk_la_SOURCES += arch/default/odp_atomic.c \ @@ -406,7 +414,7 @@ noinst_HEADERS += arch/x86/cpu_flags.h \ arch/x86/odp_random.h \ arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h + arch/default/odp_wait_until.h endif __LIB__libodp_dpdk_la_LIBADD = $(ATOMIC_LIBS) @@ -415,6 +423,7 @@ __LIB__libodp_dpdk_la_LIBADD += $(LIBCONFIG_LIBS) __LIB__libodp_dpdk_la_LIBADD += $(DPDK_LIBS_LIBODP) __LIB__libodp_dpdk_la_LIBADD += $(PTHREAD_LIBS) __LIB__libodp_dpdk_la_LIBADD += $(TIMER_LIBS) +__LIB__libodp_dpdk_la_LIBADD += $(ORT_LIBS) CHECK_GLOBALS_REGEX = " (odp_|_odp_|_deprecated_odp_|miniz_|mz_|tdefl_|tinfl_|mp_hdlr_init_odp_pool_ops)" diff --git a/platform/linux-dpdk/README b/platform/linux-dpdk/README index 421d3f958..c0298ab34 100644 --- a/platform/linux-dpdk/README +++ b/platform/linux-dpdk/README @@ -36,7 +36,8 @@ cmds below for Ubuntu, where it has been compiled and tested. On Ubuntu install pcap development library: sudo apt-get install libpcap-dev -Right now ODP-DPDK supports DPDK v21.11 and v22.11 (recommended version). +Right now ODP-DPDK supports DPDK v21.11, v22.11 (recommended version), and +v23.11. Compile DPDK ------------ diff --git a/platform/linux-dpdk/arch/aarch64/odp_cpu_idling.h b/platform/linux-dpdk/arch/aarch64/odp_cpu_idling.h deleted file mode 120000 index c8230bb63..000000000 --- a/platform/linux-dpdk/arch/aarch64/odp_cpu_idling.h +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/aarch64/odp_cpu_idling.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/aarch64/odp_llsc.h b/platform/linux-dpdk/arch/aarch64/odp_llsc.h deleted file mode 120000 index eb8d1200b..000000000 --- a/platform/linux-dpdk/arch/aarch64/odp_llsc.h +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/aarch64/odp_llsc.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/aarch64/odp_wait_until.h b/platform/linux-dpdk/arch/aarch64/odp_wait_until.h new file mode 120000 index 000000000..f7d35f0ca --- /dev/null +++ b/platform/linux-dpdk/arch/aarch64/odp_wait_until.h @@ -0,0 +1 @@ +../../../linux-generic/arch/aarch64/odp_wait_until.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/arm/odp_atomic.h b/platform/linux-dpdk/arch/arm/odp_atomic.h deleted file mode 120000 index 61a8c536b..000000000 --- a/platform/linux-dpdk/arch/arm/odp_atomic.h +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/arm/odp_atomic.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/arm/odp_cpu_idling.h b/platform/linux-dpdk/arch/arm/odp_cpu_idling.h deleted file mode 120000 index 56afe9027..000000000 --- a/platform/linux-dpdk/arch/arm/odp_cpu_idling.h +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/arm/odp_cpu_idling.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/arm/odp_llsc.h b/platform/linux-dpdk/arch/arm/odp_llsc.h deleted file mode 120000 index b3f3f371f..000000000 --- a/platform/linux-dpdk/arch/arm/odp_llsc.h +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/arm/odp_llsc.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/default/odp_cpu_idling.h b/platform/linux-dpdk/arch/default/odp_cpu_idling.h deleted file mode 120000 index eb2d21c4b..000000000 --- a/platform/linux-dpdk/arch/default/odp_cpu_idling.h +++ /dev/null @@ -1 +0,0 @@ -../../../linux-generic/arch/default/odp_cpu_idling.h
\ No newline at end of file diff --git a/platform/linux-dpdk/arch/default/odp_wait_until.h b/platform/linux-dpdk/arch/default/odp_wait_until.h new file mode 120000 index 000000000..d2e7b5316 --- /dev/null +++ b/platform/linux-dpdk/arch/default/odp_wait_until.h @@ -0,0 +1 @@ +../../../linux-generic/arch/default/odp_wait_until.h
\ No newline at end of file diff --git a/platform/linux-dpdk/example/Makefile.am b/platform/linux-dpdk/example/Makefile.am new file mode 100644 index 000000000..84f337387 --- /dev/null +++ b/platform/linux-dpdk/example/Makefile.am @@ -0,0 +1,5 @@ +SUBDIRS = + +if WITH_ML +SUBDIRS += ml +endif diff --git a/platform/linux-dpdk/example/ml/.gitignore b/platform/linux-dpdk/example/ml/.gitignore new file mode 100644 index 000000000..d845f6bb5 --- /dev/null +++ b/platform/linux-dpdk/example/ml/.gitignore @@ -0,0 +1,5 @@ +model_explorer +simple_linear +mnist +*.log +*.trs diff --git a/platform/linux-dpdk/example/ml/Makefile.am b/platform/linux-dpdk/example/ml/Makefile.am new file mode 100644 index 000000000..7abbc3828 --- /dev/null +++ b/platform/linux-dpdk/example/ml/Makefile.am @@ -0,0 +1,54 @@ +include $(top_srcdir)/example/Makefile.inc + +LDADD += -lm + +bin_PROGRAMS = model_explorer simple_linear mnist + +simple_linear_SOURCES = \ + ../../../linux-generic/example/ml/simple_linear.c \ + ../../../linux-generic/example/ml/model_read.c \ + ../../../linux-generic/example/ml/model_read.h +model_explorer_SOURCES = \ + ../../../linux-generic/example/ml/model_explorer.c \ + ../../../linux-generic/example/ml/model_read.c \ + ../../../linux-generic/example/ml/model_read.h +mnist_SOURCES = \ + ../../../linux-generic/example/ml/mnist.c \ + ../../../linux-generic/example/ml/model_read.c \ + ../../../linux-generic/example/ml/model_read.h + +EXTRA_DIST = \ + odp_ml_run_mnist.sh \ + example_digit.csv \ + mnist-12.onnx \ + odp_ml_run_model_explorer.sh \ + odp_ml_run_simple_linear.sh \ + simple_linear.onnx + +if test_example +TESTS = \ + odp_ml_run_mnist.sh \ + odp_ml_run_model_explorer.sh \ + odp_ml_run_simple_linear.sh +endif + +# If building out-of-tree, make check will not copy the scripts and data to the +# $(builddir) assuming that all commands are run locally. However this prevents +# running tests on a remote target using LOG_COMPILER. +# So copy all script and data files explicitly here. +all-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + if [ -e $(srcdir)/$$f ]; then \ + mkdir -p $(builddir)/$$(dirname $$f); \ + cp -f $(srcdir)/$$f $(builddir)/$$f; \ + fi \ + done \ + fi + +clean-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + rm -f $(builddir)/$$f; \ + done \ + fi diff --git a/platform/linux-dpdk/example/ml/README.md b/platform/linux-dpdk/example/ml/README.md new file mode 120000 index 000000000..ddeec649f --- /dev/null +++ b/platform/linux-dpdk/example/ml/README.md @@ -0,0 +1 @@ +../../../linux-generic/example/ml/README.md
\ No newline at end of file diff --git a/platform/linux-dpdk/example/ml/example_digit.csv b/platform/linux-dpdk/example/ml/example_digit.csv new file mode 120000 index 000000000..5e5514aaf --- /dev/null +++ b/platform/linux-dpdk/example/ml/example_digit.csv @@ -0,0 +1 @@ +../../../linux-generic/example/ml/example_digit.csv
\ No newline at end of file diff --git a/platform/linux-dpdk/example/ml/mnist-12.onnx b/platform/linux-dpdk/example/ml/mnist-12.onnx new file mode 120000 index 000000000..94d4515b8 --- /dev/null +++ b/platform/linux-dpdk/example/ml/mnist-12.onnx @@ -0,0 +1 @@ +../../../linux-generic/example/ml/mnist-12.onnx
\ No newline at end of file diff --git a/platform/linux-dpdk/example/ml/odp_ml_run_mnist.sh b/platform/linux-dpdk/example/ml/odp_ml_run_mnist.sh new file mode 120000 index 000000000..7d9c6f84c --- /dev/null +++ b/platform/linux-dpdk/example/ml/odp_ml_run_mnist.sh @@ -0,0 +1 @@ +../../../linux-generic/example/ml/odp_ml_run_mnist.sh
\ No newline at end of file diff --git a/platform/linux-dpdk/example/ml/odp_ml_run_model_explorer.sh b/platform/linux-dpdk/example/ml/odp_ml_run_model_explorer.sh new file mode 120000 index 000000000..f28535b64 --- /dev/null +++ b/platform/linux-dpdk/example/ml/odp_ml_run_model_explorer.sh @@ -0,0 +1 @@ +../../../linux-generic/example/ml/odp_ml_run_model_explorer.sh
\ No newline at end of file diff --git a/platform/linux-dpdk/example/ml/odp_ml_run_simple_linear.sh b/platform/linux-dpdk/example/ml/odp_ml_run_simple_linear.sh new file mode 120000 index 000000000..2691d9282 --- /dev/null +++ b/platform/linux-dpdk/example/ml/odp_ml_run_simple_linear.sh @@ -0,0 +1 @@ +../../../linux-generic/example/ml/odp_ml_run_simple_linear.sh
\ No newline at end of file diff --git a/platform/linux-dpdk/example/ml/simple_linear.onnx b/platform/linux-dpdk/example/ml/simple_linear.onnx new file mode 120000 index 000000000..5893a9176 --- /dev/null +++ b/platform/linux-dpdk/example/ml/simple_linear.onnx @@ -0,0 +1 @@ +../../../linux-generic/example/ml/simple_linear.onnx
\ No newline at end of file diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/dma_types.h b/platform/linux-dpdk/include-abi/odp/api/abi/dma_types.h index 3c009f4c4..318c2c385 100644 --- a/platform/linux-dpdk/include-abi/odp/api/abi/dma_types.h +++ b/platform/linux-dpdk/include-abi/odp/api/abi/dma_types.h @@ -11,7 +11,7 @@ extern "C" { #include <odp/api/plat/strong_types.h> -/** @ingroup odp_dma +/** @addtogroup odp_dma * @{ */ diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/ml_types.h b/platform/linux-dpdk/include-abi/odp/api/abi/ml_types.h new file mode 120000 index 000000000..18b483da1 --- /dev/null +++ b/platform/linux-dpdk/include-abi/odp/api/abi/ml_types.h @@ -0,0 +1 @@ +../../../../../linux-generic/include-abi/odp/api/abi/ml_types.h
\ No newline at end of file diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/packet_types.h b/platform/linux-dpdk/include-abi/odp/api/abi/packet_types.h index 1f5f9e6f7..9ca66db54 100644 --- a/platform/linux-dpdk/include-abi/odp/api/abi/packet_types.h +++ b/platform/linux-dpdk/include-abi/odp/api/abi/packet_types.h @@ -21,7 +21,7 @@ extern "C" { #include <odp/api/std_types.h> #include <odp/api/plat/strong_types.h> -/** @ingroup odp_packet +/** @addtogroup odp_packet * @{ */ diff --git a/platform/linux-dpdk/include/odp/api/plat/event_inline_types.h b/platform/linux-dpdk/include/odp/api/plat/event_inline_types.h index ee5490ff1..94a95a889 100644 --- a/platform/linux-dpdk/include/odp/api/plat/event_inline_types.h +++ b/platform/linux-dpdk/include/odp/api/plat/event_inline_types.h @@ -28,6 +28,7 @@ extern "C" { typedef struct _odp_event_inline_offset_t { uint16_t event_type; uint16_t base_data; + uint16_t subtype; uint16_t flow_id; uint16_t pool; uint16_t buf_len; diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h b/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h index 5bbcadd32..255db9d78 100644 --- a/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h @@ -41,7 +41,6 @@ typedef struct _odp_packet_inline_offset_t { uint16_t timestamp; uint16_t input_flags; uint16_t flags; - uint16_t subtype; uint16_t cls_mark; uint16_t ipsec_ctx; uint16_t crypto_op; diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h index 7b2764a05..b41a272ef 100644 --- a/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-dpdk/include/odp/api/plat/packet_inlines.h @@ -25,6 +25,7 @@ extern "C" { #include <odp/api/time_types.h> #include <odp/api/plat/debug_inlines.h> +#include <odp/api/plat/event_inline_types.h> #include <odp/api/plat/event_validation_external.h> #include <odp/api/plat/packet_io_inlines.h> #include <odp/api/plat/packet_inline_types.h> @@ -647,7 +648,8 @@ _ODP_INLINE void odp_packet_to_event_multi(const odp_packet_t pkt[], _ODP_INLINE odp_event_subtype_t odp_packet_subtype(odp_packet_t pkt) { - return (odp_event_subtype_t)_odp_pkt_get(pkt, int8_t, subtype); + return (odp_event_subtype_t)_odp_event_hdr_field((odp_event_t)(uintptr_t)pkt, + int8_t, subtype); } _ODP_INLINE odp_packet_tx_compl_t odp_packet_tx_compl_from_event(odp_event_t ev) diff --git a/platform/linux-dpdk/include/odp_buffer_internal.h b/platform/linux-dpdk/include/odp_buffer_internal.h index dc65fd17d..cb7f50073 100644 --- a/platform/linux-dpdk/include/odp_buffer_internal.h +++ b/platform/linux-dpdk/include/odp_buffer_internal.h @@ -73,6 +73,13 @@ static inline odp_buffer_hdr_t *_odp_buf_hdr(odp_buffer_t buf) return (odp_buffer_hdr_t *)(uintptr_t)buf; } +static inline void _odp_buffer_subtype_set(odp_buffer_t buf, int subtype) +{ + odp_buffer_hdr_t *buf_hdr = _odp_buf_hdr(buf); + + buf_hdr->event_hdr.subtype = subtype; +} + #ifdef __cplusplus } #endif diff --git a/platform/linux-dpdk/include/odp_config_internal.h b/platform/linux-dpdk/include/odp_config_internal.h index bc69610ca..c423ec14b 100644 --- a/platform/linux-dpdk/include/odp_config_internal.h +++ b/platform/linux-dpdk/include/odp_config_internal.h @@ -177,6 +177,15 @@ extern "C" { */ #define CONFIG_IPSEC_MAX_NUM_SA 4000 +/* Maximum number of ML models that can be created or loaded. */ +#define CONFIG_ML_MAX_MODELS 4 + +/* Maximum number of inputs for a ML model. */ +#define CONFIG_ML_MAX_INPUTS 4 + +/* Maximum number of outputs for a ML model. */ +#define CONFIG_ML_MAX_OUTPUTS 4 + #ifdef __cplusplus } #endif diff --git a/platform/linux-dpdk/include/odp_packet_internal.h b/platform/linux-dpdk/include/odp_packet_internal.h index c86f0646a..cae77245a 100644 --- a/platform/linux-dpdk/include/odp_packet_internal.h +++ b/platform/linux-dpdk/include/odp_packet_internal.h @@ -218,9 +218,9 @@ static inline struct rte_mbuf *pkt_to_mbuf(odp_packet_t pkt) return (struct rte_mbuf *)(uintptr_t)pkt; } -static inline void packet_subtype_set(odp_packet_t pkt, int ev) +static inline void packet_subtype_set(odp_packet_t pkt, int subtype) { - packet_hdr(pkt)->event_hdr.subtype = ev; + packet_hdr(pkt)->event_hdr.subtype = subtype; } /** @@ -236,7 +236,9 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, odp_pktio_t input) pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; - pkt_hdr->event_hdr.subtype = ODP_EVENT_PACKET_BASIC; + if (odp_unlikely(pkt_hdr->event_hdr.subtype != ODP_EVENT_PACKET_BASIC)) + pkt_hdr->event_hdr.subtype = ODP_EVENT_PACKET_BASIC; + pkt_hdr->input = input; } diff --git a/platform/linux-dpdk/include/odp_timer_internal.h b/platform/linux-dpdk/include/odp_timer_internal.h index 3cc8ca469..35a4911af 100644 --- a/platform/linux-dpdk/include/odp_timer_internal.h +++ b/platform/linux-dpdk/include/odp_timer_internal.h @@ -24,6 +24,14 @@ #include <rte_config.h> +#include <stdint.h> + +/* + * Use as the argument to timer_run() to force a scan and to ignore rate + * limit. + */ +#define TIMER_SCAN_FORCE INT32_MAX + /** * Internal Timeout header */ @@ -56,10 +64,13 @@ ODP_STATIC_ASSERT(sizeof(odp_timeout_hdr_t) <= 3 * RTE_CACHE_LINE_SIZE, void _odp_timer_run_inline(int dec); /* Static inline wrapper to minimize modification of schedulers. */ -static inline void timer_run(int dec) +static inline uint64_t timer_run(int dec) { if (odp_global_rw->inline_timers) _odp_timer_run_inline(dec); + + /* Time to the next timeout not available with DPDK timers */ + return UINT64_MAX; } #endif diff --git a/platform/linux-dpdk/libodp-dpdk.pc.in b/platform/linux-dpdk/libodp-dpdk.pc.in index c3ee4f7b9..8fcc4ac0a 100644 --- a/platform/linux-dpdk/libodp-dpdk.pc.in +++ b/platform/linux-dpdk/libodp-dpdk.pc.in @@ -8,5 +8,5 @@ Description: The ODP packet processing engine Version: @PKGCONFIG_VERSION@ Requires.private: libconfig Libs: -L${libdir} -l@ODP_LIB_NAME@ @DPDK_LIBS_NON_ABI_COMPAT@ @ATOMIC_LIBS_NON_ABI_COMPAT@ -Libs.private: @DPDK_LIBS_ABI_COMPAT@ @OPENSSL_STATIC_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ -lpthread @ATOMIC_LIBS_ABI_COMPAT@ +Libs.private: @DPDK_LIBS_ABI_COMPAT@ @OPENSSL_STATIC_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ -lpthread @ATOMIC_LIBS_ABI_COMPAT@ @ORT_LIBS@ Cflags: -I${includedir} @DPDK_CFLAGS@ diff --git a/platform/linux-dpdk/m4/configure.m4 b/platform/linux-dpdk/m4/configure.m4 index 0fcd4a5b3..535cfc5fa 100644 --- a/platform/linux-dpdk/m4/configure.m4 +++ b/platform/linux-dpdk/m4/configure.m4 @@ -11,6 +11,7 @@ m4_include([platform/linux-dpdk/m4/odp_openssl.m4]) m4_include([platform/linux-dpdk/m4/odp_pcapng.m4]) m4_include([platform/linux-dpdk/m4/odp_scheduler.m4]) m4_include([platform/linux-dpdk/m4/odp_wfe.m4]) +m4_include([platform/linux-dpdk/m4/odp_ml.m4]) ODP_EVENT_VALIDATION ODP_PTHREAD @@ -66,7 +67,7 @@ esac # Required for experimental rte_event_port_unlinks_in_progress() API DPDK_CFLAGS="${DPDK_CFLAGS} -DALLOW_EXPERIMENTAL_API" -AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS}"]) +AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS} ${ORT_LIBS}"]) # Add text to the end of configure with platform specific settings. # Make sure it's aligned same as other lines in configure.ac. @@ -77,6 +78,7 @@ AS_VAR_APPEND([PLAT_CFG_TEXT], [" pcap: ${have_pmd_pcap} pcapng: ${have_pcapng} wfe_locks: ${use_wfe_locks} + ml_support: ${ml_support} default_config_path: ${default_config_path}"]) ODP_CHECK_CFLAG([-Wno-error=cast-align]) @@ -94,6 +96,8 @@ AM_CONDITIONAL([PLATFORM_IS_LINUX_DPDK], AC_CONFIG_FILES([platform/linux-dpdk/Makefile platform/linux-dpdk/libodp-dpdk.pc platform/linux-dpdk/dumpconfig/Makefile + platform/linux-dpdk/example/Makefile + platform/linux-dpdk/example/ml/Makefile platform/linux-dpdk/test/Makefile platform/linux-dpdk/test/example/Makefile platform/linux-dpdk/test/example/classifier/Makefile @@ -108,5 +112,6 @@ AC_CONFIG_FILES([platform/linux-dpdk/Makefile platform/linux-dpdk/test/example/switch/Makefile platform/linux-dpdk/test/performance/Makefile platform/linux-dpdk/test/performance/dmafwd/Makefile + platform/linux-dpdk/test/validation/api/ml/Makefile platform/linux-dpdk/test/validation/api/pktio/Makefile]) ]) diff --git a/platform/linux-dpdk/m4/odp_libconfig.m4 b/platform/linux-dpdk/m4/odp_libconfig.m4 index 7bcfb4505..2bf89ac2e 100644 --- a/platform/linux-dpdk/m4/odp_libconfig.m4 +++ b/platform/linux-dpdk/m4/odp_libconfig.m4 @@ -3,7 +3,7 @@ ########################################################################## m4_define([_odp_config_version_generation], [0]) m4_define([_odp_config_version_major], [1]) -m4_define([_odp_config_version_minor], [25]) +m4_define([_odp_config_version_minor], [26]) m4_define([_odp_config_version], [_odp_config_version_generation._odp_config_version_major._odp_config_version_minor]) diff --git a/platform/linux-dpdk/m4/odp_ml.m4 b/platform/linux-dpdk/m4/odp_ml.m4 new file mode 120000 index 000000000..6e76047e5 --- /dev/null +++ b/platform/linux-dpdk/m4/odp_ml.m4 @@ -0,0 +1 @@ +../../linux-generic/m4/odp_ml.m4
\ No newline at end of file diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c index aeb9c11e3..6170dd8df 100644 --- a/platform/linux-dpdk/odp_crypto.c +++ b/platform/linux-dpdk/odp_crypto.c @@ -707,12 +707,11 @@ static int cipher_gen_capability(const struct rte_crypto_param_range *key_size, for (uint32_t key_len = key_size_min; key_len <= key_size_max; key_len += key_inc) { - for (uint32_t iv_size = iv_size_min; - iv_size <= iv_size_max; iv_size += iv_inc) { + for (uint32_t iv_len = iv_size_min; iv_len <= iv_size_max; iv_len += iv_inc) { odp_crypto_cipher_capability_t capa; capa.key_len = key_len; - capa.iv_len = iv_size; + capa.iv_len = iv_len; capa.bit_mode = false; idx = cipher_capa_insert(src, &capa, idx, num_copy); @@ -918,14 +917,14 @@ static int auth_gen_capability(const struct rte_crypto_param_range *key_size, for (uint16_t key_len = key_size_min; key_len <= key_size_max; key_len += key_inc) { - for (uint16_t iv_size = iv_size_min; - iv_size <= iv_size_max; - iv_size += iv_inc) { + for (uint16_t iv_len = iv_size_min; + iv_len <= iv_size_max; + iv_len += iv_inc) { odp_crypto_auth_capability_t capa; capa.digest_len = digest_len; capa.key_len = key_len; - capa.iv_len = iv_size; + capa.iv_len = iv_len; capa.bit_mode = false; capa.aad_len.min = aad_size->min; capa.aad_len.max = aad_size->max; diff --git a/platform/linux-dpdk/odp_event.c b/platform/linux-dpdk/odp_event.c index dff3e2ed2..0c2f3d22e 100644 --- a/platform/linux-dpdk/odp_event.c +++ b/platform/linux-dpdk/odp_event.c @@ -12,6 +12,7 @@ #include <odp/api/packet.h> #include <odp/api/timer.h> #include <odp/api/pool.h> +#include <odp/api/ml.h> #include <odp_buffer_internal.h> #include <odp_ipsec_internal.h> @@ -36,6 +37,7 @@ const _odp_event_inline_offset_t _odp_event_inline_offset ODP_ALIGNED_CACHE = { .event_type = offsetof(_odp_event_hdr_t, hdr.event_type), .base_data = offsetof(_odp_event_hdr_t, mb.buf_addr), + .subtype = offsetof(_odp_event_hdr_t, hdr.subtype), .flow_id = offsetof(_odp_event_hdr_t, hdr.flow_id), .pool = offsetof(_odp_event_hdr_t, hdr.pool), .buf_len = offsetof(_odp_event_hdr_t, mb.buf_len) @@ -69,6 +71,9 @@ static inline void event_free(odp_event_t event, _odp_ev_id_t id) case ODP_EVENT_DMA_COMPL: odp_dma_compl_free(odp_dma_compl_from_event(event)); break; + case ODP_EVENT_ML_COMPL: + odp_ml_compl_free(odp_ml_compl_from_event(event)); + break; default: _ODP_ABORT("Invalid event type: %d\n", odp_event_type(event)); } @@ -117,6 +122,8 @@ int odp_event_is_valid(odp_event_t event) /* Fall through */ case ODP_EVENT_DMA_COMPL: /* Fall through */ + case ODP_EVENT_ML_COMPL: + /* Fall through */ case ODP_EVENT_PACKET_TX_COMPL: break; default: diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c index 4d6f395c5..79c449f1d 100644 --- a/platform/linux-dpdk/odp_init.c +++ b/platform/linux-dpdk/odp_init.c @@ -59,6 +59,7 @@ enum init_stage { IPSEC_SAD_INIT, IPSEC_INIT, DMA_INIT, + ML_INIT, ALL_INIT /* All init stages completed */ }; @@ -103,6 +104,7 @@ static void disable_features(odp_global_data_ro_t *global_ro, global_ro->disable.traffic_mngr = init_param->not_used.feat.tm; global_ro->disable.compress = init_param->not_used.feat.compress; + global_ro->disable.ml = init_param->not_used.feat.ml; } static int read_pci_config(char **pci_cmd) @@ -331,6 +333,13 @@ static int term_global(enum init_stage stage) switch (stage) { case ALL_INIT: + case ML_INIT: + if (_odp_ml_term_global()) { + _ODP_ERR("ODP ML term failed.\n"); + rc = -1; + } + /* Fall through */ + case DMA_INIT: if (_odp_dma_term_global()) { _ODP_ERR("ODP DMA term failed.\n"); @@ -689,6 +698,12 @@ int odp_init_global(odp_instance_t *instance, } stage = DMA_INIT; + if (_odp_ml_init_global()) { + _ODP_ERR("ODP ML init failed.\n"); + goto init_failed; + } + stage = ML_INIT; + /* Dummy support for single instance */ *instance = (odp_instance_t)odp_global_ro.main_pid; diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c index a61c4c34a..bcd2c2fb3 100644 --- a/platform/linux-dpdk/odp_packet.c +++ b/platform/linux-dpdk/odp_packet.c @@ -64,7 +64,6 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .timestamp = offsetof(odp_packet_hdr_t, timestamp), .input_flags = offsetof(odp_packet_hdr_t, p.input_flags), .flags = offsetof(odp_packet_hdr_t, p.flags), - .subtype = offsetof(odp_packet_hdr_t, event_hdr.subtype), .cls_mark = offsetof(odp_packet_hdr_t, cls_mark), .ipsec_ctx = offsetof(odp_packet_hdr_t, ipsec_ctx), .crypto_op = offsetof(odp_packet_hdr_t, crypto_op_result), diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c index f7726f97b..dfd14a978 100644 --- a/platform/linux-dpdk/odp_pool.c +++ b/platform/linux-dpdk/odp_pool.c @@ -626,6 +626,7 @@ static void init_obj_priv_data(struct rte_mempool *mp ODP_UNUSED, void *arg, voi event_hdr->hdr.pool = _odp_pool_handle(pool); event_hdr->hdr.type = priv_data->type; event_hdr->hdr.event_type = priv_data->event_type; + event_hdr->hdr.subtype = ODP_EVENT_NO_SUBTYPE; switch (priv_data->type) { case ODP_POOL_BUFFER: @@ -852,6 +853,8 @@ static const char *get_short_type_str(odp_pool_type_t type) return "V"; case ODP_POOL_DMA_COMPL: return "D"; + case ODP_POOL_ML_COMPL: + return "M"; default: return "-"; } @@ -946,6 +949,11 @@ int odp_pool_info(odp_pool_t pool_hdl, odp_pool_info_t *info) info->dma_pool_param.uarea_size = pool->params.buf.uarea_size; info->dma_pool_param.cache_size = pool->params.buf.cache_size; + } else if (pool->type_2 == ODP_POOL_ML_COMPL) { + info->ml_pool_param.num = pool->params.buf.num; + info->ml_pool_param.uarea_size = pool->params.buf.uarea_size; + info->ml_pool_param.cache_size = pool->params.buf.cache_size; + } else { info->params = pool->params; } @@ -1091,6 +1099,7 @@ int odp_pool_ext_capability(odp_pool_type_t type, case ODP_POOL_TIMEOUT: case ODP_POOL_VECTOR: case ODP_POOL_DMA_COMPL: + case ODP_POOL_ML_COMPL: memset(capa, 0, sizeof(odp_pool_ext_capability_t)); return 0; default: @@ -1361,6 +1370,7 @@ static void init_ext_obj(struct rte_mempool *mp, void *arg, void *mbuf, unsigned event_hdr->hdr.pool = _odp_pool_handle(pool); event_hdr->hdr.type = mb_ctor_arg->type; event_hdr->hdr.event_type = mb_ctor_arg->event_type; + event_hdr->hdr.subtype = ODP_EVENT_NO_SUBTYPE; switch (mb_ctor_arg->type) { case ODP_POOL_BUFFER: diff --git a/platform/linux-dpdk/odp_schedule_eventdev.c b/platform/linux-dpdk/odp_schedule_eventdev.c index d6f3ba2f7..4ef8a51b7 100644 --- a/platform/linux-dpdk/odp_schedule_eventdev.c +++ b/platform/linux-dpdk/odp_schedule_eventdev.c @@ -604,8 +604,8 @@ static inline uint16_t input_cached(odp_event_t out_ev[], unsigned int max_num, uint8_t first_queue = _odp_eventdev_local.cache.event[idx].queue_id; for (i = 0; i < max_num && _odp_eventdev_local.cache.count; i++) { - uint16_t idx = _odp_eventdev_local.cache.idx; - struct rte_event *event = &_odp_eventdev_local.cache.event[idx]; + uint16_t cache_idx = _odp_eventdev_local.cache.idx; + struct rte_event *event = &_odp_eventdev_local.cache.event[cache_idx]; if (odp_unlikely(event->queue_id != first_queue)) break; diff --git a/platform/linux-dpdk/odp_system_info.c b/platform/linux-dpdk/odp_system_info.c index 0ebb3e09f..d9cddcb40 100644 --- a/platform/linux-dpdk/odp_system_info.c +++ b/platform/linux-dpdk/odp_system_info.c @@ -27,7 +27,6 @@ #include <odp_packet_internal.h> #include <errno.h> -#include <pthread.h> #include <string.h> #include <stdio.h> #include <inttypes.h> @@ -312,8 +311,9 @@ int _odp_system_info_init(void) num_cpus); /* Read and save all CPU frequencies for static mode */ - for (i = 0; i < CONFIG_NUM_CPU_IDS; i++) - odp_global_ro.system_info.cpu_hz[i] = cpu_hz_current(i); + if (odp_global_ro.system_info.cpu_hz_static) + for (i = 0; i < CONFIG_NUM_CPU_IDS; i++) + odp_global_ro.system_info.cpu_hz[i] = cpu_hz_current(i); /* By default, read max frequency from a cpufreq file */ for (i = 0; i < CONFIG_NUM_CPU_IDS; i++) { @@ -552,5 +552,8 @@ void odp_sys_config_print(void) _ODP_PRINT("CONFIG_POOLS: %i\n", CONFIG_POOLS); _ODP_PRINT("CONFIG_POOL_MAX_NUM: %i\n", CONFIG_POOL_MAX_NUM); _ODP_PRINT("CONFIG_IPSEC_MAX_NUM_SA: %i\n", CONFIG_IPSEC_MAX_NUM_SA); + _ODP_PRINT("CONFIG_ML_MAX_MODELS: %i\n", CONFIG_ML_MAX_MODELS); + _ODP_PRINT("CONFIG_ML_MAX_INPUTS: %i\n", CONFIG_ML_MAX_INPUTS); + _ODP_PRINT("CONFIG_ML_MAX_OUTPUTS: %i\n", CONFIG_ML_MAX_OUTPUTS); _ODP_PRINT("\n"); } diff --git a/platform/linux-dpdk/odp_timer.c b/platform/linux-dpdk/odp_timer.c index e01a541ad..f4c190aad 100644 --- a/platform/linux-dpdk/odp_timer.c +++ b/platform/linux-dpdk/odp_timer.c @@ -438,7 +438,7 @@ int _odp_timer_term_local(void) void _odp_timer_run_inline(int dec) { - int poll_interval = timer_global->poll_interval; + int poll_interval = (dec == TIMER_SCAN_FORCE) ? 0 : timer_global->poll_interval; odp_time_t now; int ret; diff --git a/platform/linux-dpdk/test/Makefile.am b/platform/linux-dpdk/test/Makefile.am index c210edbdc..2a33bfbcd 100644 --- a/platform/linux-dpdk/test/Makefile.am +++ b/platform/linux-dpdk/test/Makefile.am @@ -18,6 +18,12 @@ test_SCRIPTS = $(dist_check_SCRIPTS) SUBDIRS += validation/api/pktio \ example \ performance + +if WITH_ML +TESTS += validation/api/ml/ml_linux$(EXEEXT) +SUBDIRS += validation/api/ml +endif + else #performance tests refer to pktio_env if test_perf diff --git a/platform/linux-dpdk/test/crypto.conf b/platform/linux-dpdk/test/crypto.conf index 9b10b9a76..97fdea6f5 100644 --- a/platform/linux-dpdk/test/crypto.conf +++ b/platform/linux-dpdk/test/crypto.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.25" +config_file_version = "0.1.26" system: { # One crypto queue pair is required per thread for lockless operation diff --git a/platform/linux-dpdk/test/default-timer.conf b/platform/linux-dpdk/test/default-timer.conf index 495972c5e..3219854de 100644 --- a/platform/linux-dpdk/test/default-timer.conf +++ b/platform/linux-dpdk/test/default-timer.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.25" +config_file_version = "0.1.26" timer: { # Use DPDK default timer API based implementation diff --git a/platform/linux-dpdk/test/example/ipsec_api/Makefile.am b/platform/linux-dpdk/test/example/ipsec_api/Makefile.am index 101c97cdf..2535ad466 100644 --- a/platform/linux-dpdk/test/example/ipsec_api/Makefile.am +++ b/platform/linux-dpdk/test/example/ipsec_api/Makefile.am @@ -19,5 +19,3 @@ clean-local: rm -f $(builddir)/$$f; \ done \ fi - -.NOTPARALLEL: diff --git a/platform/linux-dpdk/test/example/ipsec_crypto/Makefile.am b/platform/linux-dpdk/test/example/ipsec_crypto/Makefile.am index 101c97cdf..2535ad466 100644 --- a/platform/linux-dpdk/test/example/ipsec_crypto/Makefile.am +++ b/platform/linux-dpdk/test/example/ipsec_crypto/Makefile.am @@ -19,5 +19,3 @@ clean-local: rm -f $(builddir)/$$f; \ done \ fi - -.NOTPARALLEL: diff --git a/platform/linux-dpdk/test/process-mode.conf b/platform/linux-dpdk/test/process-mode.conf index a40fd7aa0..827eb6074 100644 --- a/platform/linux-dpdk/test/process-mode.conf +++ b/platform/linux-dpdk/test/process-mode.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.25" +config_file_version = "0.1.26" dpdk: { process_mode_memory_mb = 1024 diff --git a/platform/linux-dpdk/test/sched-basic.conf b/platform/linux-dpdk/test/sched-basic.conf index a11d35706..2c11cb419 100644 --- a/platform/linux-dpdk/test/sched-basic.conf +++ b/platform/linux-dpdk/test/sched-basic.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.25" +config_file_version = "0.1.26" # Test scheduler with an odd spread value and without dynamic load balance sched_basic: { diff --git a/platform/linux-dpdk/test/stash-custom.conf b/platform/linux-dpdk/test/stash-custom.conf index bb9c37fda..62f314c4e 100644 --- a/platform/linux-dpdk/test/stash-custom.conf +++ b/platform/linux-dpdk/test/stash-custom.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-dpdk" -config_file_version = "0.1.25" +config_file_version = "0.1.26" # Test overflow safe stash variant stash: { diff --git a/platform/linux-dpdk/test/validation/api/ml/.gitignore b/platform/linux-dpdk/test/validation/api/ml/.gitignore new file mode 100644 index 000000000..e31f902c4 --- /dev/null +++ b/platform/linux-dpdk/test/validation/api/ml/.gitignore @@ -0,0 +1 @@ +ml_linux diff --git a/platform/linux-dpdk/test/validation/api/ml/Makefile.am b/platform/linux-dpdk/test/validation/api/ml/Makefile.am new file mode 100644 index 000000000..40910d5c6 --- /dev/null +++ b/platform/linux-dpdk/test/validation/api/ml/Makefile.am @@ -0,0 +1,29 @@ +include ../Makefile.inc + +test_PROGRAMS = ml_linux +ml_linux_SOURCES = ../../../../../linux-generic/test/validation/api/ml/ml_linux.c + +EXTRA_DIST = \ + batch_add.onnx \ + simple_linear.onnx + +# If building out-of-tree, make check will not copy the scripts and data to the +# $(builddir) assuming that all commands are run locally. However this prevents +# running tests on a remote target using LOG_COMPILER. +# So copy all script and data files explicitly here. +all-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + if [ -e $(srcdir)/$$f ]; then \ + mkdir -p $(builddir)/$$(dirname $$f); \ + cp -f $(srcdir)/$$f $(builddir)/$$f; \ + fi \ + done \ + fi + +clean-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + rm -f $(builddir)/$$f; \ + done \ + fi diff --git a/platform/linux-dpdk/test/validation/api/ml/README.md b/platform/linux-dpdk/test/validation/api/ml/README.md new file mode 120000 index 000000000..d121a477d --- /dev/null +++ b/platform/linux-dpdk/test/validation/api/ml/README.md @@ -0,0 +1 @@ +../../../../../linux-generic/test/validation/api/ml/README.md
\ No newline at end of file diff --git a/platform/linux-dpdk/test/validation/api/ml/batch_add.onnx b/platform/linux-dpdk/test/validation/api/ml/batch_add.onnx new file mode 120000 index 000000000..b827c0e58 --- /dev/null +++ b/platform/linux-dpdk/test/validation/api/ml/batch_add.onnx @@ -0,0 +1 @@ +../../../../../linux-generic/test/validation/api/ml/batch_add.onnx
\ No newline at end of file diff --git a/platform/linux-dpdk/test/validation/api/ml/batch_add_gen.py b/platform/linux-dpdk/test/validation/api/ml/batch_add_gen.py new file mode 120000 index 000000000..695b6d399 --- /dev/null +++ b/platform/linux-dpdk/test/validation/api/ml/batch_add_gen.py @@ -0,0 +1 @@ +../../../../../linux-generic/test/validation/api/ml/batch_add_gen.py
\ No newline at end of file diff --git a/platform/linux-dpdk/test/validation/api/ml/gen_models.sh b/platform/linux-dpdk/test/validation/api/ml/gen_models.sh new file mode 120000 index 000000000..e9b25d58f --- /dev/null +++ b/platform/linux-dpdk/test/validation/api/ml/gen_models.sh @@ -0,0 +1 @@ +../../../../../linux-generic/test/validation/api/ml/gen_models.sh
\ No newline at end of file diff --git a/platform/linux-dpdk/test/validation/api/ml/requirements.txt b/platform/linux-dpdk/test/validation/api/ml/requirements.txt new file mode 120000 index 000000000..b94d5d389 --- /dev/null +++ b/platform/linux-dpdk/test/validation/api/ml/requirements.txt @@ -0,0 +1 @@ +../../../../../linux-generic/test/validation/api/ml/requirements.txt
\ No newline at end of file diff --git a/platform/linux-dpdk/test/validation/api/ml/simple_linear.onnx b/platform/linux-dpdk/test/validation/api/ml/simple_linear.onnx new file mode 120000 index 000000000..f471922d1 --- /dev/null +++ b/platform/linux-dpdk/test/validation/api/ml/simple_linear.onnx @@ -0,0 +1 @@ +../../../../../linux-generic/test/validation/api/ml/simple_linear.onnx
\ No newline at end of file diff --git a/platform/linux-dpdk/test/validation/api/ml/simple_linear_gen.py b/platform/linux-dpdk/test/validation/api/ml/simple_linear_gen.py new file mode 120000 index 000000000..53fb4f6ed --- /dev/null +++ b/platform/linux-dpdk/test/validation/api/ml/simple_linear_gen.py @@ -0,0 +1 @@ +../../../../../linux-generic/test/validation/api/ml/simple_linear_gen.py
\ No newline at end of file diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index f3707ab3a..11cdb4c64 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -13,6 +13,7 @@ AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/default AM_CPPFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/common AM_CPPFLAGS += $(OPENSSL_CPPFLAGS) +AM_CPPFLAGS += $(ORT_CPPFLAGS) AM_CFLAGS += $(AARCH64CRYPTO_CFLAGS) AM_CFLAGS += $(DPDK_CFLAGS) @@ -90,6 +91,7 @@ odpapiabiarchinclude_HEADERS += \ include-abi/odp/api/abi/init.h \ include-abi/odp/api/abi/ipsec.h \ include-abi/odp/api/abi/ipsec_types.h \ + include-abi/odp/api/abi/ml_types.h \ include-abi/odp/api/abi/packet.h \ include-abi/odp/api/abi/packet_types.h \ include-abi/odp/api/abi/packet_flags.h \ @@ -140,6 +142,7 @@ noinst_HEADERS = \ include/odp_event_validation_internal.h \ include/odp_fdserver_internal.h \ include/odp_forward_typedefs_internal.h \ + include/odp_ml_fp16.h \ include/odp_global_data.h \ include/odp_init_internal.h \ include/odp_ipsec_internal.h \ @@ -228,6 +231,8 @@ __LIB__libodp_linux_la_SOURCES = \ odp_ishmphy.c \ odp_ishmpool.c \ odp_libconfig.c \ + odp_ml_fp16.c \ + odp_ml_quantize.c \ odp_name_table.c \ odp_packet.c \ odp_packet_vector.c \ @@ -297,6 +302,15 @@ __LIB__libodp_linux_la_SOURCES += \ endif endif endif + +if WITH_ML +__LIB__libodp_linux_la_SOURCES += \ + odp_ml.c +else +__LIB__libodp_linux_la_SOURCES += \ + odp_ml_null.c +endif + if ODP_ABI_COMPAT __LIB__libodp_linux_la_SOURCES += \ odp_atomic_api.c \ @@ -345,14 +359,11 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ arch/default/odp/api/abi/wait_until_generic.h \ arch/default/odp/api/abi/wait_until.h endif -noinst_HEADERS += arch/arm/odp_atomic.h \ - arch/arm/odp_cpu.h \ - arch/arm/odp_cpu_idling.h \ - arch/arm/odp_llsc.h \ +noinst_HEADERS += arch/arm/odp_cpu.h \ arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h \ - arch/default/odp_random.h + arch/default/odp_random.h \ + arch/default/odp_wait_until.h endif if ARCH_IS_AARCH64 __LIB__libodp_linux_la_SOURCES += arch/aarch64/odp_atomic.c \ @@ -380,9 +391,8 @@ endif noinst_HEADERS += arch/aarch64/odp_atomic.h \ arch/aarch64/odp_cpu.h \ arch/aarch64/cpu_flags.h \ - arch/aarch64/odp_cpu_idling.h \ - arch/aarch64/odp_llsc.h \ - arch/aarch64/odp_random.h + arch/aarch64/odp_random.h \ + arch/aarch64/odp_wait_until.h endif if ARCH_IS_DEFAULT __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ @@ -405,8 +415,8 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h \ - arch/default/odp_random.h + arch/default/odp_random.h \ + arch/default/odp_wait_until.h endif if ARCH_IS_POWERPC __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ @@ -429,8 +439,8 @@ odpapiabiarchinclude_HEADERS += arch/default/odp/api/abi/atomic_generic.h \ endif noinst_HEADERS += arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h \ - arch/default/odp_random.h + arch/default/odp_random.h \ + arch/default/odp_wait_until.h endif if ARCH_IS_X86 __LIB__libodp_linux_la_SOURCES += arch/default/odp_atomic.c \ @@ -460,7 +470,7 @@ noinst_HEADERS += arch/x86/cpu_flags.h \ arch/x86/odp_random.h \ arch/default/odp_atomic.h \ arch/default/odp_cpu.h \ - arch/default/odp_cpu_idling.h + arch/default/odp_wait_until.h endif if ODP_PKTIO_PCAP @@ -476,6 +486,7 @@ __LIB__libodp_linux_la_LIBADD += $(PTHREAD_LIBS) __LIB__libodp_linux_la_LIBADD += $(TIMER_LIBS) __LIB__libodp_linux_la_LIBADD += $(LIBXDP_LIBS) __LIB__libodp_linux_la_LIBADD += $(IPSEC_MB_LIBS) +__LIB__libodp_linux_la_LIBADD += $(ORT_LIBS) if ODP_PKTIO_PCAP __LIB__libodp_linux_la_LIBADD += $(PCAP_LIBS) diff --git a/platform/linux-generic/arch/aarch64/odp_cpu.h b/platform/linux-generic/arch/aarch64/odp_cpu.h index 84bc4dffd..ad8b36d87 100644 --- a/platform/linux-generic/arch/aarch64/odp_cpu.h +++ b/platform/linux-generic/arch/aarch64/odp_cpu.h @@ -14,6 +14,7 @@ #endif #include <odp_debug_internal.h> +#include <odp_types_internal.h> /* * Use LLD/SCD atomic primitives instead of lock-based code path in llqueue @@ -31,20 +32,6 @@ */ #define CONFIG_DMBSTR -/* - * Use ARM event signalling mechanism - * Event signalling minimises spinning (busy waiting) which decreases - * cache coherency traffic when spinning on shared locations (thus faster and - * more scalable) and enables the CPU to enter a sleep state (lower power - * consumption). - */ -#define CONFIG_WFE - -static inline void _odp_dmb(void) -{ - __asm__ volatile("dmb" : : : "memory"); -} - /* Only ARMv8 supports DMB ISHLD */ /* A load only barrier is much cheaper than full barrier */ #define _odp_release_barrier(ro) \ @@ -55,9 +42,156 @@ do { \ __asm__ volatile("dmb ish" ::: "memory"); \ } while (0) -#include "odp_llsc.h" +static inline uint16_t ll8(uint8_t *var, int mm) +{ + uint16_t old; + + _ODP_ASSERT(mm == __ATOMIC_ACQUIRE || mm == __ATOMIC_RELAXED); + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxrb %w0, [%1]" + : "=&r" (old) + : "r" (var) + : "memory"); + else + __asm__ volatile("ldxrb %w0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + return old; +} + +static inline uint32_t ll32(uint32_t *var, int mm) +{ + uint32_t old; + + _ODP_ASSERT(mm == __ATOMIC_ACQUIRE || mm == __ATOMIC_RELAXED); + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxr %w0, [%1]" + : "=&r" (old) + : "r" (var) + : "memory"); + else + __asm__ volatile("ldxr %w0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + return old; +} + +/* Return 0 on success, 1 on failure */ +static inline uint32_t sc32(uint32_t *var, uint32_t neu, int mm) +{ + uint32_t ret; + + _ODP_ASSERT(mm == __ATOMIC_RELEASE || mm == __ATOMIC_RELAXED); + + if (mm == __ATOMIC_RELEASE) + __asm__ volatile("stlxr %w0, %w1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : "memory"); + else + __asm__ volatile("stxr %w0, %w1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : ); + return ret; +} + +static inline uint64_t ll64(uint64_t *var, int mm) +{ + uint64_t old; + + _ODP_ASSERT(mm == __ATOMIC_ACQUIRE || mm == __ATOMIC_RELAXED); + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxr %0, [%1]" + : "=&r" (old) + : "r" (var) + : "memory"); + else + __asm__ volatile("ldxr %0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + return old; +} + +/* Return 0 on success, 1 on failure */ +static inline uint32_t sc64(uint64_t *var, uint64_t neu, int mm) +{ + uint32_t ret; + + _ODP_ASSERT(mm == __ATOMIC_RELEASE || mm == __ATOMIC_RELAXED); + + if (mm == __ATOMIC_RELEASE) + __asm__ volatile("stlxr %w0, %1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : "memory"); + else + __asm__ volatile("stxr %w0, %1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : ); + return ret; +} + +union i128 { + _odp_u128_t i128; + int64_t i64[2]; +}; + +static inline _odp_u128_t lld(_odp_u128_t *var, int mm) +{ + union i128 old; + + _ODP_ASSERT(mm == __ATOMIC_ACQUIRE || mm == __ATOMIC_RELAXED); + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxp %0, %1, [%2]" + : "=&r" (old.i64[0]), "=&r" (old.i64[1]) + : "r" (var) + : "memory"); + else + __asm__ volatile("ldxp %0, %1, [%2]" + : "=&r" (old.i64[0]), "=&r" (old.i64[1]) + : "r" (var) + : ); + return old.i128; +} + +/* Return 0 on success, 1 on failure */ +static inline uint32_t scd(_odp_u128_t *var, _odp_u128_t neu, int mm) +{ + uint32_t ret; + + _ODP_ASSERT(mm == __ATOMIC_RELEASE || mm == __ATOMIC_RELAXED); + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" + if (mm == __ATOMIC_RELEASE) + __asm__ volatile("stlxp %w0, %1, %2, [%3]" + : "=&r" (ret) + : "r" (((*(union i128 *)&neu)).i64[0]), + "r" (((*(union i128 *)&neu)).i64[1]), + "r" (var) + : "memory"); + else + __asm__ volatile("stxp %w0, %1, %2, [%3]" + : "=&r" (ret) + : "r" (((*(union i128 *)&neu)).i64[0]), + "r" (((*(union i128 *)&neu)).i64[1]), + "r" (var) + : ); +#pragma GCC diagnostic pop + return ret; +} + #include "odp_atomic.h" -#include "odp_cpu_idling.h" +#include "odp_wait_until.h" #ifdef __ARM_FEATURE_UNALIGNED #define _ODP_UNALIGNED 1 diff --git a/platform/linux-generic/arch/aarch64/odp_cpu_idling.h b/platform/linux-generic/arch/aarch64/odp_cpu_idling.h deleted file mode 100644 index a6cea8c63..000000000 --- a/platform/linux-generic/arch/aarch64/odp_cpu_idling.h +++ /dev/null @@ -1,39 +0,0 @@ -/* Copyright (c) 2017, ARM Limited. All rights reserved. - * - * Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_CPU_IDLING_H -#define PLATFORM_LINUXGENERIC_ARCH_ARM_CPU_IDLING_H - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H -#error This file should not be included directly, please include odp_cpu.h -#endif - -#ifndef CONFIG_WFE - -#include "../default/odp_cpu_idling.h" - -#else /* CONFIG_WFE */ - -static inline void sevl(void) -{ - __asm__ volatile("sevl" : : : ); -} - -static inline int wfe(void) -{ - __asm__ volatile("wfe" : : : "memory"); - return 1; -} - -#define monitor128(addr, mo) lld((addr), (mo)) -#define monitor64(addr, mo) ll64((addr), (mo)) -#define monitor32(addr, mo) ll32((addr), (mo)) -#define monitor8(addr, mo) ll8((addr), (mo)) -#endif /* CONFIG_WFE */ - -#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_CPU_IDLING_H */ diff --git a/platform/linux-generic/arch/aarch64/odp_llsc.h b/platform/linux-generic/arch/aarch64/odp_llsc.h deleted file mode 100644 index 498785bd4..000000000 --- a/platform/linux-generic/arch/aarch64/odp_llsc.h +++ /dev/null @@ -1,170 +0,0 @@ -/* Copyright (c) 2017, ARM Limited. All rights reserved. - * - * Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_LLSC_H -#define PLATFORM_LINUXGENERIC_ARCH_ARM_LLSC_H - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H -#error This file should not be included directly, please include odp_cpu.h -#endif - -#include <odp_types_internal.h> - -static inline uint16_t ll8(uint8_t *var, int mm) -{ - uint16_t old; - - if (mm == __ATOMIC_ACQUIRE) - __asm__ volatile("ldaxrb %w0, [%1]" - : "=&r" (old) - : "r" (var) - : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm__ volatile("ldxrb %w0, [%1]" - : "=&r" (old) - : "r" (var) - : ); - else - _ODP_ABORT(); - return old; -} - -static inline uint32_t ll32(uint32_t *var, int mm) -{ - uint32_t old; - - if (mm == __ATOMIC_ACQUIRE) - __asm__ volatile("ldaxr %w0, [%1]" - : "=&r" (old) - : "r" (var) - : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm__ volatile("ldxr %w0, [%1]" - : "=&r" (old) - : "r" (var) - : ); - else - _ODP_ABORT(); - return old; -} - -/* Return 0 on success, 1 on failure */ -static inline uint32_t sc32(uint32_t *var, uint32_t neu, int mm) -{ - uint32_t ret; - - if (mm == __ATOMIC_RELEASE) - __asm__ volatile("stlxr %w0, %w1, [%2]" - : "=&r" (ret) - : "r" (neu), "r" (var) - : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm__ volatile("stxr %w0, %w1, [%2]" - : "=&r" (ret) - : "r" (neu), "r" (var) - : ); - else - _ODP_ABORT(); - return ret; -} - -static inline uint64_t ll(uint64_t *var, int mm) -{ - uint64_t old; - - if (mm == __ATOMIC_ACQUIRE) - __asm__ volatile("ldaxr %0, [%1]" - : "=&r" (old) - : "r" (var) - : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm__ volatile("ldxr %0, [%1]" - : "=&r" (old) - : "r" (var) - : ); - else - _ODP_ABORT(); - return old; -} - -#define ll64(a, b) ll((a), (b)) - -/* Return 0 on success, 1 on failure */ -static inline uint32_t sc(uint64_t *var, uint64_t neu, int mm) -{ - uint32_t ret; - - if (mm == __ATOMIC_RELEASE) - __asm__ volatile("stlxr %w0, %1, [%2]" - : "=&r" (ret) - : "r" (neu), "r" (var) - : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm__ volatile("stxr %w0, %1, [%2]" - : "=&r" (ret) - : "r" (neu), "r" (var) - : ); - else - _ODP_ABORT(); - return ret; -} - -#define sc64(a, b, c) sc((a), (b), (c)) - -union i128 { - _odp_u128_t i128; - int64_t i64[2]; -}; - -static inline _odp_u128_t lld(_odp_u128_t *var, int mm) -{ - union i128 old; - - if (mm == __ATOMIC_ACQUIRE) - __asm__ volatile("ldaxp %0, %1, [%2]" - : "=&r" (old.i64[0]), "=&r" (old.i64[1]) - : "r" (var) - : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm__ volatile("ldxp %0, %1, [%2]" - : "=&r" (old.i64[0]), "=&r" (old.i64[1]) - : "r" (var) - : ); - else - _ODP_ABORT(); - return old.i128; -} - -/* Return 0 on success, 1 on failure */ -static inline uint32_t scd(_odp_u128_t *var, _odp_u128_t neu, int mm) -{ - uint32_t ret; - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" - if (mm == __ATOMIC_RELEASE) - __asm__ volatile("stlxp %w0, %1, %2, [%3]" - : "=&r" (ret) - : "r" (((*(union i128 *)&neu)).i64[0]), - "r" (((*(union i128 *)&neu)).i64[1]), - "r" (var) - : "memory"); - else if (mm == __ATOMIC_RELAXED) - __asm__ volatile("stxp %w0, %1, %2, [%3]" - : "=&r" (ret) - : "r" (((*(union i128 *)&neu)).i64[0]), - "r" (((*(union i128 *)&neu)).i64[1]), - "r" (var) - : ); - else - _ODP_ABORT(); -#pragma GCC diagnostic pop - return ret; -} - -#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_LLSC_H */ diff --git a/platform/linux-generic/arch/aarch64/odp_wait_until.h b/platform/linux-generic/arch/aarch64/odp_wait_until.h new file mode 100644 index 000000000..eca3f9ce5 --- /dev/null +++ b/platform/linux-generic/arch/aarch64/odp_wait_until.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017 ARM Limited + * Copyright (c) 2017-2018 Linaro Limited + * Copyright (c) 2024 Nokia + */ + +#ifndef ODP_AARCH64_WAIT_UNTIL_H_ +#define ODP_AARCH64_WAIT_UNTIL_H_ + +#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H +#error This file should not be included directly, please include odp_cpu.h +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/cpu.h> + +#include <odp_cpu.h> + +#include <stdint.h> + +static inline void _odp_sevl(void) +{ + __asm__ volatile("sevl" : : : ); +} + +static inline int _odp_wfe(void) +{ + __asm__ volatile("wfe" : : : "memory"); + return 1; +} + +#define _odp_monitor_u8(addr, mo) ll8((addr), (mo)) +#define _odp_monitor_u32(addr, mo) ll32((addr), (mo)) +#define _odp_monitor_u64(addr, mo) ll64((addr), (mo)) +#define _odp_monitor_u128(addr, mo) lld((addr), (mo)) + +#if ATOM_BITSET_SIZE <= 32 +static inline bitset_t _odp_bitset_monitor(bitset_t *bs, int mo) +{ + return _odp_monitor_u32(bs, mo); +} +#elif ATOM_BITSET_SIZE <= 64 +static inline bitset_t _odp_bitset_monitor(bitset_t *bs, int mo) +{ + return _odp_monitor_u64(bs, mo); +} +#elif ATOM_BITSET_SIZE <= 128 +static inline bitset_t _odp_bitset_monitor(bitset_t *bs, int mo) +{ + return _odp_monitor_u128(bs, mo); +} +#else +#error Unsupported size of bit sets (ATOM_BITSET_SIZE) +#endif + +/** + * The _odp_wait_until_eq_*() functions defined in this header are intended to + * be used only with the scalable scheduler and queue implementations. Even + * though these functions use standard non-atomic parameter types, the + * parameters must only be operated using atomic operations. If new functions + * are added to this file, they should use _odp_wait_until_equal_*() prefix and + * atomic parameter types. + */ + +static inline void _odp_wait_until_eq_u32(uint32_t *val, uint32_t expected) +{ + _odp_sevl(); + while (_odp_wfe() && _odp_monitor_u32(val, __ATOMIC_RELAXED) != expected) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_bitset(bitset_t *val, bitset_t expected) +{ + _odp_sevl(); + while (_odp_wfe() && _odp_bitset_monitor(val, __ATOMIC_RELAXED != expected)) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_acq_u8(uint8_t *val, uint8_t expected) +{ + _odp_sevl(); + while (_odp_wfe() && _odp_monitor_u8(val, __ATOMIC_ACQUIRE) != expected) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_acq_u32(uint32_t *val, uint32_t expected) +{ + _odp_sevl(); + while (_odp_wfe() && _odp_monitor_u32(val, __ATOMIC_ACQUIRE) != expected) + odp_cpu_pause(); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/arm/odp_atomic.h b/platform/linux-generic/arch/arm/odp_atomic.h deleted file mode 100644 index e400f52d4..000000000 --- a/platform/linux-generic/arch/arm/odp_atomic.h +++ /dev/null @@ -1,109 +0,0 @@ -/* Copyright (c) 2017-2021, ARM Limited. All rights reserved. - * Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H -#define PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H -#error This file should not be included directly, please include odp_cpu.h -#endif - -#include <odp_types_internal.h> -#include <limits.h> - -#ifdef CONFIG_DMBSTR - -#define atomic_store_release(loc, val, ro) \ -do { \ - _odp_release_barrier(ro); \ - __atomic_store_n(loc, val, __ATOMIC_RELAXED); \ -} while (0) - -#else - -#define atomic_store_release(loc, val, ro) \ - __atomic_store_n(loc, val, __ATOMIC_RELEASE) - -#endif /* CONFIG_DMBSTR */ - -/** Atomic bit set operations with memory ordering */ -#if __GCC_ATOMIC_LLONG_LOCK_FREE == 2 && \ - __SIZEOF_LONG_LONG__ != __SIZEOF_LONG__ -typedef unsigned long long bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG_LONG__) - -#elif __GCC_ATOMIC_LONG_LOCK_FREE == 2 && __SIZEOF_LONG__ != __SIZEOF_INT__ -typedef unsigned long bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG__) - -#elif __GCC_ATOMIC_INT_LOCK_FREE == 2 -typedef unsigned int bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) - -#else -/* Target does not support lock-free atomic operations */ -typedef unsigned int bitset_t; -#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) -#endif - -#if ATOM_BITSET_SIZE <= 32 - -static inline bitset_t bitset_mask(uint32_t bit) -{ - return 1UL << bit; -} - -#elif ATOM_BITSET_SIZE <= 64 - -static inline bitset_t bitset_mask(uint32_t bit) -{ - return 1ULL << bit; -} - -#elif ATOM_BITSET_SIZE <= 128 - -static inline bitset_t bitset_mask(uint32_t bit) -{ - if (bit < 64) - return 1ULL << bit; - else - return (_odp_u128_t)(1ULL << (bit - 64)) << 64; -} - -#else -#error Unsupported size of bit sets (ATOM_BITSET_SIZE) -#endif - -static inline bitset_t atom_bitset_load(bitset_t *bs, int mo) -{ - return __atomic_load_n(bs, mo); -} - -static inline void atom_bitset_set(bitset_t *bs, uint32_t bit, int mo) -{ - (void)__atomic_fetch_or(bs, bitset_mask(bit), mo); -} - -static inline void atom_bitset_clr(bitset_t *bs, uint32_t bit, int mo) -{ - (void)__atomic_fetch_and(bs, ~bitset_mask(bit), mo); -} - -static inline bitset_t atom_bitset_xchg(bitset_t *bs, bitset_t neu, int mo) -{ - return __atomic_exchange_n(bs, neu, mo); -} - -static inline bitset_t atom_bitset_cmpxchg(bitset_t *bs, bitset_t *old, - bitset_t neu, bool weak, - int mo_success, int mo_failure) -{ - return __atomic_compare_exchange_n(bs, old, neu, weak, mo_success, - mo_failure); -} - -#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H */ diff --git a/platform/linux-generic/arch/arm/odp_cpu.h b/platform/linux-generic/arch/arm/odp_cpu.h index 82d47325f..6b2674736 100644 --- a/platform/linux-generic/arch/arm/odp_cpu.h +++ b/platform/linux-generic/arch/arm/odp_cpu.h @@ -31,26 +31,52 @@ */ #define CONFIG_DMBSTR -/* - * Use ARM event signalling mechanism - * Event signalling minimises spinning (busy waiting) which decreases - * cache coherency traffic when spinning on shared locations (thus faster and - * more scalable) and enables the CPU to enter a sleep state (lower power - * consumption). - */ -/* #define CONFIG_WFE */ +static inline uint64_t lld(uint64_t *var, int mm) +{ + uint64_t old; -static inline void _odp_dmb(void) + __asm__ volatile("ldrexd %0, %H0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + /* Barrier after an acquiring load */ + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("dmb" : : : "memory"); + return old; +} + +/* Return 0 on success, 1 on failure */ +static inline uint32_t scd(uint64_t *var, uint64_t neu, int mm) { - __asm__ volatile("dmb" : : : "memory"); + uint32_t ret; + + /* Barrier before a releasing store */ + if (mm == __ATOMIC_RELEASE) + __asm__ volatile("dmb" : : : "memory"); + __asm__ volatile("strexd %0, %1, %H1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : ); + return ret; } -#define _odp_release_barrier(ro) \ - __atomic_thread_fence(__ATOMIC_RELEASE) +#ifdef CONFIG_DMBSTR + +#define atomic_store_release(loc, val, ro) \ +do { \ + __atomic_thread_fence(__ATOMIC_RELEASE); \ + __atomic_store_n(loc, val, __ATOMIC_RELAXED); \ +} while (0) + +#else + +#define atomic_store_release(loc, val, ro) \ + __atomic_store_n(loc, val, __ATOMIC_RELEASE) + +#endif /* CONFIG_DMBSTR */ -#include "odp_llsc.h" -#include "odp_atomic.h" -#include "odp_cpu_idling.h" +#include "../default/odp_atomic.h" +#include "../default/odp_wait_until.h" #ifdef __ARM_FEATURE_UNALIGNED #define _ODP_UNALIGNED 1 diff --git a/platform/linux-generic/arch/arm/odp_cpu_idling.h b/platform/linux-generic/arch/arm/odp_cpu_idling.h deleted file mode 100644 index a6cea8c63..000000000 --- a/platform/linux-generic/arch/arm/odp_cpu_idling.h +++ /dev/null @@ -1,39 +0,0 @@ -/* Copyright (c) 2017, ARM Limited. All rights reserved. - * - * Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_CPU_IDLING_H -#define PLATFORM_LINUXGENERIC_ARCH_ARM_CPU_IDLING_H - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H -#error This file should not be included directly, please include odp_cpu.h -#endif - -#ifndef CONFIG_WFE - -#include "../default/odp_cpu_idling.h" - -#else /* CONFIG_WFE */ - -static inline void sevl(void) -{ - __asm__ volatile("sevl" : : : ); -} - -static inline int wfe(void) -{ - __asm__ volatile("wfe" : : : "memory"); - return 1; -} - -#define monitor128(addr, mo) lld((addr), (mo)) -#define monitor64(addr, mo) ll64((addr), (mo)) -#define monitor32(addr, mo) ll32((addr), (mo)) -#define monitor8(addr, mo) ll8((addr), (mo)) -#endif /* CONFIG_WFE */ - -#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_CPU_IDLING_H */ diff --git a/platform/linux-generic/arch/arm/odp_llsc.h b/platform/linux-generic/arch/arm/odp_llsc.h deleted file mode 100644 index 2fea6a0dc..000000000 --- a/platform/linux-generic/arch/arm/odp_llsc.h +++ /dev/null @@ -1,96 +0,0 @@ -/* Copyright (c) 2017, ARM Limited. All rights reserved. - * - * Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_LLSC_H -#define PLATFORM_LINUXGENERIC_ARCH_ARM_LLSC_H - -#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H -#error This file should not be included directly, please include odp_cpu.h -#endif - -static inline uint32_t ll8(uint8_t *var, int mm) -{ - uint8_t old; - - __asm__ volatile("ldrexb %0, [%1]" - : "=&r" (old) - : "r" (var) - : ); - /* Barrier after an acquiring load */ - if (mm == __ATOMIC_ACQUIRE) - _odp_dmb(); - return old; -} - -static inline uint32_t ll(uint32_t *var, int mm) -{ - uint32_t old; - - __asm__ volatile("ldrex %0, [%1]" - : "=&r" (old) - : "r" (var) - : ); - /* Barrier after an acquiring load */ - if (mm == __ATOMIC_ACQUIRE) - _odp_dmb(); - return old; -} - -#define ll32(a, b) ll((a), (b)) - -/* Return 0 on success, 1 on failure */ -static inline uint32_t sc(uint32_t *var, uint32_t neu, int mm) -{ - uint32_t ret; - - /* Barrier before a releasing store */ - if (mm == __ATOMIC_RELEASE) - _odp_dmb(); - __asm__ volatile("strex %0, %1, [%2]" - : "=&r" (ret) - : "r" (neu), "r" (var) - : ); - return ret; -} - -#define sc32(a, b, c) sc((a), (b), (c)) - -static inline uint64_t lld(uint64_t *var, int mm) -{ - uint64_t old; - - __asm__ volatile("ldrexd %0, %H0, [%1]" - : "=&r" (old) - : "r" (var) - : ); - /* Barrier after an acquiring load */ - if (mm == __ATOMIC_ACQUIRE) - _odp_dmb(); - return old; -} - -#define ll64(a, b) lld((a), (b)) - -/* Return 0 on success, 1 on failure */ -static inline uint32_t scd(uint64_t *var, uint64_t neu, int mm) -{ - uint32_t ret; - - /* Barrier before a releasing store */ - if (mm == __ATOMIC_RELEASE) - _odp_dmb(); - __asm__ volatile("strexd %0, %1, %H1, [%2]" - : "=&r" (ret) - : "r" (neu), "r" (var) - : ); - return ret; -} - -#define sc64(a, b, c) scd((a), (b), (c)) - -#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_LLSC_H */ diff --git a/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h b/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h index af435e495..c6ed86363 100644 --- a/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h +++ b/platform/linux-generic/arch/default/odp/api/abi/atomic_generic.h @@ -197,11 +197,11 @@ static inline int _odp_atomic_cas_acq_rel_u128(odp_atomic_u128_t *atom, odp_u128 #define ATOMIC_CAS_OP_128(ret_ptr, old_val, new_val) \ __extension__ ({ \ int *_ret_ptr = ret_ptr; \ - odp_u128_t *_old_val = old_val; \ - odp_u128_t _new_val = new_val; \ - if (((_atom)->v.u64[0] == (_old_val)->u64[0]) && \ - ((_atom)->v.u64[1] == (_old_val)->u64[1])) { \ - (_atom)->v = (_new_val); \ + odp_u128_t *_cas_old = old_val; \ + odp_u128_t _cas_new = new_val; \ + if (((_atom)->v.u64[0] == (_cas_old)->u64[0]) && \ + ((_atom)->v.u64[1] == (_cas_old)->u64[1])) { \ + (_atom)->v = (_cas_new); \ *(_ret_ptr) = 1; \ } else { \ *(_ret_ptr) = 0; \ diff --git a/platform/linux-generic/arch/default/odp_cpu.h b/platform/linux-generic/arch/default/odp_cpu.h index 821956819..6b10966c6 100644 --- a/platform/linux-generic/arch/default/odp_cpu.h +++ b/platform/linux-generic/arch/default/odp_cpu.h @@ -21,6 +21,6 @@ __atomic_store_n(loc, val, __ATOMIC_RELEASE) #include "odp_atomic.h" -#include "odp_cpu_idling.h" +#include "odp_wait_until.h" #endif diff --git a/platform/linux-generic/arch/default/odp_cpu_idling.h b/platform/linux-generic/arch/default/odp_cpu_idling.h deleted file mode 100644 index 9d23ad20d..000000000 --- a/platform/linux-generic/arch/default/odp_cpu_idling.h +++ /dev/null @@ -1,31 +0,0 @@ -/* Copyright (c) 2017, ARM Limited. All rights reserved. - * - * Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_DEFAULT_CPU_IDLING_H_ -#define ODP_DEFAULT_CPU_IDLING_H_ - -/****************************************************************************** - * Idle mgmt - *****************************************************************************/ - -static inline void sevl(void) -{ - /* empty */ -} - -static inline int wfe(void) -{ - return 1; -} - -#define monitor128(addr, mo) __atomic_load_n((addr), (mo)) -#define monitor64(addr, mo) __atomic_load_n((addr), (mo)) -#define monitor32(addr, mo) __atomic_load_n((addr), (mo)) -#define monitor8(addr, mo) __atomic_load_n((addr), (mo)) - -#endif diff --git a/platform/linux-generic/arch/default/odp_wait_until.h b/platform/linux-generic/arch/default/odp_wait_until.h new file mode 100644 index 000000000..c51f4355e --- /dev/null +++ b/platform/linux-generic/arch/default/odp_wait_until.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2024 Nokia + */ + +#ifndef ODP_DEFAULT_WAIT_UNTIL_H_ +#define ODP_DEFAULT_WAIT_UNTIL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/plat/cpu_inlines.h> + +#include <stdint.h> + +/** + * The _odp_wait_until_eq_*() functions defined in this header are intended to + * be used only with the scalable scheduler and queue implementations. Even + * though these functions use standard non-atomic parameter types, the + * parameters must only be operated using atomic operations. If new functions + * are added to this file, they should use _odp_wait_until_equal_*() prefix and + * atomic parameter types. + */ + +static inline void _odp_wait_until_eq_u32(uint32_t *val, uint32_t expected) +{ + while (__atomic_load_n(val, __ATOMIC_RELAXED) != expected) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_bitset(bitset_t *val, bitset_t expected) +{ + while (__atomic_load_n(val, __ATOMIC_RELAXED) != expected) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_acq_u8(uint8_t *val, uint8_t expected) +{ + while (__atomic_load_n(val, __ATOMIC_ACQUIRE) != expected) + odp_cpu_pause(); +} + +static inline void _odp_wait_until_eq_acq_u32(uint32_t *val, uint32_t expected) +{ + while (__atomic_load_n(val, __ATOMIC_ACQUIRE) != expected) + odp_cpu_pause(); +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/x86/odp_time_cpu.c b/platform/linux-generic/arch/x86/odp_time_cpu.c index aa00ac04e..ab897296d 100644 --- a/platform/linux-generic/arch/x86/odp_time_cpu.c +++ b/platform/linux-generic/arch/x86/odp_time_cpu.c @@ -1,7 +1,6 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2018 Linaro Limited + * Copyright (c) 2024 Nokia */ #include <odp_posix_extensions.h> @@ -14,42 +13,77 @@ #include <odp_debug_internal.h> #include <time.h> +#include <errno.h> +#include <string.h> -/* Measure TSC frequency. Frequency information registers are defined for x86, - * but those are often not enumerated. */ +static int nwait(uint64_t nsec) +{ + struct timespec ts1, ts2; + uint64_t diff; + + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts1)) + return 1; + + do { + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts2)) + return 1; + + diff = (ts2.tv_sec - ts1.tv_sec) * ODP_TIME_SEC_IN_NS + + ts2.tv_nsec - ts1.tv_nsec; + } while (diff < nsec); + + return 0; +} + +static void sort(uint64_t values[], int num) +{ + for (int n = 0; n < num; n++) { + for (int i = n + 1; i < num; i++) { + if (values[i] < values[n]) { + uint64_t tmp = values[i]; + + values[i] = values[n]; + values[n] = tmp; + } + } + } +} + +static uint64_t median(uint64_t values[], int num) +{ + sort(values, num); + if (num % 2 == 0) + return (values[num / 2 - 1] + values[num / 2]) / 2; + else + return values[num / 2]; +} + +/* Measure TSC frequency. */ uint64_t _odp_time_cpu_global_freq(void) { - struct timespec sleep, ts1, ts2; - uint64_t t1, t2, ts_nsec, cycles, hz; + struct timespec ts1, ts2; + uint64_t t1, t2, ts_nsec, cycles; int i; - uint64_t avg = 0; - int rounds = 3; + const int rounds = 6; /* first round is warmup */ int warm_up = 1; + uint64_t hz[rounds]; for (i = 0; i < rounds; i++) { - sleep.tv_sec = 0; + uint64_t wait_nsec = ODP_TIME_SEC_IN_NS / 50; if (warm_up) - sleep.tv_nsec = ODP_TIME_SEC_IN_NS / 1000; - else - sleep.tv_nsec = ODP_TIME_SEC_IN_NS / 4; + wait_nsec = ODP_TIME_SEC_IN_NS / 1000; - if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts1)) { - _ODP_ERR("clock_gettime() failed\n"); - return 0; - } + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts1)) + goto err_out; t1 = _odp_time_cpu_global(); - if (nanosleep(&sleep, NULL) < 0) { - _ODP_ERR("nanosleep() failed\n"); - return 0; - } + if (nwait(wait_nsec)) + goto err_out; - if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts2)) { - _ODP_ERR("clock_gettime() failed\n"); - return 0; - } + if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts2)) + goto err_out; t2 = _odp_time_cpu_global(); @@ -58,13 +92,15 @@ uint64_t _odp_time_cpu_global_freq(void) cycles = t2 - t1; - hz = (cycles * ODP_TIME_SEC_IN_NS) / ts_nsec; + hz[i] = (cycles * ODP_TIME_SEC_IN_NS) / ts_nsec; if (warm_up) warm_up = 0; - else - avg += hz; } - return avg / (rounds - 1); + return median(&hz[1], rounds - 1); + +err_out: + _ODP_ERR("clock_gettime() failed (%s)\n", strerror(errno)); + return 0; } diff --git a/platform/linux-generic/example/Makefile.am b/platform/linux-generic/example/Makefile.am new file mode 100644 index 000000000..84f337387 --- /dev/null +++ b/platform/linux-generic/example/Makefile.am @@ -0,0 +1,5 @@ +SUBDIRS = + +if WITH_ML +SUBDIRS += ml +endif diff --git a/platform/linux-generic/example/ml/.gitignore b/platform/linux-generic/example/ml/.gitignore new file mode 100644 index 000000000..d845f6bb5 --- /dev/null +++ b/platform/linux-generic/example/ml/.gitignore @@ -0,0 +1,5 @@ +model_explorer +simple_linear +mnist +*.log +*.trs diff --git a/platform/linux-generic/example/ml/Makefile.am b/platform/linux-generic/example/ml/Makefile.am new file mode 100644 index 000000000..3692b704e --- /dev/null +++ b/platform/linux-generic/example/ml/Makefile.am @@ -0,0 +1,46 @@ +include $(top_srcdir)/example/Makefile.inc + +LDADD += -lm + +bin_PROGRAMS = model_explorer simple_linear mnist + +simple_linear_SOURCES = simple_linear.c model_read.c model_read.h +model_explorer_SOURCES = model_explorer.c model_read.c model_read.h +mnist_SOURCES = mnist.c model_read.c model_read.h + +EXTRA_DIST = \ + odp_ml_run_mnist.sh \ + example_digit.csv \ + mnist-12.onnx \ + odp_ml_run_model_explorer.sh \ + odp_ml_run_simple_linear.sh \ + simple_linear.onnx \ + README.md + +if test_example +TESTS = \ + odp_ml_run_mnist.sh \ + odp_ml_run_model_explorer.sh \ + odp_ml_run_simple_linear.sh +endif + +# If building out-of-tree, make check will not copy the scripts and data to the +# $(builddir) assuming that all commands are run locally. However this prevents +# running tests on a remote target using LOG_COMPILER. +# So copy all script and data files explicitly here. +all-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + if [ -e $(srcdir)/$$f ]; then \ + mkdir -p $(builddir)/$$(dirname $$f); \ + cp -f $(srcdir)/$$f $(builddir)/$$f; \ + fi \ + done \ + fi + +clean-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + rm -f $(builddir)/$$f; \ + done \ + fi diff --git a/platform/linux-generic/example/ml/README.md b/platform/linux-generic/example/ml/README.md new file mode 100644 index 000000000..fc6a57c0a --- /dev/null +++ b/platform/linux-generic/example/ml/README.md @@ -0,0 +1,94 @@ +# ML examples + +Machine Learning API examples demonstrate how to use ODP ML API in different tasks: +for example simple linear computation and predicting a handwritten digit in +a given image. + +## Simple Linear + +This example runs on a very simple model of form y = 3 * x + 4 where x is given +as the second argument. + +### Generate model + +```bash +python3 <odp_directory>/platform/linux-generic/test/validation/api/ml/simple_linear_gen.py +``` + +### Run simple linear + +```bash +$ ./simple_linear 3 +. +. +. +y = 3 * 3 + 4: 13 +. +``` + +Or run the program with multiple threads, each thread inferences on one x given in +the input. Thus, the number of threads is the number of numbers in the second argument. + +```bash +$ ./simple_linear [2,4,5] +. +. +. +y = 3 * 2 + 4: 10 +y = 3 * 5 + 4: 19 +y = 3 * 4 + 4: 16 +. +``` + +## MNIST + +This example predicts a handwritten digit in a given image. Refer to +https://github.com/onnx/models/tree/main/validated/vision/classification/mnist +for more information. The model file is from +https://github.com/onnx/models/raw/main/validated/vision/classification/mnist/model/mnist-12.onnx +(SPDX-License-Identifier: MIT). + +### Prepare input data + +The input image is stored in a csv file which contains, comma separated, the +digit label (a number from 0 to 9) and the 784 pixel values (a number from 0 to +255). Pixel order is left to right and then top down. The MNIST dataset is +available in this format at https://www.kaggle.com/oddrationale/mnist-in-csv. + +### Run mnist + +```bash +$ ./mnist mnist-12.onnx example_digit.csv +. +. +. +predicted_digit: 4, expected_digit: 4 +. +``` + +## Model Explorer + +The example prints basic model information. + +### Run model_explorer + +```bash +$ ./model_explorer simple_linear.onnx +. +. +. +Model info +---------- + Model handle: 0x7fe8426ce1d8 + Name: model-explorer + Model version: 1 + Model interface version: 0 + Index: 0 + Number of inputs: 1 + Input[0]: Name: x, Data_type: int32, Shape: static [1], Size: 4 + Number of outputs: 1 + Output[0]: Name: y, Data_type: int32, Shape: static [1], Size: 4 +. +. +. +``` diff --git a/platform/linux-generic/example/ml/example_digit.csv b/platform/linux-generic/example/ml/example_digit.csv new file mode 100644 index 000000000..2ab0f4a0c --- /dev/null +++ b/platform/linux-generic/example/ml/example_digit.csv @@ -0,0 +1 @@ +4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,55,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,36,215,98,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,36,249,144,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,34,246,148,0,0,0,0,0,0,0,7,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,39,255,139,0,0,0,0,0,0,2,95,117,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,51,255,97,0,0,0,0,0,0,8,203,211,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,70,255,58,0,0,0,0,0,0,13,238,167,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,111,255,23,0,0,0,0,0,0,24,255,110,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,209,222,1,0,0,0,0,0,0,62,255,51,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,55,255,125,0,0,0,0,0,0,0,117,255,19,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,164,255,60,0,0,0,0,0,0,0,171,230,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,24,235,255,178,120,89,74,72,72,72,74,246,241,121,141,153,148,83,1,0,0,0,0,0,0,0,0,0,6,121,231,255,255,255,255,255,255,255,255,255,255,255,255,255,253,173,14,0,0,0,0,0,0,0,0,0,0,1,19,44,63,76,83,83,83,83,100,255,192,66,52,45,46,34,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,39,255,138,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,68,255,113,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,104,255,84,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,147,255,52,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,190,255,23,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,25,229,210,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,255,117,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,91,255,34,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,49,120,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 diff --git a/platform/linux-generic/example/ml/mnist-12.onnx b/platform/linux-generic/example/ml/mnist-12.onnx Binary files differnew file mode 100644 index 000000000..6661bfe3c --- /dev/null +++ b/platform/linux-generic/example/ml/mnist-12.onnx diff --git a/platform/linux-generic/example/ml/mnist.c b/platform/linux-generic/example/ml/mnist.c new file mode 100644 index 000000000..4c1066302 --- /dev/null +++ b/platform/linux-generic/example/ml/mnist.c @@ -0,0 +1,300 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#include <odp_api.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <math.h> +#include <inttypes.h> + +#include "model_read.h" + +/** + * About MNIST model used in this example. + * + * The model predicts handwritten digits. It has one input and one output whose + * detailed information is as follows: + * + * Input: + * Name: Input3, type: float32, shape: [1, 1, 28, 28] + * + * Output: + * Name: Plus214_Output_0, type: float32, shape: [1, 10] + * + * Refer https://github.com/onnx/models/tree/main/validated/vision/classification/mnist + * for more information about the model. + * + * The model outputs the likelihood of each number before softmax, so we need to + * map the output to probabilities across the 10 classes with softmax function. + * + * In this example, the input image is stored in example_digit.csv file, which + * contains, comma separated, the digit label (a number from 0 to 9) and the 784 + * pixel values (a number from 0 to 255). Pixel order is first left to right and + * then top down. The MNIST dataset is available in this format at + * https://www.kaggle.com/oddrationale/mnist-in-csv. + */ + +#define MAX_MODEL_SIZE 30000 +#define INPUT_NUM_ELEMS 784 /* Total shape for input: 1 * 1 * 28 * 28 */ +#define OUTPUT_NUM_ELEMS 10 /* Total shape for output: 1 * 10 */ + +static int read_digit_csv(const char *file_name, uint8_t *expected_digit, float *pixels) +{ + char *tmp; + char *token; + char *end; + FILE *digit_file; + size_t size, num_elem; + const char *delim = ","; /* Delimiter */ + size_t num_pixel = 0; + + /* Get the model file size in bytes */ + digit_file = fopen(file_name, "rb"); + fseek(digit_file, 0, SEEK_END); + size = ftell(digit_file); + rewind(digit_file); + + tmp = malloc(size); + memset(tmp, 0, size); + num_elem = fread(tmp, size, 1, digit_file); + + fclose(digit_file); + if (num_elem != 1) { + printf("Read digit file failed\n"); + free(tmp); + return -1; + } + + /* Get the first token which is the expected digit */ + token = strtok(tmp, delim); + *expected_digit = (uint8_t)strtol(token, &end, 10); + if ((*expected_digit > 9) || (end == token)/*No numeric character*/) { + printf("Invalid digit %u or no numeric character available\n", + *expected_digit); + free(tmp); + return -1; + } + + /* The rest 784 numbers are pixel values */ + token = strtok(NULL, delim); + while (token != NULL) { + pixels[num_pixel] = strtof(token, NULL); + num_pixel++; + token = strtok(NULL, delim); + } + + if (num_pixel != INPUT_NUM_ELEMS) { + printf("Wrong number of pixels: %zu (expected:784)\n", num_pixel); + free(tmp); + return -1; + } + + free(tmp); + return 0; +} + +static int prepare_run_params(const char *file_name, uint8_t *expected_digit, + odp_ml_data_seg_t *input, odp_ml_data_seg_t *output) +{ + input->size = INPUT_NUM_ELEMS * sizeof(float); + input->addr = malloc(input->size); + memset(input->addr, 0, input->size); + + if (read_digit_csv(file_name, expected_digit, input->addr)) { + free(input->addr); + return -1; + } + + output->size = OUTPUT_NUM_ELEMS * sizeof(float); + output->addr = malloc(output->size); + memset(output->addr, 0, output->size); + + return 0; +} + +static float array_max(float *arr, uint8_t arr_len) +{ + float max = arr[0]; + + for (size_t i = 1; i < arr_len; i++) { + if (arr[i] > max) + max = arr[i]; + } + + return max; +} + +static void softmax(float *input, uint8_t input_len) +{ + float rowmax = array_max(input, input_len); + + float input_exp[input_len]; + float sum = 0.0f; + + for (size_t i = 0; i != input_len; ++i) { + input_exp[i] = exp(input[i] - rowmax); + sum += input_exp[i]; + } + + for (size_t i = 0; i != input_len; ++i) + input[i] = input_exp[i] / sum; +} + +static uint8_t index_of_max(float *arr, uint8_t arr_len) +{ + uint8_t i = 0; + uint8_t max_index = 0; + float max = arr[0]; + + for (i = 1; i < arr_len; i++) { + if (arr[i] > max) { + max = arr[i]; + max_index = i; + } + } + + return max_index; +} + +int main(int argc, char *argv[]) +{ + const char *model_file; + const char *input_file; + float *probabilities; + uint8_t expected_digit; + uint8_t predicted_digit; + odp_instance_t inst; + odp_ml_data_t data; + odp_ml_model_t ml_model; + odp_ml_data_seg_t input; + odp_ml_data_seg_t output; + odp_ml_capability_t capa; + odp_ml_config_t ml_config; + odp_ml_model_param_t model_param; + int ret = 0; + + if (argc != 3) { + printf("Please provide an input image file for classification.\n" + "\nUsage:\n" + " %s model_file input_image\n" + "\nThis example classifies digit written on the input image.\n\n", + argv[0]); + return -1; + } + + model_file = argv[1]; + input_file = argv[2]; + + if (odp_init_global(&inst, NULL, NULL)) { + printf("Global init failed.\n"); + return -1; + } + + if (odp_init_local(inst, ODP_THREAD_CONTROL)) { + printf("Local init failed.\n"); + return -1; + } + + if (odp_ml_capability(&capa)) { + printf("odp_ml_capability() failed\n"); + ret = -1; + goto odp_term; + } + + if (MAX_MODEL_SIZE > capa.max_model_size) { + printf("Configured max model size %d exceeds max mode size %" PRIu64 " in capa\n", + MAX_MODEL_SIZE, capa.max_model_size); + ret = -1; + goto odp_term; + } + + odp_ml_config_init(&ml_config); + ml_config.max_model_size = MAX_MODEL_SIZE; + ml_config.load_mode_mask = ODP_ML_COMPL_MODE_SYNC; + ml_config.run_mode_mask = ODP_ML_COMPL_MODE_SYNC; + + if (odp_ml_config(&ml_config)) { + printf("odp_ml_config() failed\n"); + ret = -1; + goto odp_term; + } + + odp_ml_model_param_init(&model_param); + if (read_model_from_file(model_file, &model_param)) { + printf("Read model file failed\n"); + ret = -1; + goto odp_term; + } + + ml_model = odp_ml_model_create("mnist", &model_param); + free(model_param.model); + if (ml_model == ODP_ML_MODEL_INVALID) { + printf("odp_ml_model_create() failed\n"); + ret = -1; + goto odp_term; + } + + odp_ml_model_print(ml_model); + + if (odp_ml_model_load(ml_model, NULL)) { + printf("odp_ml_model_load() failed\n"); + ret = -1; + goto destroy_model; + } + + data.num_input_seg = 1; + data.num_output_seg = 1; + data.input_seg = &input; + data.output_seg = &output; + if (prepare_run_params(input_file, &expected_digit, &input, &output)) { + printf("prepare_run_params() failed\n"); + ret = -1; + goto unload; + } + + if (odp_ml_run(ml_model, &data, NULL) != 1) { + printf("odp_ml_model_run() failed\n"); + ret = -1; + goto free_model_io; + } + + probabilities = output.addr; + + /* Post-process the model output */ + softmax(probabilities, OUTPUT_NUM_ELEMS); + predicted_digit = index_of_max(probabilities, OUTPUT_NUM_ELEMS); + printf("predicted_digit: %u, expected_digit: %u\n", predicted_digit, expected_digit); + +free_model_io: + free(input.addr); + free(output.addr); + +unload: + if (odp_ml_model_unload(ml_model, NULL)) { + printf("odp_ml_model_unload() failed\n"); + ret = -1; + goto odp_term; + } + +destroy_model: + /* Destroy the model */ + if (odp_ml_model_destroy(ml_model)) { + printf("odp_ml_model_destroy() failed\n"); + ret = -1; + } + +odp_term: + if (odp_term_local()) { + printf("Local term failed.\n"); + return -1; + } + + if (odp_term_global(inst)) { + printf("Global term failed.\n"); + return -1; + } + + return ret; +} diff --git a/platform/linux-generic/example/ml/model_explorer.c b/platform/linux-generic/example/ml/model_explorer.c new file mode 100644 index 000000000..bd449b032 --- /dev/null +++ b/platform/linux-generic/example/ml/model_explorer.c @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#include <odp_api.h> +#include <stdio.h> +#include <stdlib.h> + +#include "model_read.h" + +/** + * Read basic model information, e.g. inputs/outputs. + */ + +int main(int argc, char *argv[]) +{ + odp_instance_t inst; + odp_ml_model_t ml_model; + odp_ml_capability_t capa; + odp_ml_config_t ml_config; + odp_ml_model_param_t model_param; + int ret = 0; + + if (argc != 2) { + printf("Please specify model path\n" + "\nUsage:\n" + " %s model_path\n" + "\nThis example prints model information\n\n", + argv[0]); + return -1; + } + + if (odp_init_global(&inst, NULL, NULL)) { + printf("Global init failed.\n"); + return -1; + } + + if (odp_init_local(inst, ODP_THREAD_CONTROL)) { + printf("Local init failed.\n"); + return -1; + } + + if (odp_ml_capability(&capa)) { + printf("odp_ml_capability() failed\n"); + ret = -1; + goto odp_term; + } + + odp_ml_config_init(&ml_config); + ml_config.max_model_size = capa.max_model_size; + ml_config.load_mode_mask = ODP_ML_COMPL_MODE_SYNC; + ml_config.run_mode_mask = ODP_ML_COMPL_MODE_SYNC; + + if (odp_ml_config(&ml_config)) { + printf("odp_ml_config() failed\n"); + ret = -1; + goto odp_term; + } + + odp_ml_model_param_init(&model_param); + if (read_model_from_file(argv[1], &model_param)) { + ret = -1; + goto odp_term; + } + + ml_model = odp_ml_model_create("model-explorer", &model_param); + free(model_param.model); + if (ml_model == ODP_ML_MODEL_INVALID) { + printf("odp_ml_model_create failed.\n"); + ret = -1; + goto odp_term; + } + + odp_ml_model_print(ml_model); + +odp_term: + if (odp_term_local()) { + printf("Local term failed.\n"); + return -1; + } + + if (odp_term_global(inst)) { + printf("Global term failed.\n"); + return -1; + } + + return ret; +} diff --git a/platform/linux-generic/example/ml/model_read.c b/platform/linux-generic/example/ml/model_read.c new file mode 100644 index 000000000..7aa20bf35 --- /dev/null +++ b/platform/linux-generic/example/ml/model_read.c @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <odp_api.h> + +#include "model_read.h" + +int read_model_from_file(const char *file_name, odp_ml_model_param_t *model_param) +{ + FILE *model_file; + /* Number of elements successfully read */ + size_t num_elem; + + /* Get the model file size in bytes */ + model_file = fopen(file_name, "rb"); + if (model_file == NULL) { + perror("Failed to open model file"); + return -1; + } + + fseek(model_file, 0, SEEK_END); + model_param->size = ftell(model_file); + rewind(model_file); + + /* Allocate memory for model buffer */ + model_param->model = malloc(model_param->size); + memset(model_param->model, 0, model_param->size); + if (!model_param->model) { + printf("Allocating memory for model buffer failed\n"); + return -1; + } + + /* Read the model file */ + num_elem = fread(model_param->model, model_param->size, 1, model_file); + fclose(model_file); + if (num_elem != 1) { + printf("Read model file failed\n"); + free(model_param->model); + return -1; + } + + return 0; +} diff --git a/platform/linux-generic/example/ml/model_read.h b/platform/linux-generic/example/ml/model_read.h new file mode 100644 index 000000000..df2062d5f --- /dev/null +++ b/platform/linux-generic/example/ml/model_read.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_MODEL_READ_H_ +#define ODP_MODEL_READ_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp_api.h> + +/** + * Read model binaries from model file + * + * @param file_name The name of model file + * @param model_param Model parameter where model content and size are read to + * + * @retval 0 on success + * @retval < 0 on failure + */ +int read_model_from_file(const char *file_name, odp_ml_model_param_t *model_param); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/example/ml/odp_ml_run_mnist.sh b/platform/linux-generic/example/ml/odp_ml_run_mnist.sh new file mode 100755 index 000000000..f83d6f60d --- /dev/null +++ b/platform/linux-generic/example/ml/odp_ml_run_mnist.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Nokia +# +set -e + +# wget https://github.com/onnx/models/raw/main/validated/vision/classification/mnist/model/mnist-12.onnx +./mnist${EXEEXT} mnist-12.onnx example_digit.csv diff --git a/platform/linux-generic/example/ml/odp_ml_run_model_explorer.sh b/platform/linux-generic/example/ml/odp_ml_run_model_explorer.sh new file mode 100755 index 000000000..7f9fed5a6 --- /dev/null +++ b/platform/linux-generic/example/ml/odp_ml_run_model_explorer.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Nokia +# +set -e + +./model_explorer${EXEEXT} simple_linear.onnx diff --git a/platform/linux-generic/example/ml/odp_ml_run_simple_linear.sh b/platform/linux-generic/example/ml/odp_ml_run_simple_linear.sh new file mode 100755 index 000000000..b394b61a8 --- /dev/null +++ b/platform/linux-generic/example/ml/odp_ml_run_simple_linear.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Nokia +# +set -e + +./simple_linear${EXEEXT} [2,4,5] diff --git a/platform/linux-generic/example/ml/simple_linear.c b/platform/linux-generic/example/ml/simple_linear.c new file mode 100644 index 000000000..3417219c7 --- /dev/null +++ b/platform/linux-generic/example/ml/simple_linear.c @@ -0,0 +1,281 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#include <odp_api.h> +#include <odp/helper/odph_api.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <inttypes.h> + +#include "model_read.h" + +/** + * About model simple_linear.onnx used in this example. + * + * Model info: + * Inputs: name: x, type: int32, shape: [1] + * Outputs: name: y, type: int32, shape: [1] + * + * The model is of form y = 3 * x + 4 where x is given as the second argument. + * Thus when x = 5, the output y should be 19. + */ + +#define NUM_INPUTS 1 +#define NUM_OUTPUTS 1 +#define MAX_NUM_WORKERS 10 +#define MAX_MODEL_SIZE 500 + +typedef struct infer_param_t { + int32_t x; + odp_ml_model_t ml_model; +} infer_param_t; + +typedef struct { + odp_shm_t shm; + /* Thread specific arguments */ + infer_param_t infer_param[MAX_NUM_WORKERS]; +} thread_args_t; + +/* Global pointer to thread_args */ +static thread_args_t *thread_args; + +static int run_inference(void *infer_param) +{ + int32_t y; + odp_ml_data_t data; + odp_ml_data_seg_t input; + odp_ml_data_seg_t output; + infer_param_t *param = (infer_param_t *)infer_param; + + data.num_input_seg = NUM_INPUTS; + data.input_seg = &input; + input.addr = ¶m->x; + input.size = sizeof(int32_t); + + data.num_output_seg = NUM_OUTPUTS; + data.output_seg = &output; + output.addr = &y; + output.size = sizeof(int32_t); + + while (1) { + int ret = odp_ml_run(param->ml_model, &data, NULL); + + if (ret == 1) + break; + + if (ret < 0) { + ODPH_ERR("odp_ml_model_run() failed: %d\n", ret); + return -1; + } + } + + printf("y = 3 * %d + 4: %d\n", param->x, y); + + return 0; +} + +static int parse_argv1(char *argv1, uint32_t *num, int32_t *x) +{ + char *token; + int i; + + if (!strstr(argv1, "[")) { + *num = 1; + *x = strtol(argv1, NULL, 10); + return 0; + } + + token = strtok(argv1, "[,]"); + if (token == NULL) { + ODPH_ERR("Invalid argv[1]\n"); + return -1; + } + x[0] = strtol(token, NULL, 10); + + for (i = 0; i < MAX_NUM_WORKERS; i++) { + token = strtok(NULL, "[,]"); + if (token == NULL) + break; + + x[i + 1] = strtol(token, NULL, 10); + } + + if (i == MAX_NUM_WORKERS) { + ODPH_ERR("Too much xs, maximum number is: %d\n", MAX_NUM_WORKERS); + return -1; + } + + *num = i + 1; + return 0; +} + +int main(int argc, char *argv[]) +{ + odp_shm_t shm; + int num_workers; + odp_instance_t inst; + odp_cpumask_t cpumask; + odp_ml_model_t ml_model; + odp_ml_capability_t capa; + odp_ml_config_t ml_config; + int32_t x[MAX_NUM_WORKERS]; + odp_ml_model_param_t model_param; + odph_thread_t thread_tbl[MAX_NUM_WORKERS]; + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_param[MAX_NUM_WORKERS]; + char cpumaskstr[ODP_CPUMASK_STR_SIZE]; + int ret = 0; + uint32_t num = 0; + + if (argc != 2) { + ODPH_ERR("Please specify x\n" + "\nUsage:\n" + " %s x\n" + "\nThis example runs inference on model y = 3x + 4\n\n", + argv[0]); + return -1; + } + + if (parse_argv1(argv[1], &num, x)) + return -1; + + if (odp_init_global(&inst, NULL, NULL)) { + ODPH_ERR("Global init failed.\n"); + return -1; + } + + if (odp_init_local(inst, ODP_THREAD_CONTROL)) { + ODPH_ERR("Local init failed.\n"); + return -1; + } + + if (odp_ml_capability(&capa)) { + ODPH_ERR("odp_ml_capability() failed\n"); + ret = -1; + goto odp_term; + } + + if (MAX_MODEL_SIZE > capa.max_model_size) { + ODPH_ERR("Configured max model size %d exceeds max mode size %" PRIu64 " in capa\n", + MAX_MODEL_SIZE, capa.max_model_size); + ret = -1; + goto odp_term; + } + + /* Set ML configuration parameter */ + odp_ml_config_init(&ml_config); + ml_config.max_model_size = MAX_MODEL_SIZE; + ml_config.load_mode_mask = ODP_ML_COMPL_MODE_SYNC; + ml_config.run_mode_mask = ODP_ML_COMPL_MODE_SYNC; + + if (odp_ml_config(&ml_config)) { + ODPH_ERR("odp_ml_config() failed\n"); + ret = -1; + goto odp_term; + } + + odp_ml_model_param_init(&model_param); + if (read_model_from_file("simple_linear.onnx", &model_param)) { + ret = -1; + goto odp_term; + } + + ml_model = odp_ml_model_create("simple linear", &model_param); + free(model_param.model); + if (ml_model == ODP_ML_MODEL_INVALID) { + ODPH_ERR("odp_ml_model_create() failed\n"); + ret = -1; + goto odp_term; + } + + odp_ml_model_print(ml_model); + odp_ml_print(); + + if (odp_ml_model_load(ml_model, NULL)) { + ODPH_ERR("odp_ml_model_load() failed\n"); + ret = -1; + goto destroy_model; + } + + /* Reserve memory for args from shared mem */ + shm = odp_shm_reserve("_thread_args", sizeof(thread_args_t), + ODP_CACHE_LINE_SIZE, 0); + if (shm == ODP_SHM_INVALID) { + ODPH_ERR("Error: shared mem reserve failed.\n"); + ret = -1; + goto unload; + } + + thread_args = odp_shm_addr(shm); + if (thread_args == NULL) { + ODPH_ERR("Error: shared mem alloc failed.\n"); + ret = -1; + goto free_shm; + } + thread_args->shm = shm; + memset(thread_args, 0, sizeof(thread_args_t)); + + /* Prepare inference parameter */ + for (uint32_t i = 0; i < num; i++) { + thread_args->infer_param[i].x = x[i]; + thread_args->infer_param[i].ml_model = ml_model; + } + + num_workers = odp_cpumask_default_worker(&cpumask, num); + (void)odp_cpumask_to_str(&cpumask, cpumaskstr, sizeof(cpumaskstr)); + + printf("num worker threads: %i\n", num_workers); + printf("first CPU: %i\n", odp_cpumask_first(&cpumask)); + printf("cpu mask: %s\n", cpumaskstr); + + /* Create and init worker threads */ + memset(thread_tbl, 0, sizeof(thread_tbl)); + odph_thread_common_param_init(&thr_common); + thr_common.instance = inst; + thr_common.cpumask = &cpumask; + + for (int i = 0; i < num_workers; ++i) { + odph_thread_param_init(&thr_param[i]); + thr_param[i].start = run_inference; + thr_param[i].arg = &thread_args->infer_param[i]; + thr_param[i].thr_type = ODP_THREAD_WORKER; + } + + odph_thread_create(thread_tbl, &thr_common, thr_param, num_workers); + + odph_thread_join(thread_tbl, num_workers); + +free_shm: + if (odp_shm_free(shm)) { + ODPH_ERR("Error: shm free global data\n"); + return -1; + } + +unload: + /* Unload a model */ + if (odp_ml_model_unload(ml_model, NULL)) { + ODPH_ERR("odp_ml_model_load() failed\n"); + ret = -1; + } + +destroy_model: + if (odp_ml_model_destroy(ml_model)) { + ODPH_ERR("odp_ml_model_destroy() failed\n"); + ret = -1; + } + +odp_term: + if (odp_term_local()) { + ODPH_ERR("Local term failed.\n"); + return -1; + } + + if (odp_term_global(inst)) { + ODPH_ERR("Global term failed.\n"); + return -1; + } + + return ret; +} diff --git a/platform/linux-generic/example/ml/simple_linear.onnx b/platform/linux-generic/example/ml/simple_linear.onnx Binary files differnew file mode 100644 index 000000000..45c4b95b9 --- /dev/null +++ b/platform/linux-generic/example/ml/simple_linear.onnx diff --git a/platform/linux-generic/include-abi/odp/api/abi/atomic.h b/platform/linux-generic/include-abi/odp/api/abi/atomic.h index 9c87f9cb8..4f481f913 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/atomic.h +++ b/platform/linux-generic/include-abi/odp/api/abi/atomic.h @@ -80,7 +80,7 @@ typedef struct ODP_ALIGNED(sizeof(odp_u128_t)) odp_atomic_u128_s { #endif -/** @ingroup odp_atomic +/** @addtogroup odp_atomic * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/buffer_types.h b/platform/linux-generic/include-abi/odp/api/abi/buffer_types.h index 1d54bab07..63067268c 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/buffer_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/buffer_types.h @@ -21,7 +21,7 @@ extern "C" { #include <odp/api/std_types.h> #include <odp/api/plat/strong_types.h> -/** @ingroup odp_buffer +/** @addtogroup odp_buffer * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/classification.h b/platform/linux-generic/include-abi/odp/api/abi/classification.h index 342f4124c..d63763dbd 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/classification.h +++ b/platform/linux-generic/include-abi/odp/api/abi/classification.h @@ -19,7 +19,7 @@ extern "C" { #include <odp/api/plat/strong_types.h> -/** @ingroup odp_classification +/** @addtogroup odp_classification * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/comp.h b/platform/linux-generic/include-abi/odp/api/abi/comp.h index ac3d3a4a9..45681e961 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/comp.h +++ b/platform/linux-generic/include-abi/odp/api/abi/comp.h @@ -15,7 +15,7 @@ extern "C" { #include <stdint.h> -/** @ingroup odp_compression +/** @addtogroup odp_compression * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/crypto_types.h b/platform/linux-generic/include-abi/odp/api/abi/crypto_types.h index d49caf89a..b1e4aa5ae 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/crypto_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/crypto_types.h @@ -22,7 +22,7 @@ extern "C" { #include <odp/api/plat/strong_types.h> -/** @ingroup odp_crypto +/** @addtogroup odp_crypto * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/dma_types.h b/platform/linux-generic/include-abi/odp/api/abi/dma_types.h index 768591b10..d5bee0374 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/dma_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/dma_types.h @@ -13,7 +13,7 @@ extern "C" { #include <odp/api/plat/strong_types.h> -/** @ingroup odp_dma +/** @addtogroup odp_dma * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/event_types.h b/platform/linux-generic/include-abi/odp/api/abi/event_types.h index 8ff5acd6b..01ee66cd3 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/event_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/event_types.h @@ -1,5 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2022, Nokia + * Copyright (c) 2022-2023, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -20,7 +20,7 @@ extern "C" { #include <odp/api/plat/strong_types.h> -/** @ingroup odp_event +/** @addtogroup odp_event * @{ */ @@ -36,6 +36,7 @@ typedef enum odp_event_type_t { ODP_EVENT_PACKET_VECTOR = 6, ODP_EVENT_PACKET_TX_COMPL = 7, ODP_EVENT_DMA_COMPL = 8, + ODP_EVENT_ML_COMPL = 9 } odp_event_type_t; typedef enum odp_event_subtype_t { @@ -43,7 +44,9 @@ typedef enum odp_event_subtype_t { ODP_EVENT_PACKET_BASIC = 1, ODP_EVENT_PACKET_CRYPTO = 2, ODP_EVENT_PACKET_IPSEC = 3, - ODP_EVENT_PACKET_COMP = 4 + ODP_EVENT_PACKET_COMP = 4, + ODP_EVENT_ML_COMPL_LOAD = 5, + ODP_EVENT_ML_COMPL_RUN = 6 } odp_event_subtype_t; /** diff --git a/platform/linux-generic/include-abi/odp/api/abi/ipsec_types.h b/platform/linux-generic/include-abi/odp/api/abi/ipsec_types.h index 376666ded..1c5501997 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/ipsec_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/ipsec_types.h @@ -22,7 +22,7 @@ extern "C" { #include <odp/api/plat/strong_types.h> -/** @ingroup odp_ipsec +/** @addtogroup odp_ipsec * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/ml_types.h b/platform/linux-generic/include-abi/odp/api/abi/ml_types.h new file mode 100644 index 000000000..0fdb7a8dc --- /dev/null +++ b/platform/linux-generic/include-abi/odp/api/abi/ml_types.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2021-2023 Nokia + */ + +#ifndef ODP_API_ABI_ML_TYPES_H_ +#define ODP_API_ABI_ML_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/std_types.h> +#include <odp/api/plat/strong_types.h> + +/** @internal Implementation specific ML parameters */ +struct _odp_ml_model_extra_param_t { + /** @internal Dummy field to avoid empty struct */ + char dummy; +}; + +/** @addtogroup odp_ml + * @{ + */ + +typedef ODP_HANDLE_T(odp_ml_model_t); +typedef ODP_HANDLE_T(odp_ml_compl_t); +typedef struct _odp_ml_model_extra_param_t odp_ml_model_extra_param_t; + +#define ODP_ML_MODEL_INVALID _odp_cast_scalar(odp_ml_model_t, 0) +#define ODP_ML_COMPL_INVALID _odp_cast_scalar(odp_ml_compl_t, 0) + +#define ODP_ML_MODEL_NAME_LEN 64 +#define ODP_ML_MODEL_IO_NAME_LEN 64 +#define ODP_ML_SHAPE_NAME_LEN 16 +#define ODP_ML_EXTRA_STAT_NAME_LEN 64 + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include-abi/odp/api/abi/packet_io_types.h b/platform/linux-generic/include-abi/odp/api/abi/packet_io_types.h index 87e297f1d..76b162020 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/packet_io_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/packet_io_types.h @@ -21,7 +21,7 @@ extern "C" { #include <odp/api/std_types.h> #include <odp/api/plat/strong_types.h> -/** @ingroup odp_packet_io +/** @addtogroup odp_packet_io * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/packet_types.h b/platform/linux-generic/include-abi/odp/api/abi/packet_types.h index 4da9332ba..90b2af107 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/packet_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/packet_types.h @@ -21,7 +21,7 @@ extern "C" { #include <odp/api/std_types.h> #include <odp/api/plat/strong_types.h> -/** @ingroup odp_packet +/** @addtogroup odp_packet * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/pool_types.h b/platform/linux-generic/include-abi/odp/api/abi/pool_types.h index 0c0dbc97f..77b0ff638 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/pool_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/pool_types.h @@ -19,7 +19,7 @@ extern "C" { #include <odp/api/plat/strong_types.h> -/** @ingroup odp_pool +/** @addtogroup odp_pool * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/proto_stats_types.h b/platform/linux-generic/include-abi/odp/api/abi/proto_stats_types.h index 2ebddce62..d9db29188 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/proto_stats_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/proto_stats_types.h @@ -21,7 +21,7 @@ extern "C" { #include <odp/api/std_types.h> #include <odp/api/plat/strong_types.h> -/** @ingroup odp_proto_stats +/** @addtogroup odp_proto_stats * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/queue_types.h b/platform/linux-generic/include-abi/odp/api/abi/queue_types.h index 1a56c7682..4eff762bd 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/queue_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/queue_types.h @@ -21,7 +21,7 @@ extern "C" { #include <odp/api/std_types.h> #include <odp/api/plat/strong_types.h> -/** @ingroup odp_queue +/** @addtogroup odp_queue * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/shared_memory.h b/platform/linux-generic/include-abi/odp/api/abi/shared_memory.h index 551d49e30..bfcb9ebe5 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/shared_memory.h +++ b/platform/linux-generic/include-abi/odp/api/abi/shared_memory.h @@ -21,7 +21,7 @@ extern "C" { #include <odp/api/std_types.h> #include <odp/api/plat/strong_types.h> -/** @ingroup odp_shared_memory +/** @addtogroup odp_shared_memory * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/stash_types.h b/platform/linux-generic/include-abi/odp/api/abi/stash_types.h index 960f3ef17..2a4115886 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/stash_types.h +++ b/platform/linux-generic/include-abi/odp/api/abi/stash_types.h @@ -17,7 +17,7 @@ extern "C" { #include <odp/api/plat/strong_types.h> -/** @ingroup odp_stash +/** @addtogroup odp_stash * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/sync.h b/platform/linux-generic/include-abi/odp/api/abi/sync.h index 236e92c8c..276514b58 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/sync.h +++ b/platform/linux-generic/include-abi/odp/api/abi/sync.h @@ -17,7 +17,7 @@ extern "C" { #endif -/** @ingroup odp_barrier +/** @addtogroup odp_barrier * @{ */ diff --git a/platform/linux-generic/include-abi/odp/api/abi/ticketlock.h b/platform/linux-generic/include-abi/odp/api/abi/ticketlock.h index 6543a1cf7..b621bea7e 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/ticketlock.h +++ b/platform/linux-generic/include-abi/odp/api/abi/ticketlock.h @@ -19,7 +19,7 @@ extern "C" { #include <odp/api/atomic.h> -/** @ingroup odp_locks +/** @addtogroup odp_locks * @{ */ diff --git a/platform/linux-generic/include/odp/api/plat/event_inline_types.h b/platform/linux-generic/include/odp/api/plat/event_inline_types.h index caa075871..cbf01588f 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/event_inline_types.h @@ -28,6 +28,7 @@ extern "C" { typedef struct _odp_event_inline_offset_t { uint16_t event_type; uint16_t base_data; + uint16_t subtype; uint16_t flow_id; uint16_t pool; diff --git a/platform/linux-generic/include/odp/api/plat/event_inlines.h b/platform/linux-generic/include/odp/api/plat/event_inlines.h index b68ced244..990575166 100644 --- a/platform/linux-generic/include/odp/api/plat/event_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/event_inlines.h @@ -49,6 +49,15 @@ static inline odp_event_type_t __odp_event_type_get(odp_event_t event) return (odp_event_type_t)type; } +static inline odp_event_subtype_t __odp_event_subtype_get(odp_event_t event) +{ + int8_t type; + + type = _odp_event_hdr_field(event, int8_t, subtype); + + return (odp_event_subtype_t)type; +} + _ODP_INLINE odp_event_type_t odp_event_type(odp_event_t event) { return __odp_event_type_get(event); @@ -90,6 +99,7 @@ _ODP_INLINE void *odp_event_user_area(odp_event_t event) switch (type) { case ODP_EVENT_BUFFER: + case ODP_EVENT_ML_COMPL: case ODP_EVENT_DMA_COMPL: return _odp_buffer_get((odp_buffer_t)event, void *, uarea_addr); case ODP_EVENT_PACKET: @@ -112,6 +122,7 @@ _ODP_INLINE void *odp_event_user_area_and_flag(odp_event_t event, int *flag) switch (type) { case ODP_EVENT_BUFFER: case ODP_EVENT_DMA_COMPL: + case ODP_EVENT_ML_COMPL: *flag = -1; return _odp_buffer_get((odp_buffer_t)event, void *, uarea_addr); case ODP_EVENT_PACKET: @@ -145,10 +156,7 @@ _ODP_INLINE void *odp_event_user_area_and_flag(odp_event_t event, int *flag) _ODP_INLINE odp_event_subtype_t odp_event_subtype(odp_event_t event) { - if (__odp_event_type_get(event) != ODP_EVENT_PACKET) - return ODP_EVENT_NO_SUBTYPE; - - return (odp_event_subtype_t)_odp_pkt_get((odp_packet_t)event, int8_t, subtype); + return __odp_event_subtype_get(event); } _ODP_INLINE odp_event_type_t odp_event_types(odp_event_t event, @@ -156,9 +164,7 @@ _ODP_INLINE odp_event_type_t odp_event_types(odp_event_t event, { odp_event_type_t event_type = __odp_event_type_get(event); - *subtype = event_type == ODP_EVENT_PACKET ? - (odp_event_subtype_t)_odp_pkt_get((odp_packet_t)event, int8_t, subtype) : - ODP_EVENT_NO_SUBTYPE; + *subtype = __odp_event_subtype_get(event); return event_type; } @@ -172,11 +178,8 @@ _ODP_INLINE void odp_event_types_multi(const odp_event_t event[], odp_event_type if (subtype == NULL) return; - for (int i = 0; i < num; i++) { - subtype[i] = (type[i] == ODP_EVENT_PACKET) ? - (odp_event_subtype_t)_odp_pkt_get((odp_packet_t)event[i], int8_t, - subtype) : ODP_EVENT_NO_SUBTYPE; - } + for (int i = 0; i < num; i++) + subtype[i] = __odp_event_subtype_get(event[i]); } _ODP_INLINE uint32_t odp_event_flow_id(odp_event_t event) diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index eb20ca7d7..691965624 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -50,7 +50,6 @@ typedef struct _odp_packet_inline_offset_t { uint16_t timestamp; uint16_t input_flags; uint16_t flags; - uint16_t subtype; uint16_t cls_mark; uint16_t ipsec_ctx; uint16_t crypto_op; diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 960dbc5fc..2dd74fa29 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -24,6 +24,7 @@ #include <odp/api/plat/packet_io_inlines.h> #include <odp/api/plat/packet_inline_types.h> #include <odp/api/plat/pool_inline_types.h> +#include <odp/api/plat/event_inline_types.h> #include <stdint.h> #include <string.h> @@ -571,7 +572,8 @@ _ODP_INLINE void odp_packet_to_event_multi(const odp_packet_t pkt[], _ODP_INLINE odp_event_subtype_t odp_packet_subtype(odp_packet_t pkt) { - return (odp_event_subtype_t)_odp_pkt_get(pkt, int8_t, subtype); + return (odp_event_subtype_t)_odp_event_hdr_field((odp_event_t)(uintptr_t)pkt, + int8_t, subtype); } _ODP_INLINE odp_packet_tx_compl_t odp_packet_tx_compl_from_event(odp_event_t ev) diff --git a/platform/linux-generic/include/odp_bitset.h b/platform/linux-generic/include/odp_bitset.h index 0931fb337..e55b9ef1a 100644 --- a/platform/linux-generic/include/odp_bitset.h +++ b/platform/linux-generic/include/odp_bitset.h @@ -32,12 +32,6 @@ static inline uint32_t bitset_ffs(bitset_t b) return __builtin_ffsl(b); } -/* Load-exclusive with memory ordering */ -static inline bitset_t bitset_monitor(bitset_t *bs, int mo) -{ - return monitor32(bs, mo); -} - #elif ATOM_BITSET_SIZE <= 64 /* Return first-bit-set with StdC ffs() semantics */ @@ -46,12 +40,6 @@ static inline uint32_t bitset_ffs(bitset_t b) return __builtin_ffsll(b); } -/* Load-exclusive with memory ordering */ -static inline bitset_t bitset_monitor(bitset_t *bs, int mo) -{ - return monitor64(bs, mo); -} - #elif ATOM_BITSET_SIZE <= 128 /* Return first-bit-set with StdC ffs() semantics */ @@ -65,12 +53,6 @@ static inline uint32_t bitset_ffs(bitset_t b) return 0; } -/* Load-exclusive with memory ordering */ -static inline bitset_t bitset_monitor(bitset_t *bs, int mo) -{ - return monitor128(bs, mo); -} - #else #error Unsupported size of bit sets (ATOM_BITSET_SIZE) #endif diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 1cececb99..676b9f116 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -53,6 +53,13 @@ static inline odp_buffer_hdr_t *_odp_buf_hdr(odp_buffer_t buf) return (odp_buffer_hdr_t *)(uintptr_t)buf; } +static inline void _odp_buffer_subtype_set(odp_buffer_t buffer, int subtype) +{ + odp_buffer_hdr_t *buf_hdr = _odp_buf_hdr(buffer); + + buf_hdr->event_hdr.subtype = subtype; +} + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 8fd8c4be7..89d89936c 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -199,6 +199,15 @@ extern "C" { /* Enable timer scan performance benchmark. This works with inline enabled. */ #define CONFIG_TIMER_PROFILE_INLINE 0 +/* Maximum number of ML models that can be created or loaded. */ +#define CONFIG_ML_MAX_MODELS 4 + +/* Maximum number of inputs for a ML model. */ +#define CONFIG_ML_MAX_INPUTS 4 + +/* Maximum number of outputs for a ML model. */ +#define CONFIG_ML_MAX_OUTPUTS 4 + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_event_internal.h b/platform/linux-generic/include/odp_event_internal.h index d9957e530..1b85d64fc 100644 --- a/platform/linux-generic/include/odp_event_internal.h +++ b/platform/linux-generic/include/odp_event_internal.h @@ -65,6 +65,9 @@ typedef struct _odp_event_hdr_t { /* Event type. Maybe different than pool type (crypto compl event) */ int8_t event_type; + /* Event subtype */ + int8_t subtype; + /* Event flow id */ uint8_t flow_id; diff --git a/platform/linux-generic/include/odp_global_data.h b/platform/linux-generic/include/odp_global_data.h index f883cefd9..2a87192df 100644 --- a/platform/linux-generic/include/odp_global_data.h +++ b/platform/linux-generic/include/odp_global_data.h @@ -21,7 +21,6 @@ extern "C" { #include <odp_config_internal.h> #include <libconfig.h> -#include <pthread.h> #include <stdint.h> #include <sys/types.h> @@ -81,6 +80,7 @@ typedef struct odp_global_data_ro_t { uint8_t ipsec; uint8_t stash; uint8_t traffic_mngr; + uint8_t ml; } disable; diff --git a/platform/linux-generic/include/odp_init_internal.h b/platform/linux-generic/include/odp_init_internal.h index 24e8346ad..ca5d68c87 100644 --- a/platform/linux-generic/include/odp_init_internal.h +++ b/platform/linux-generic/include/odp_init_internal.h @@ -105,6 +105,9 @@ int _odp_stash_term_global(void); int _odp_dma_init_global(void); int _odp_dma_term_global(void); +int _odp_ml_init_global(void); +int _odp_ml_term_global(void); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_ipsec_internal.h b/platform/linux-generic/include/odp_ipsec_internal.h index 571796691..b97aa7031 100644 --- a/platform/linux-generic/include/odp_ipsec_internal.h +++ b/platform/linux-generic/include/odp_ipsec_internal.h @@ -30,7 +30,7 @@ extern "C" { #include <protocols/ip.h> #include <stdint.h> -/** @ingroup odp_ipsec +/** @addtogroup odp_ipsec * @{ */ diff --git a/platform/linux-generic/include/odp_macros_internal.h b/platform/linux-generic/include/odp_macros_internal.h index abf017aec..047e550f9 100644 --- a/platform/linux-generic/include/odp_macros_internal.h +++ b/platform/linux-generic/include/odp_macros_internal.h @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2022, Nokia + * Copyright (c) 2022-2024, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -26,19 +26,35 @@ extern "C" { #define _ODP_MIN(a, b) \ __extension__ ({ \ - __typeof__(a) tmp_a = (a); \ - __typeof__(b) tmp_b = (b); \ - tmp_a < tmp_b ? tmp_a : tmp_b; \ + __typeof__(a) min_a = (a); \ + __typeof__(b) min_b = (b); \ + min_a < min_b ? min_a : min_b; \ }) #define _ODP_MAX(a, b) \ __extension__ ({ \ - __typeof__(a) tmp_a = (a); \ - __typeof__(b) tmp_b = (b); \ - tmp_a > tmp_b ? tmp_a : tmp_b; \ + __typeof__(a) max_a = (a); \ + __typeof__(b) max_b = (b); \ + max_a > max_b ? max_a : max_b; \ }) -#define _ODP_MAX3(a, b, c) (_ODP_MAX(_ODP_MAX((a), (b)), (c))) +#define _ODP_MIN3(a, b, c) \ +__extension__ ({ \ + __typeof__(a) min3_a = (a); \ + __typeof__(b) min3_b = (b); \ + __typeof__(c) min3_c = (c); \ + (min3_a < min3_b ? (min3_a < min3_c ? min3_a : min3_c) : \ + (min3_b < min3_c ? min3_b : min3_c)); \ +}) + +#define _ODP_MAX3(a, b, c) \ +__extension__ ({ \ + __typeof__(a) max3_a = (a); \ + __typeof__(b) max3_b = (b); \ + __typeof__(c) max3_c = (c); \ + (max3_a > max3_b ? (max3_a > max3_c ? max3_a : max3_c) : \ + (max3_b > max3_c ? max3_b : max3_c)); \ +}) /* Macros to calculate ODP_ROUNDUP_POWER2_U32() in five rounds of shift * and OR operations. */ diff --git a/platform/linux-generic/include/odp_ml_fp16.h b/platform/linux-generic/include/odp_ml_fp16.h new file mode 100644 index 000000000..476028cb4 --- /dev/null +++ b/platform/linux-generic/include/odp_ml_fp16.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef ODP_ML_FP16_H_ +#define ODP_ML_FP16_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +uint16_t _odp_float32_to_float16(float x); +float _odp_float16_to_float32(uint16_t f16); +uint16_t _odp_float32_to_bfloat16(float x); +float _odp_bfloat16_to_float32(uint16_t f16); + +#ifdef __cplusplus +} +#endif + +#endif /* ODP_ML_FP16_H_ */ diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 41a44b83c..0b03aa211 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -107,8 +107,8 @@ typedef struct ODP_ALIGNED_CACHE odp_packet_hdr_t { uint16_t tailroom; - /* Event subtype */ - int8_t subtype; + /* Classifier handle index */ + uint16_t cos; /* Used as classifier destination queue, in IPsec inline input processing and as Tx * completion event queue. */ @@ -134,9 +134,6 @@ typedef struct ODP_ALIGNED_CACHE odp_packet_hdr_t { /* Classifier mark */ uint16_t cls_mark; - /* Classifier handle index */ - uint16_t cos; - /* Offset to payload start */ uint16_t payload_offset; @@ -214,9 +211,11 @@ static inline odp_packet_hdr_t *packet_last_seg(odp_packet_hdr_t *hdr) return hdr; } -static inline void packet_subtype_set(odp_packet_t pkt, int ev) +static inline void packet_subtype_set(odp_packet_t pkt, int subtype) { - packet_hdr(pkt)->subtype = ev; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + pkt_hdr->event_hdr.subtype = subtype; } /** @@ -258,8 +257,8 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len) pkt_hdr->headroom = pool->headroom; pkt_hdr->tailroom = pool->seg_len - seg_len + pool->tailroom; - if (odp_unlikely(pkt_hdr->subtype != ODP_EVENT_PACKET_BASIC)) - pkt_hdr->subtype = ODP_EVENT_PACKET_BASIC; + if (odp_unlikely(pkt_hdr->event_hdr.subtype != ODP_EVENT_PACKET_BASIC)) + pkt_hdr->event_hdr.subtype = ODP_EVENT_PACKET_BASIC; pkt_hdr->input = ODP_PKTIO_INVALID; } @@ -304,7 +303,7 @@ static inline void _odp_packet_copy_md(odp_packet_hdr_t *dst_hdr, odp_packet_hdr_t *src_hdr, odp_bool_t uarea_copy) { - int8_t subtype = src_hdr->subtype; + int8_t subtype = src_hdr->event_hdr.subtype; /* Lengths and segmentation data are not copied: * .frame_len @@ -316,7 +315,7 @@ static inline void _odp_packet_copy_md(odp_packet_hdr_t *dst_hdr, * .seg_count */ dst_hdr->input = src_hdr->input; - dst_hdr->subtype = subtype; + dst_hdr->event_hdr.subtype = subtype; dst_hdr->dst_queue = src_hdr->dst_queue; dst_hdr->cos = src_hdr->cos; dst_hdr->cls_mark = src_hdr->cls_mark; diff --git a/platform/linux-generic/include/odp_timer_internal.h b/platform/linux-generic/include/odp_timer_internal.h index 01ee4a0f3..38192d917 100644 --- a/platform/linux-generic/include/odp_timer_internal.h +++ b/platform/linux-generic/include/odp_timer_internal.h @@ -22,6 +22,12 @@ #include <odp_global_data.h> #include <odp_pool_internal.h> +/* + * Use as the argument to timer_run() to force a scan and to ignore rate + * limit. + */ +#define TIMER_SCAN_FORCE INT32_MAX + /** * Internal Timeout header */ @@ -48,13 +54,15 @@ ODP_STATIC_ASSERT(sizeof(odp_timeout_hdr_t) <= ODP_CACHE_LINE_SIZE, /* A larger decrement value should be used after receiving events compared to * an 'empty' call. */ -void _odp_timer_run_inline(int dec); +uint64_t _odp_timer_run_inline(int dec); /* Static inline wrapper to minimize modification of schedulers. */ -static inline void timer_run(int dec) +static inline uint64_t timer_run(int dec) { if (odp_global_rw->inline_timers) - _odp_timer_run_inline(dec); + return _odp_timer_run_inline(dec); + + return UINT64_MAX; } #endif diff --git a/platform/linux-generic/libodp-linux.pc.in b/platform/linux-generic/libodp-linux.pc.in index 05ba5b9d6..62589c1a3 100644 --- a/platform/linux-generic/libodp-linux.pc.in +++ b/platform/linux-generic/libodp-linux.pc.in @@ -8,5 +8,5 @@ Description: The ODP packet processing engine Version: @PKGCONFIG_VERSION@ Requires.private: libconfig@AARCH64CRYPTO_PKG@ Libs: -L${libdir} -l@ODP_LIB_NAME@ @ATOMIC_LIBS_NON_ABI_COMPAT@ -Libs.private: @OPENSSL_STATIC_LIBS@ @DPDK_LIBS@ @PCAP_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ @LIBXDP_LIBS@ -lpthread @ATOMIC_LIBS_ABI_COMPAT@ @IPSEC_MB_LIBS@ +Libs.private: @OPENSSL_STATIC_LIBS@ @DPDK_LIBS@ @PCAP_LIBS@ @PTHREAD_LIBS@ @TIMER_LIBS@ @LIBXDP_LIBS@ -lpthread @ATOMIC_LIBS_ABI_COMPAT@ @IPSEC_MB_LIBS@ @ORT_LIBS@ Cflags: -I${includedir} diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4 index 61b65540f..3306849d2 100644 --- a/platform/linux-generic/m4/configure.m4 +++ b/platform/linux-generic/m4/configure.m4 @@ -31,10 +31,11 @@ m4_include([platform/linux-generic/m4/odp_pcapng.m4]) m4_include([platform/linux-generic/m4/odp_dpdk.m4]) m4_include([platform/linux-generic/m4/odp_wfe.m4]) m4_include([platform/linux-generic/m4/odp_xdp.m4]) +m4_include([platform/linux-generic/m4/odp_ml.m4]) ODP_EVENT_VALIDATION ODP_SCHEDULER -AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${AARCH64CRYPTO_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${IPSEC_MB_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS} ${LIBXDP_LIBS}"]) +AS_VAR_APPEND([PLAT_DEP_LIBS], ["${ATOMIC_LIBS} ${AARCH64CRYPTO_LIBS} ${LIBCONFIG_LIBS} ${OPENSSL_LIBS} ${IPSEC_MB_LIBS} ${DPDK_LIBS_LT} ${LIBCLI_LIBS} ${LIBXDP_LIBS} ${ORT_LIBS}"]) # Add text to the end of configure with platform specific settings. # Make sure it's aligned same as other lines in configure.ac. @@ -46,6 +47,7 @@ AS_VAR_APPEND([PLAT_CFG_TEXT], [" pcap: ${have_pcap} pcapng: ${have_pcapng} wfe_locks: ${use_wfe_locks} + ml_support: ${ml_support} default_config_path: ${default_config_path}"]) # Ignore Clang specific errors about fields with variable sized type not at the @@ -59,6 +61,8 @@ AM_CONDITIONAL([PLATFORM_IS_LINUX_GENERIC], AC_CONFIG_FILES([platform/linux-generic/Makefile platform/linux-generic/libodp-linux.pc platform/linux-generic/dumpconfig/Makefile + platform/linux-generic/example/Makefile + platform/linux-generic/example/ml/Makefile platform/linux-generic/test/Makefile platform/linux-generic/test/example/Makefile platform/linux-generic/test/example/classifier/Makefile @@ -73,6 +77,7 @@ AC_CONFIG_FILES([platform/linux-generic/Makefile platform/linux-generic/test/example/switch/Makefile platform/linux-generic/test/validation/api/shmem/Makefile platform/linux-generic/test/validation/api/pktio/Makefile + platform/linux-generic/test/validation/api/ml/Makefile platform/linux-generic/test/performance/Makefile platform/linux-generic/test/performance/dmafwd/Makefile platform/linux-generic/test/pktio_ipc/Makefile]) diff --git a/platform/linux-generic/m4/odp_libconfig.m4 b/platform/linux-generic/m4/odp_libconfig.m4 index a6d19f661..77095e0fe 100644 --- a/platform/linux-generic/m4/odp_libconfig.m4 +++ b/platform/linux-generic/m4/odp_libconfig.m4 @@ -3,7 +3,7 @@ ########################################################################## m4_define([_odp_config_version_generation], [0]) m4_define([_odp_config_version_major], [1]) -m4_define([_odp_config_version_minor], [27]) +m4_define([_odp_config_version_minor], [28]) m4_define([_odp_config_version], [_odp_config_version_generation._odp_config_version_major._odp_config_version_minor]) diff --git a/platform/linux-generic/m4/odp_ml.m4 b/platform/linux-generic/m4/odp_ml.m4 new file mode 100644 index 000000000..a7b9a4fd6 --- /dev/null +++ b/platform/linux-generic/m4/odp_ml.m4 @@ -0,0 +1,46 @@ +########################################################################## +# Onnxruntime library path and name +########################################################################## +# Optional configure parameter for a non-standard install prefix of onnxruntime +AC_ARG_WITH([ort-path], + [AS_HELP_STRING([--with-ort-path=DIR], + [path to onnxruntime libs and headers [default=system]])], + [ort_path_given=yes + ORT_CPPFLAGS="-I$withval/include" + ORT_LIBS="-L$withval/lib" + ORT_RPATH="-R$withval/lib"], + []) + +########################################################################## +# Save and set temporary compilation flags +########################################################################## +OLD_CPPFLAGS=$CPPFLAGS +OLD_LIBS=$LIBS +CPPFLAGS="$ORT_CPPFLAGS $CPPFLAGS" +LIBS="$ORT_LIBS $LIBS" + +######################################################################### +# If ort is available, enable ML API +######################################################################### +ml_support=no +AC_CHECK_HEADERS([onnxruntime_c_api.h], + [AC_CHECK_LIB(onnxruntime, OrtGetApiBase, [ml_support=yes], [], [])], + [AS_IF([test "x$ort_path_given" = "xyes"], + [AC_MSG_ERROR([ort not found at the specified path (--with-ort-path)])])]) + +AS_IF([test "x$ml_support" != "xno"], + [ORT_LIBS="$ORT_RPATH $ORT_LIBS -lonnxruntime -lm"], + [ORT_CPPFLAGS="" ORT_LIBS="-lm"]) + +AC_CONFIG_COMMANDS_PRE([dnl +AM_CONDITIONAL([WITH_ML], [test x$ml_support = xyes ]) +]) + +########################################################################## +# Restore old saved variables +########################################################################## +LIBS=$OLD_LIBS +CPPFLAGS=$OLD_CPPFLAGS + +AC_SUBST([ORT_CPPFLAGS]) +AC_SUBST([ORT_LIBS]) diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c index 0e6eea3ae..016a8f0c5 100644 --- a/platform/linux-generic/odp_classification.c +++ b/platform/linux-generic/odp_classification.c @@ -299,10 +299,11 @@ odp_cos_t odp_cls_cos_create(const char *name, const odp_cls_cos_param_t *param_ param.hash_proto); tbl_index = i * CLS_COS_QUEUE_MAX; for (j = 0; j < param.num_queue; j++) { - char name[ODP_QUEUE_NAME_LEN]; + char hq_name[ODP_QUEUE_NAME_LEN]; - snprintf(name, sizeof(name), "_odp_cos_hq_%u_%u", i, j); - queue = odp_queue_create(name, &cos->queue_param); + snprintf(hq_name, sizeof(hq_name), "_odp_cos_hq_%u_%u", + i, j); + queue = odp_queue_create(hq_name, &cos->queue_param); if (queue == ODP_QUEUE_INVALID) { /* unwind the queues */ _cls_queue_unwind(tbl_index, j); diff --git a/platform/linux-generic/odp_cpumask.c b/platform/linux-generic/odp_cpumask.c index d0a9953f7..7d7575f51 100644 --- a/platform/linux-generic/odp_cpumask.c +++ b/platform/linux-generic/odp_cpumask.c @@ -7,7 +7,6 @@ #include <odp_posix_extensions.h> #include <sched.h> -#include <pthread.h> #include <odp/api/cpumask.h> #include <odp/api/init.h> diff --git a/platform/linux-generic/odp_cpumask_task.c b/platform/linux-generic/odp_cpumask_task.c index 0807e231e..a579b2e7e 100644 --- a/platform/linux-generic/odp_cpumask_task.c +++ b/platform/linux-generic/odp_cpumask_task.c @@ -12,7 +12,6 @@ #include <odp_debug_internal.h> #include <odp_global_data.h> -#include <pthread.h> #include <sched.h> int odp_cpumask_default_worker(odp_cpumask_t *mask, int max_num) diff --git a/platform/linux-generic/odp_event.c b/platform/linux-generic/odp_event.c index e15cb1c50..f3644f02b 100644 --- a/platform/linux-generic/odp_event.c +++ b/platform/linux-generic/odp_event.c @@ -12,6 +12,7 @@ #include <odp/api/packet.h> #include <odp/api/timer.h> #include <odp/api/pool.h> +#include <odp/api/ml.h> #include <odp_buffer_internal.h> #include <odp_ipsec_internal.h> @@ -36,6 +37,7 @@ const _odp_event_inline_offset_t _odp_event_inline_offset ODP_ALIGNED_CACHE = { .event_type = offsetof(_odp_event_hdr_t, event_type), .base_data = offsetof(_odp_event_hdr_t, base_data), + .subtype = offsetof(_odp_event_hdr_t, subtype), .flow_id = offsetof(_odp_event_hdr_t, flow_id), .pool = offsetof(_odp_event_hdr_t, pool), }; @@ -68,6 +70,9 @@ static inline void event_free(odp_event_t event, _odp_ev_id_t id) case ODP_EVENT_DMA_COMPL: odp_dma_compl_free(odp_dma_compl_from_event(event)); break; + case ODP_EVENT_ML_COMPL: + odp_ml_compl_free(odp_ml_compl_from_event(event)); + break; default: _ODP_ABORT("Invalid event type: %d\n", odp_event_type(event)); } @@ -116,6 +121,8 @@ int odp_event_is_valid(odp_event_t event) /* Fall through */ case ODP_EVENT_DMA_COMPL: /* Fall through */ + case ODP_EVENT_ML_COMPL: + /* Fall through */ case ODP_EVENT_PACKET_TX_COMPL: break; default: diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index 05b693c94..795252df1 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -51,6 +51,7 @@ enum init_stage { IPSEC_SAD_INIT, IPSEC_INIT, DMA_INIT, + ML_INIT, ALL_INIT /* All init stages completed */ }; @@ -95,6 +96,7 @@ static void disable_features(odp_global_data_ro_t *global_ro, global_ro->disable.traffic_mngr = init_param->not_used.feat.tm; global_ro->disable.compress = init_param->not_used.feat.compress; + global_ro->disable.ml = init_param->not_used.feat.ml; } void odp_init_param_init(odp_init_t *param) @@ -145,6 +147,13 @@ static int term_global(enum init_stage stage) switch (stage) { case ALL_INIT: + case ML_INIT: + if (_odp_ml_term_global()) { + _ODP_ERR("ODP ML term failed.\n"); + rc = -1; + } + /* Fall through */ + case DMA_INIT: if (_odp_dma_term_global()) { _ODP_ERR("ODP DMA term failed.\n"); @@ -509,6 +518,12 @@ int odp_init_global(odp_instance_t *instance, } stage = DMA_INIT; + if (_odp_ml_init_global()) { + _ODP_ERR("ODP ML init failed.\n"); + goto init_failed; + } + stage = ML_INIT; + *instance = (odp_instance_t)odp_global_ro.main_pid; return 0; diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 8c97a0f55..ee402b935 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -2180,7 +2180,7 @@ finish: int odp_ipsec_in(const odp_packet_t pkt_in[], int num_in, odp_packet_t pkt_out[], int *num_out, const odp_ipsec_in_param_t *param) { - int max_out = _ODP_MIN(_ODP_MIN(num_in, *num_out), MAX_BURST), num_crypto; + int max_out = _ODP_MIN3(num_in, *num_out, MAX_BURST), num_crypto; odp_packet_t crypto_pkts[MAX_BURST]; odp_crypto_packet_op_param_t crypto_param[MAX_BURST]; ipsec_op_t ops[MAX_BURST], *crypto_ops[MAX_BURST]; @@ -2288,7 +2288,7 @@ finish: int odp_ipsec_out(const odp_packet_t pkt_in[], int num_in, odp_packet_t pkt_out[], int *num_out, const odp_ipsec_out_param_t *param) { - int max_out = _ODP_MIN(_ODP_MIN(num_in, *num_out), MAX_BURST), num_crypto; + int max_out = _ODP_MIN3(num_in, *num_out, MAX_BURST), num_crypto; odp_packet_t crypto_pkts[MAX_BURST]; odp_crypto_packet_op_param_t crypto_param[MAX_BURST]; ipsec_op_t ops[MAX_BURST], *crypto_ops[MAX_BURST]; diff --git a/platform/linux-generic/odp_ishmpool.c b/platform/linux-generic/odp_ishmpool.c index 9b6340d7e..89ec10695 100644 --- a/platform/linux-generic/odp_ishmpool.c +++ b/platform/linux-generic/odp_ishmpool.c @@ -135,8 +135,8 @@ static inline uintptr_t get_bblock_nr(pool_t *bpool, void *addr) static inline void remove_from_list(pool_t *bpool, uint8_t order, bblock_t *bblock) { - bblock_t *curr; /* current bblock (when parsing list) */ - bblock_t *prev; /* previous bblock (when parsing list) */ + bblock_t *curr; + bblock_t *prev = NULL; curr = bpool->ctrl.free_heads[order]; if (!curr) diff --git a/platform/linux-generic/odp_ml.c b/platform/linux-generic/odp_ml.c new file mode 100644 index 000000000..6ab9e7177 --- /dev/null +++ b/platform/linux-generic/odp_ml.c @@ -0,0 +1,2646 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#include <odp/autoheader_external.h> + +#include <odp/api/atomic.h> +#include <odp/api/buffer.h> +#include <odp/api/event.h> +#include <odp/api/hints.h> +#include <odp/api/ml.h> +#include <odp/api/pool.h> +#include <odp/api/queue.h> +#include <odp/api/shared_memory.h> +#include <odp/api/std_types.h> +#include <odp/api/ticketlock.h> + +#include <odp/api/plat/event_inline_types.h> +#include <odp/api/plat/strong_types.h> + +#include <odp_buffer_internal.h> +#include <odp_config_internal.h> +#include <odp_debug_internal.h> +#include <odp_global_data.h> +#include <odp_init_internal.h> +#include <odp_libconfig_internal.h> +#include <odp_macros_internal.h> +#include <odp_pool_internal.h> + +#include <onnxruntime_c_api.h> + +#include <inttypes.h> +#include <stdint.h> +#include <stdio.h> +#include <string.h> + +#define ML_MAX_IO_SEGS UINT32_MAX +#define ML_MAX_COMPL_ID 32 +#define ML_MAX_CONFIG_STR_LEN 65 +#define ML_MAX_MODEL_SIZE (1024 * 1024 * 1024) +#define ML_MAX_MODELS_CREATED CONFIG_ML_MAX_MODELS +#define ML_MAX_MODELS_LOADED CONFIG_ML_MAX_MODELS + +/* Error codes */ +enum { + /* Feature not supported */ + ML_FEATURE_NOT_SUPPORTED = 1, + + /* Model is not created */ + ML_NOT_CREATED, + + /* Model was not loaded */ + ML_NOT_LOADED, + + /* Model has already loaded */ + ML_LOADED, + + /* Bad input */ + ML_BAD_INPUT, + + /* Fail from underlying library onnxruntime */ + ML_LIB_FAILED, + + /* Bad output */ + ML_BAD_OUTPUT, + + /* Bad handle */ + ML_BAD_HDL +}; + +typedef struct ort_run_opts_t { + int enable_profiling; + + ExecutionMode execution_mode; + + int inter_op_num_threads; + + int intra_op_num_threads; + + GraphOptimizationLevel graph_opt_level; + + char opt_model_filepath[ML_MAX_CONFIG_STR_LEN]; +} ort_run_opts_t; + +typedef struct ml_input_t { + /* Combined input start address */ + void *addr; + /* Data size in bytes */ + uint64_t size; +} ml_input_t; + +/* Onnxruntime model info */ +typedef struct ml_model_t { + /* Guards state, which must be accessed atomically */ + odp_ticketlock_t lock; + + enum { + ML_STATE_FREE = 0, /* Not allocated */ + ML_STATE_CREATED, /* Model is created */ + ML_STATE_LOADED, /* Model is loaded */ + ML_STATE_INFERENCING, /* Model is inferencing */ + } state; + + OrtSession *session; + OrtSessionOptions *session_opts; + uint32_t max_compl_id; + odp_atomic_u32_t compl_status[ML_MAX_COMPL_ID]; + + odp_ml_model_info_t info; + odp_ml_input_info_t input_info[CONFIG_ML_MAX_INPUTS]; + uint64_t input_sizes[CONFIG_ML_MAX_INPUTS]; + odp_ml_output_info_t output_info[CONFIG_ML_MAX_OUTPUTS]; + uint64_t output_sizes[CONFIG_ML_MAX_OUTPUTS]; + + struct { + void *user_ptr; + } result[ML_MAX_COMPL_ID]; +} ml_model_t; + +typedef struct ml_global_t { + odp_shm_t shm; + + odp_ml_capability_t capa; + odp_ml_config_t ml_config; + + odp_pool_param_t pool_param; + + const OrtApi *ort_api; + OrtEnv *env; + ort_run_opts_t ort_run_opts; + + ml_model_t models[ML_MAX_MODELS_CREATED]; + +} ml_global_t; + +static ml_global_t *_odp_ml_glb; + +static inline ml_model_t *ml_model_from_handle(odp_ml_model_t model) +{ + return (ml_model_t *)(uintptr_t)model; +} + +int odp_ml_capability(odp_ml_capability_t *capa) +{ + odp_pool_capability_t pool_capa; + + memset(capa, 0, sizeof(odp_ml_capability_t)); + + if (odp_global_ro.disable.ml) { + _ODP_PRINT("ML is disabled\n"); + return 0; + } + + capa->max_model_size = ML_MAX_MODEL_SIZE; + capa->max_models = ML_MAX_MODELS_CREATED; + capa->max_models_loaded = ML_MAX_MODELS_LOADED; + capa->max_compl_id = ML_MAX_COMPL_ID; + capa->max_inputs = CONFIG_ML_MAX_INPUTS; + capa->max_outputs = CONFIG_ML_MAX_OUTPUTS; + capa->max_segs_per_input = ML_MAX_IO_SEGS; + capa->max_segs_per_output = ML_MAX_IO_SEGS; + capa->min_input_align = 1; + capa->min_output_align = 1; + + capa->load.compl_mode_mask = ODP_ML_COMPL_MODE_SYNC | + ODP_ML_COMPL_MODE_POLL | + ODP_ML_COMPL_MODE_EVENT; + capa->load.compl_queue_plain = 1; + capa->load.compl_queue_sched = 1; + + capa->run.compl_mode_mask = ODP_ML_COMPL_MODE_SYNC | + ODP_ML_COMPL_MODE_POLL | + ODP_ML_COMPL_MODE_EVENT; + capa->run.compl_queue_plain = 1; + capa->run.compl_queue_sched = 1; + + if (odp_pool_capability(&pool_capa)) { + _ODP_ERR("Pool capability failed\n"); + return -1; + } + + capa->pool.max_pools = pool_capa.buf.max_pools; + capa->pool.max_num = pool_capa.buf.max_num; + capa->pool.max_uarea_size = pool_capa.buf.max_uarea_size; + capa->pool.uarea_persistence = pool_capa.buf.uarea_persistence; + capa->pool.max_cache_size = pool_capa.buf.max_cache_size; + capa->pool.min_cache_size = pool_capa.buf.min_cache_size; + + return 0; +} + +void odp_ml_config_init(odp_ml_config_t *config) +{ + memset(config, 0, sizeof(odp_ml_config_t)); + config->max_models_created = 1; + config->max_models_loaded = 1; +} + +int odp_ml_config(const odp_ml_config_t *config) +{ + if (!config) { + _ODP_ERR("Error: config must not be NULL\n"); + return -1; + } + + if (config->max_model_size == 0 || config->max_models_created == 0 || + config->max_models_loaded == 0) { + _ODP_ERR("Error: max_model_size, max_models_created and max_models_loaded" + " must be bigger than 0\n"); + return -1; + } + + if (config->max_models_loaded > config->max_models_created) { + _ODP_ERR("Error: max_models_loaded %d exceeds max_models_created %d\n", + config->max_models_loaded, config->max_models_created); + return -1; + } + + if (config->max_models_created > ML_MAX_MODELS_CREATED) { + _ODP_ERR("Error: max_models_created %d exceeds maximum number" + " of models that can be created in this driver %d\n", + config->max_models_created, ML_MAX_MODELS_CREATED); + return -1; + } + + if (config->max_models_loaded > ML_MAX_MODELS_LOADED) { + _ODP_ERR("Error: max_models_loaded %d exceeds maximum number" + " of models that can be loaded in this driver %d\n", + config->max_models_loaded, ML_MAX_MODELS_LOADED); + return -1; + } + + if (config->max_model_size > ML_MAX_MODEL_SIZE) { + _ODP_ERR("max_model_size %" PRIu64 " exceeds supported maximum model size %d\n", + config->max_model_size, ML_MAX_MODEL_SIZE); + return -1; + } + + _odp_ml_glb->ml_config = *config; + return 0; +} + +void odp_ml_model_param_init(odp_ml_model_param_t *param) +{ + memset(param, 0, sizeof(odp_ml_model_param_t)); +} + +static int check_ortstatus(OrtStatus * const status) +{ + if (status != NULL) { + const char *msg = _odp_ml_glb->ort_api->GetErrorMessage(status); + + _ODP_ERR("%s\n", msg); + _odp_ml_glb->ort_api->ReleaseStatus(status); + return -1; + } + + return 0; +} + +/* Get model input and output count */ +static int get_model_io_count(OrtSession *model, uint32_t *num_inputs, uint32_t *num_outputs) +{ + size_t num = 0; + OrtStatus *status = NULL; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + + status = ort_api->SessionGetInputCount(model, &num); + if (check_ortstatus(status)) { + _ODP_ERR("Get model input count failed\n"); + return -1; + } + + *num_inputs = num; + _ODP_DBG("num_inputs: %u\n", *num_inputs); + + status = ort_api->SessionGetOutputCount(model, &num); + if (check_ortstatus(status)) { + _ODP_ERR("Get model output count failed\n"); + return -1; + } + + *num_outputs = num; + _ODP_DBG("num_outputs: %u\n", *num_outputs); + + return 0; +} + +static odp_ml_data_type_t onnx_dtype_to_odp_dtype(ONNXTensorElementDataType onnx_dtype) +{ + switch (onnx_dtype) { + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: + return ODP_ML_DATA_TYPE_FP32; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: + return ODP_ML_DATA_TYPE_UINT8; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: + return ODP_ML_DATA_TYPE_INT8; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16: + return ODP_ML_DATA_TYPE_UINT16; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16: + return ODP_ML_DATA_TYPE_INT16; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: + return ODP_ML_DATA_TYPE_INT32; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32: + return ODP_ML_DATA_TYPE_UINT32; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: + return ODP_ML_DATA_TYPE_INT64; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64: + return ODP_ML_DATA_TYPE_UINT64; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16: + return ODP_ML_DATA_TYPE_FP16; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16: + return ODP_ML_DATA_TYPE_BFP16; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: + return ODP_ML_DATA_TYPE_FP64; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: + /* Fall through */ + case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64: + /* Fall through */ + case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128: + /* Fall through */ + default: + _ODP_ERR("onnx_dtype %d not supported by odp_ml\n", onnx_dtype); + return ODP_ML_DATA_TYPE_NONE; + } +} + +/* Get the size of given odp_ml_data_type_t in bytes */ +static uint32_t size_of_odp_ml_data_type(odp_ml_data_type_t data_type) +{ + switch (data_type) { + case ODP_ML_DATA_TYPE_NONE: + return 0; + + case ODP_ML_DATA_TYPE_INT8: + /* Fall through */ + case ODP_ML_DATA_TYPE_UINT8: + return 1; + + case ODP_ML_DATA_TYPE_INT16: + /* Fall through */ + case ODP_ML_DATA_TYPE_UINT16: + /* Fall through */ + case ODP_ML_DATA_TYPE_FP16: + /* Fall through */ + case ODP_ML_DATA_TYPE_BFP16: + return 2; + + case ODP_ML_DATA_TYPE_INT24: + /* Fall through */ + case ODP_ML_DATA_TYPE_UINT24: + return 3; + + case ODP_ML_DATA_TYPE_INT32: + /* Fall through */ + case ODP_ML_DATA_TYPE_UINT32: + /* Fall through */ + case ODP_ML_DATA_TYPE_FP32: + return 4; + + case ODP_ML_DATA_TYPE_INT64: + /* Fall through */ + case ODP_ML_DATA_TYPE_UINT64: + /* Fall through */ + case ODP_ML_DATA_TYPE_FP64: + return 8; + + default: + return 0; + } +} + +static int get_shape(int64_t dims[], odp_ml_shape_info_t *shape) +{ + uint32_t dyn_cnt = 0; + + for (uint32_t i = 0; i < shape->num_dim; i++) { + if (dims[i] == 0) { + _ODP_ERR("Dimension value: %" PRId64 " must be at least 1\n", dims[i]); + return -1; + } else if (dims[i] == -1) { /* Symbolic dimension */ + dyn_cnt++; + shape->dim[i] = ODP_ML_DIM_DYNAMIC; + shape->dim_min[i] = 0; /*unknown*/ + shape->dim_max[i] = 0; /*unknown*/ + } else if (dims[i] > 0 && dims[i] < UINT32_MAX) { + shape->dim[i] = dims[i]; + shape->dim_min[i] = dims[i]; + shape->dim_max[i] = dims[i]; + } else { + _ODP_ERR("Dimension value: %" PRId64 " invalid\n", dims[i]); + return -1; + } + } + + if (dyn_cnt == 0) { + shape->type = ODP_ML_SHAPE_STATIC; + } else if (dyn_cnt == 1) { + shape->type = ODP_ML_SHAPE_BATCH; + } else { + _ODP_ERR("Data shape type not supported by ODP\n"); + return -1; + } + + return 0; +} + +static inline void calculate_model_io_size(const odp_ml_shape_info_t *shape, uint64_t *size) +{ + /* Calculate the data size in bytes of this tensor, 0 for tensors with + * dynamic batch sizes */ + for (size_t i = 0; i < shape->num_dim; i++) { + /* Skip dynamic dimension size */ + if (shape->dim[i] == ODP_ML_DIM_DYNAMIC) { + *size = 0; + break; + } + (*size) *= shape->dim[i]; + } +} + +static int get_model_io_type_shape_size(OrtTypeInfo *type_info, odp_ml_shape_info_t *shape, + odp_ml_data_type_t *data_type, uint32_t *data_type_size, + uint64_t *size) +{ + ONNXTensorElementDataType tensor_type; + const OrtTensorTypeAndShapeInfo *tensor_info; + size_t num_dim = 0; + OrtStatus *status = NULL; + int64_t dims[ODP_ML_MAX_DIMS] = {0}; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + + status = ort_api->CastTypeInfoToTensorInfo(type_info, &tensor_info); + if (check_ortstatus(status)) { + _ODP_ERR("CastTypeInfoToTensorInfo failed\n"); + return -1; + } + + status = ort_api->GetTensorElementType(tensor_info, &tensor_type); + if (check_ortstatus(status)) { + _ODP_ERR("GetTensorElementType failed\n"); + return -1; + } + + *data_type = onnx_dtype_to_odp_dtype(tensor_type); + if (*data_type == ODP_ML_DATA_TYPE_NONE) /* Type not supported by odp */ + return -1; + + status = ort_api->GetDimensionsCount(tensor_info, &num_dim); + if (check_ortstatus(status)) { + _ODP_ERR("GetDimensionsCount failed\n"); + return -1; + } + + if (num_dim > ODP_ML_MAX_DIMS) { + _ODP_ERR("Number of dimensions: %zu exceeds supported maximum number" + " of dimensions: %d\n", num_dim, ODP_ML_MAX_DIMS); + return -1; + } + shape->num_dim = num_dim; + + status = ort_api->GetDimensions(tensor_info, dims, num_dim); + if (check_ortstatus(status)) { + _ODP_ERR("GetDimensions failed\n"); + return -1; + } + + if (get_shape(dims, shape)) + return -1; + + *data_type_size = size_of_odp_ml_data_type(*data_type); + + *size = *data_type_size; + calculate_model_io_size(shape, size); + + return 0; +} + +/* Get model input and output info */ +static int get_model_io_info(OrtSession *session, ml_model_t *mdl, + const odp_ml_model_param_t *param) +{ + char *name; + OrtTypeInfo *type_info; + const odp_ml_data_format_t *data_format; + OrtStatus *status = NULL; + OrtAllocator *allocator = NULL; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + odp_ml_input_info_t *input_info = mdl->input_info; + odp_ml_output_info_t *output_info = mdl->output_info; + + status = ort_api->GetAllocatorWithDefaultOptions(&allocator); + if (check_ortstatus(status)) { + _ODP_ERR("GetAllocatorWithDefaultOptions failed\n"); + return -1; + } + + /* Retrieve info about input array. */ + memset(input_info, 0, sizeof(mdl->input_info)); + for (uint32_t i = 0; i < mdl->info.num_inputs; i++) { + name = NULL; + status = ort_api->SessionGetInputName(session, i, allocator, &name); + if (check_ortstatus(status)) { + _ODP_ERR("Get %uth input name failed\n", i); + return -1; + } + + strncpy(input_info[i].name, name, ODP_ML_MODEL_IO_NAME_LEN - 1); + input_info[i].name[ODP_ML_MODEL_IO_NAME_LEN - 1] = 0; + + /* Free memory allocated by SessionGetInputName */ + status = ort_api->AllocatorFree(allocator, name); + if (check_ortstatus(status)) { + _ODP_ERR("AllocatorFree %uth input_name failed\n", i); + return -1; + } + + if (param->extra_info.num_inputs) { + data_format = ¶m->extra_info.input_format[i]; + + input_info[i].shape = data_format->shape; + input_info[i].data_type = data_format->data_type; + input_info[i].data_type_size = data_format->data_type_size; + + mdl->input_sizes[i] = input_info[i].data_type_size; + calculate_model_io_size(&data_format->shape, &mdl->input_sizes[i]); + continue; + } + + type_info = NULL; + status = ort_api->SessionGetInputTypeInfo(session, i, &type_info); + if (check_ortstatus(status)) { + _ODP_ERR("SessionGetInputTypeInfo failed\n"); + return -1; + } + + if (get_model_io_type_shape_size(type_info, &input_info[i].shape, + &input_info[i].data_type, + &input_info[i].data_type_size, + &mdl->input_sizes[i])) { + _ODP_ERR("get_model_io_type_shape_size() for input failed\n"); + ort_api->ReleaseTypeInfo(type_info); + return -1; + } + + ort_api->ReleaseTypeInfo(type_info); + } + + /* Retrieve info about output array. */ + memset(output_info, 0, sizeof(mdl->output_info)); + for (uint32_t i = 0; i < mdl->info.num_outputs; i++) { + name = NULL; + status = ort_api->SessionGetOutputName(session, i, allocator, &name); + if (check_ortstatus(status)) { + _ODP_ERR("Get %uth output name failed\n", i); + return -1; + } + + strncpy(output_info[i].name, name, ODP_ML_MODEL_IO_NAME_LEN - 1); + output_info[i].name[ODP_ML_MODEL_IO_NAME_LEN - 1] = 0; + + /* Free memory allocated by SessionGetOutputName */ + status = ort_api->AllocatorFree(allocator, name); + if (check_ortstatus(status)) { + _ODP_ERR("AllocatorFree %uth output_name failed\n", i); + return -1; + } + + if (param->extra_info.num_outputs) { + data_format = ¶m->extra_info.output_format[i]; + + output_info[i].shape = data_format->shape; + output_info[i].data_type = data_format->data_type; + output_info[i].data_type_size = data_format->data_type_size; + + mdl->output_sizes[i] = output_info[i].data_type_size; + calculate_model_io_size(&data_format->shape, &mdl->output_sizes[i]); + continue; + } + + type_info = NULL; + status = ort_api->SessionGetOutputTypeInfo(session, i, &type_info); + if (check_ortstatus(status)) { + _ODP_ERR("SessionGetOutputTypeInfo failed\n"); + return -1; + } + + if (get_model_io_type_shape_size(type_info, &output_info[i].shape, + &output_info[i].data_type, + &output_info[i].data_type_size, + &mdl->output_sizes[i])) { + _ODP_ERR("get_model_io_type_shape_size() for output failed\n"); + ort_api->ReleaseTypeInfo(type_info); + return -1; + } + + ort_api->ReleaseTypeInfo(type_info); + } + + return 0; +} + +static inline int check_model_io_num(const odp_ml_model_param_t *param, + uint32_t num_inputs, uint32_t num_outputs) +{ + /* Make sure the number of inputs/outputs not exceeding the supported + * model max inputs/outputs */ + if (num_inputs > CONFIG_ML_MAX_INPUTS) { + _ODP_ERR("The model's number of inputs %u exceeds the maximum " + "number of inputs supported in a model %u\n", + num_inputs, CONFIG_ML_MAX_INPUTS); + return -1; + } + + if (num_outputs > CONFIG_ML_MAX_OUTPUTS) { + _ODP_ERR("The model's number of outputs %u exceeds the maximum " + "number of outputs supported in a model %u\n", + num_outputs, CONFIG_ML_MAX_OUTPUTS); + + return -1; + } + + /* Make sure the numbers of inputs/outputs provided in the extra_info of + * param match the numbers defined in model metadata. */ + if (param->extra_info.num_inputs && + param->extra_info.num_inputs != num_inputs) { + _ODP_ERR("Provided param->extra_info.num_inputs %u does not match the" + " number of inputs defined in model metadata: %u\n", + param->extra_info.num_inputs, num_inputs); + return -1; + } + + if (param->extra_info.num_outputs && param->extra_info.num_outputs != num_outputs) { + _ODP_ERR("Provided param->extra_info.num_outputs %u does not match the" + " number of outputs defined in model metadata: %u\n", + param->extra_info.num_outputs, num_outputs); + return -1; + } + + if (param->extra_info.num_inputs && !param->extra_info.input_format) { + _ODP_ERR("num_inputs is provided but not input_format in param->extra_info\n"); + return -1; + } + + if (param->extra_info.num_outputs && !param->extra_info.output_format) { + _ODP_ERR("num_outputs is provided but not output_format in param->extra_info\n"); + return -1; + } + + return 0; +} + +static int create_ort_model(const odp_ml_model_param_t *param, OrtSession **session, + ml_model_t *mdl, OrtSessionOptions *session_opts) +{ + OrtStatus *status; + int64_t model_version; + uint32_t num_inputs = 0; + uint32_t num_outputs = 0; + OrtModelMetadata *metadata = {0}; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + + status = ort_api->CreateSessionFromArray(_odp_ml_glb->env, + param->model, + param->size, + session_opts, + session); + if (check_ortstatus(status) || !(*session)) { + _ODP_ERR("CreateSessionFromArray failed\n"); + return -1; + } + + if (get_model_io_count(*session, &num_inputs, &num_outputs)) { + _ODP_ERR("get_model_io_count() failed\n"); + ort_api->ReleaseSession(*session); + return -1; + } + + if (check_model_io_num(param, num_inputs, num_outputs)) { + ort_api->ReleaseSession(*session); + return -1; + } + + mdl->max_compl_id = param->max_compl_id; + mdl->info.num_inputs = num_inputs; + mdl->info.num_outputs = num_outputs; + + /* Get metadata */ + status = ort_api->SessionGetModelMetadata(*session, &metadata); + if (check_ortstatus(status) || !metadata) { + _ODP_ERR("SessionGetModelMetadata failed\n"); + ort_api->ReleaseSession(*session); + return -1; + } + + /* Get model version */ + status = ort_api->ModelMetadataGetVersion(metadata, &model_version); + if (check_ortstatus(status)) { + _ODP_ERR("ModelMetadataGetVersion failed\n"); + ort_api->ReleaseModelMetadata(metadata); + ort_api->ReleaseSession(*session); + return -1; + } + mdl->info.model_version = model_version; + mdl->info.interface_version = 0; + + if (get_model_io_info(*session, mdl, param)) { + _ODP_ERR("get_model_io_info() failed\n"); + ort_api->ReleaseModelMetadata(metadata); + ort_api->ReleaseSession(*session); + return -1; + } + + ort_api->ReleaseModelMetadata(metadata); + return 0; +} + +static int set_ort_run_opts(const char *name, OrtSessionOptions *se_opts) +{ + OrtStatus *status; + ort_run_opts_t *opts = &_odp_ml_glb->ort_run_opts; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + + if (opts->enable_profiling) { + status = ort_api->EnableProfiling(se_opts, name); + if (check_ortstatus(status)) { + _ODP_ERR("Enable profiling failed\n"); + return -1; + } + } + + status = ort_api->SetSessionExecutionMode(se_opts, opts->execution_mode); + if (check_ortstatus(status)) { + _ODP_ERR("SetSessionExecutionMode failed\n"); + return -1; + } + + if (opts->intra_op_num_threads) { + status = ort_api->SetIntraOpNumThreads(se_opts, opts->intra_op_num_threads); + if (check_ortstatus(status)) { + _ODP_ERR("SetIntraOpNumThreads failed\n"); + return -1; + } + } + + if (opts->inter_op_num_threads) { + status = ort_api->SetInterOpNumThreads(se_opts, opts->inter_op_num_threads); + if (check_ortstatus(status)) { + _ODP_ERR("SetInterOpNumThreads failed\n"); + return -1; + } + } + + status = ort_api->SetSessionGraphOptimizationLevel(se_opts, opts->graph_opt_level); + if (check_ortstatus(status)) { + _ODP_ERR("SetSessionGraphOptimizationLevel failed\n"); + return -1; + } + + /* Optimized model file path is not provided */ + if (opts->opt_model_filepath[0] == '\0') + return 0; + + status = ort_api->SetOptimizedModelFilePath(se_opts, opts->opt_model_filepath); + if (check_ortstatus(status)) { + _ODP_ERR("SetOptimizedModelFilePath failed\n"); + return -1; + } + + return 0; +} + +static inline void reset_mdl_info_sizes(ml_model_t *mdl) +{ + memset(&mdl->info, 0, sizeof(odp_ml_model_info_t)); + memset(mdl->input_info, 0, sizeof(mdl->input_info)); + memset(mdl->output_info, 0, sizeof(mdl->output_info)); + memset(mdl->input_sizes, 0, sizeof(mdl->input_sizes)); + memset(mdl->output_sizes, 0, sizeof(mdl->output_sizes)); +} + +static int check_io_shape(ml_model_t *mdl) +{ + odp_ml_shape_info_t *shape; + + for (uint32_t i = 0; i < mdl->info.num_inputs; i++) { + shape = &mdl->input_info[i].shape; + + if (shape->type == ODP_ML_SHAPE_NONE) { + _ODP_ERR("Undefined shape type for model input[%u]\n", i); + return -1; + } + + if (shape->type == ODP_ML_SHAPE_STATIC) + continue; + + /* shape->type == ODP_ML_SHAPE_BATCH */ + for (uint32_t j = 0; j < shape->num_dim; j++) { + if (shape->dim[j] == ODP_ML_DIM_DYNAMIC && !shape->dim_max[j]) { + _ODP_ERR("Missing dim_max[%u] for dynamic sized input[%u], please" + " provide via the extra_info of model param\n", j, i); + return -1; + } + } + } + + for (uint32_t i = 0; i < mdl->info.num_outputs; i++) { + if (mdl->output_info[i].shape.type == ODP_ML_SHAPE_NONE) { + _ODP_ERR("Undefined shape type for model output[%u]\n", i); + return -1; + } + } + + return 0; +} + +odp_ml_model_t odp_ml_model_create(const char *name, const odp_ml_model_param_t *param) +{ + OrtStatus *status; + odp_ml_model_info_t *info; + OrtSessionOptions *session_opts; + uint32_t i = 0; + ml_model_t *mdl = NULL; + OrtSession *session = NULL; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + + if (odp_unlikely(odp_global_ro.disable.ml)) { + _ODP_ERR("ML is disabled\n"); + return ODP_ML_MODEL_INVALID; + } + + if (odp_unlikely(param->size > _odp_ml_glb->ml_config.max_model_size)) { + _ODP_ERR("Model size %" PRIu64 " exceeds maximum model size configured %" PRIu64 "\n", + param->size, _odp_ml_glb->ml_config.max_model_size); + return ODP_ML_MODEL_INVALID; + } + + if (odp_unlikely(!param->size || !param->model)) { + _ODP_ERR("Invalid model param: param->model: %p, param->size: %" PRIu64 "\n", + param->model, param->size); + return ODP_ML_MODEL_INVALID; + } + + if (odp_unlikely(param->max_compl_id > ML_MAX_COMPL_ID)) { + _ODP_ERR("param->max_compl_id: %u exceeds maximum completion id supported: %d\n", + param->max_compl_id, ML_MAX_COMPL_ID); + return ODP_ML_MODEL_INVALID; + } + + /* Find an emtpy slot to store the new model */ + for (i = 0; i < ML_MAX_MODELS_CREATED; i++) { + if (_odp_ml_glb->models[i].state) + continue; + + odp_ticketlock_lock(&_odp_ml_glb->models[i].lock); + + if (_odp_ml_glb->models[i].state) { + odp_ticketlock_unlock(&_odp_ml_glb->models[i].lock); + continue; + } + + mdl = &_odp_ml_glb->models[i]; + break; + } + + if (i == ML_MAX_MODELS_CREATED) { + _ODP_ERR("Maximum number of models has already been created!\n"); + return ODP_ML_MODEL_INVALID; + } + + /* Free model entry was found and is now locked */ + mdl->state = ML_STATE_CREATED; + + status = ort_api->CreateSessionOptions(&session_opts); + if (check_ortstatus(status) || !session_opts) { + _ODP_ERR("Error: CreateSessionOptions failed.\n"); + mdl->state = ML_STATE_FREE; + odp_ticketlock_unlock(&mdl->lock); + return ODP_ML_MODEL_INVALID; + } + + if (set_ort_run_opts(name, session_opts)) { + _odp_ml_glb->ort_api->ReleaseSessionOptions(session_opts); + mdl->state = ML_STATE_FREE; + odp_ticketlock_unlock(&mdl->lock); + return ODP_ML_MODEL_INVALID; + } + + /* Store model info */ + info = &mdl->info; + memset(info, 0, sizeof(odp_ml_model_info_t)); + + if (create_ort_model(param, &session, mdl, session_opts)) { + mdl->state = ML_STATE_FREE; + + /* Initialize info back to 0 when some fields have been filled + * while later failed */ + reset_mdl_info_sizes(mdl); + odp_ticketlock_unlock(&mdl->lock); + + _odp_ml_glb->ort_api->ReleaseSessionOptions(session_opts); + _ODP_ERR("create_ort_model() failed\n"); + return ODP_ML_MODEL_INVALID; + } + + if (check_io_shape(mdl)) { + mdl->state = ML_STATE_FREE; + reset_mdl_info_sizes(mdl); + odp_ticketlock_unlock(&mdl->lock); + + ort_api->ReleaseSession(session); + _odp_ml_glb->ort_api->ReleaseSessionOptions(session_opts); + return ODP_ML_MODEL_INVALID; + } + + mdl->session = session; + mdl->session_opts = session_opts; + info->index = i; + + if (name) { + strncpy(info->name, name, ODP_ML_MODEL_NAME_LEN - 1); + info->name[ODP_ML_MODEL_NAME_LEN - 1] = 0; + } + + mdl->max_compl_id = param->max_compl_id; + for (uint32_t j = 0; j < ML_MAX_COMPL_ID; j++) + odp_atomic_init_u32(&mdl->compl_status[j], 1); + + odp_ticketlock_unlock(&mdl->lock); + return (odp_ml_model_t)mdl; +} + +int odp_ml_model_destroy(odp_ml_model_t model) +{ + ml_model_t *mdl = ml_model_from_handle(model); + + if (model == ODP_ML_MODEL_INVALID) { + _ODP_ERR("Bad ML model handle\n"); + return -1; + } + + odp_ticketlock_lock(&mdl->lock); + + if (mdl->state != ML_STATE_CREATED) { + _ODP_ERR("Model not created\n"); + odp_ticketlock_unlock(&mdl->lock); + return -1; + } + + _odp_ml_glb->ort_api->ReleaseSessionOptions(mdl->session_opts); + _odp_ml_glb->ort_api->ReleaseSession(mdl->session); + mdl->state = ML_STATE_FREE; + mdl->session = NULL; + odp_ticketlock_unlock(&mdl->lock); + + return 0; +} + +int odp_ml_model_info(odp_ml_model_t model, odp_ml_model_info_t *info) +{ + ml_model_t *mdl = ml_model_from_handle(model); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad ML model handle\n"); + return -1; + } + + if (odp_unlikely(!info)) { + _ODP_ERR("info must not be NULL\n"); + return -1; + } + + odp_ticketlock_lock(&mdl->lock); + if (odp_unlikely(mdl->state == ML_STATE_FREE)) { + _ODP_ERR("Model not created\n"); + odp_ticketlock_unlock(&mdl->lock); + return -1; + } + + *info = mdl->info; + + odp_ticketlock_unlock(&mdl->lock); + return 0; +} + +uint32_t odp_ml_model_input_info(odp_ml_model_t model, odp_ml_input_info_t info[], uint32_t num) +{ + uint32_t num_model_inputs; + uint32_t num_written; + ml_model_t *mdl = ml_model_from_handle(model); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad ML model handle\n"); + return 0; + } + + odp_ticketlock_lock(&mdl->lock); + num_model_inputs = mdl->info.num_inputs; + num_written = num_model_inputs >= num ? num : num_model_inputs; + + if (num == 0) { + odp_ticketlock_unlock(&mdl->lock); + return num_model_inputs; + } + + for (uint32_t i = 0; i < num_written; i++) + info[i] = mdl->input_info[i]; + + odp_ticketlock_unlock(&mdl->lock); + return num_model_inputs; +} + +uint32_t odp_ml_model_output_info(odp_ml_model_t model, odp_ml_output_info_t info[], uint32_t num) +{ + uint32_t num_model_outputs; + uint32_t num_written; + ml_model_t *mdl = ml_model_from_handle(model); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad ML model handle\n"); + return 0; + } + + odp_ticketlock_lock(&mdl->lock); + num_model_outputs = mdl->info.num_outputs; + num_written = num_model_outputs >= num ? num : num_model_outputs; + + if (num == 0) { + odp_ticketlock_unlock(&mdl->lock); + return num_model_outputs; + } + + for (uint32_t i = 0; i < num_written; i++) + info[i] = mdl->output_info[i]; + + odp_ticketlock_unlock(&mdl->lock); + return num_model_outputs; +} + +odp_ml_model_t odp_ml_model_lookup(const char *name) +{ + uint32_t i; + ml_model_t *mdl; + + for (i = 0; i < ML_MAX_MODELS_CREATED; i++) { + mdl = &_odp_ml_glb->models[i]; + + odp_ticketlock_lock(&mdl->lock); + + if (mdl->state == ML_STATE_FREE) { + odp_ticketlock_unlock(&mdl->lock); + continue; + } + + if (!strcmp(mdl->info.name, name)) { + /* found it */ + odp_ticketlock_unlock(&mdl->lock); + return (odp_ml_model_t)mdl; + } + odp_ticketlock_unlock(&mdl->lock); + } + + return ODP_ML_MODEL_INVALID; +} + +uint64_t odp_ml_model_to_u64(odp_ml_model_t model) +{ + return _odp_pri(model); +} + +static const char *data_type_str(odp_ml_data_type_t data_type) +{ + switch (data_type) { + case ODP_ML_DATA_TYPE_INT8: + return "int8"; + case ODP_ML_DATA_TYPE_UINT8: + return "uint8"; + case ODP_ML_DATA_TYPE_UINT16: + return "uint16"; + case ODP_ML_DATA_TYPE_INT16: + return "int16"; + case ODP_ML_DATA_TYPE_INT32: + return "int32"; + case ODP_ML_DATA_TYPE_UINT32: + return "uint32"; + case ODP_ML_DATA_TYPE_INT64: + return "int64"; + case ODP_ML_DATA_TYPE_UINT64: + return "uint64"; + case ODP_ML_DATA_TYPE_FP16: + return "fp16"; + case ODP_ML_DATA_TYPE_FP32: + return "fp32"; + case ODP_ML_DATA_TYPE_BFP16: + return "bfp16"; + default: + return "unknown"; + } +} + +static const char *shape_type_str(odp_ml_shape_type_t shape_type) +{ + switch (shape_type) { + case ODP_ML_SHAPE_NONE: + return "none"; + case ODP_ML_SHAPE_STATIC: + return "static"; + case ODP_ML_SHAPE_BATCH: + return "batch"; + default: + return "Unknown"; + } +} + +static void print_shape(const odp_ml_shape_info_t *shape) +{ + /* Print shape */ + _ODP_PRINT("Shape: %s [", shape_type_str(shape->type)); + + for (uint32_t i = 0; i < shape->num_dim; i++) { + if (shape->dim[i] == ODP_ML_DIM_DYNAMIC) + _ODP_PRINT("Dyn"); + else + _ODP_PRINT("%" PRIu32, shape->dim[i]); + + if (i == (shape->num_dim - 1)) + _ODP_PRINT("]\n"); + else + _ODP_PRINT(", "); + } + + /* The number of dimensions for a scalar input is 0, in which case did not + * go into above for loop */ + if (shape->num_dim == 0) + _ODP_PRINT("]\n"); +} + +void odp_ml_model_print(odp_ml_model_t model) +{ + ml_model_t *mdl = ml_model_from_handle(model); + const odp_ml_model_info_t * const info = &mdl->info; + const odp_ml_input_info_t * const input_info = mdl->input_info; + const odp_ml_output_info_t * const output_info = mdl->output_info; + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad ML model handle\n"); + return; + } + + odp_ticketlock_lock(&mdl->lock); + if (odp_unlikely(mdl->state == ML_STATE_FREE)) { + odp_ticketlock_unlock(&mdl->lock); + _ODP_ERR("Model not created\n"); + return; + } + + _ODP_PRINT("\nModel info\n"); + _ODP_PRINT("----------\n"); + _ODP_PRINT(" Model handle: 0x%" PRIx64 "\n", odp_ml_model_to_u64(model)); + _ODP_PRINT(" Name: %s\n", info->name); + _ODP_PRINT(" Model version: %" PRIu64 "\n", info->model_version); + _ODP_PRINT(" Model interface version: %" PRIu64 "\n", info->interface_version); + _ODP_PRINT(" Index: %u\n", info->index); + _ODP_PRINT(" Number of inputs: %u\n", info->num_inputs); + + for (uint32_t i = 0; i < info->num_inputs; i++) { + _ODP_PRINT(" Input[%u]: ", i); + _ODP_PRINT("Name: %s, ", input_info[i].name); + _ODP_PRINT("Data_type: %s, ", data_type_str(input_info[i].data_type)); + print_shape(&input_info[i].shape); + } + + _ODP_PRINT(" Number of outputs: %u\n", info->num_outputs); + for (uint32_t i = 0; i < info->num_outputs; i++) { + _ODP_PRINT(" Output[%u]: ", i); + _ODP_PRINT("Name: %s, ", output_info[i].name); + _ODP_PRINT("Data_type: %s, ", data_type_str(output_info[i].data_type)); + print_shape(&output_info[i].shape); + } + + odp_ticketlock_unlock(&mdl->lock); + + _ODP_PRINT("\n"); +} + +static inline void mode_print(odp_ml_compl_mode_t compl_mode_mask) +{ + if (compl_mode_mask & ODP_ML_COMPL_MODE_SYNC) + _ODP_PRINT(" syn"); + + if (compl_mode_mask & ODP_ML_COMPL_MODE_POLL) + _ODP_PRINT(" poll"); + + if (compl_mode_mask & ODP_ML_COMPL_MODE_EVENT) + _ODP_PRINT(" event"); +} + +void odp_ml_print(void) +{ + _ODP_PRINT("\nML info\n"); + _ODP_PRINT("-----------\n"); + _ODP_PRINT(" max_model_size: %u\n", ML_MAX_MODEL_SIZE); + _ODP_PRINT(" max_compl_id: %u\n", ML_MAX_COMPL_ID); + _ODP_PRINT(" max_models_created: %u\n", ML_MAX_MODELS_CREATED); + _ODP_PRINT(" max_models_loaded: %u\n", ML_MAX_MODELS_LOADED); + _ODP_PRINT(" model_max_inputs: %u\n", CONFIG_ML_MAX_INPUTS); + _ODP_PRINT(" model_max_outputs: %u\n", CONFIG_ML_MAX_OUTPUTS); + + _ODP_PRINT(" load:\n"); + _ODP_PRINT(" completion mode: "); + mode_print(_odp_ml_glb->capa.load.compl_mode_mask); + _ODP_PRINT(", plain queue: %c, schedule queue: %c\n", + _odp_ml_glb->capa.load.compl_queue_plain ? 'Y' : 'N', + _odp_ml_glb->capa.load.compl_queue_sched ? 'Y' : 'N'); + + _ODP_PRINT(" run:\n"); + _ODP_PRINT(" completion mode:"); + mode_print(_odp_ml_glb->capa.run.compl_mode_mask); + _ODP_PRINT(", plain queue: %c, schedule queue: %c\n", + _odp_ml_glb->capa.run.compl_queue_plain ? 'Y' : 'N', + _odp_ml_glb->capa.run.compl_queue_sched ? 'Y' : 'N'); + _ODP_PRINT("\n"); +} + +int odp_ml_model_extra_stat_info(odp_ml_model_t model, + odp_ml_extra_stat_info_t info[] ODP_UNUSED, + int num ODP_UNUSED) +{ + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad ML model handle\n"); + return -1; + } + + return 0; +} + +int odp_ml_model_extra_stats(odp_ml_model_t model, uint64_t stats[] ODP_UNUSED, int num ODP_UNUSED) +{ + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad ML model handle\n"); + return -1; + } + + return 0; +} + +void odp_ml_compl_pool_param_init(odp_ml_compl_pool_param_t *pool_param) +{ + if (odp_unlikely(!pool_param)) { + _ODP_ERR("Param 'pool_param' must not NULL\n"); + return; + } + + memset(pool_param, 0, sizeof(odp_ml_compl_pool_param_t)); + + pool_param->cache_size = _odp_ml_glb->pool_param.buf.cache_size; +} + +odp_pool_t odp_ml_compl_pool_create(const char *name, const odp_ml_compl_pool_param_t *pool_param) +{ + odp_pool_t pool; + odp_pool_param_t ml_pool_param; + uint32_t num = pool_param->num; + uint32_t uarea_size = pool_param->uarea_size; + uint32_t cache_size = pool_param->cache_size; + uint32_t buf_size = _ODP_MAX(sizeof(odp_ml_run_result_t), + sizeof(odp_ml_load_result_t)); + + if (num > _odp_ml_glb->capa.pool.max_num) { + _ODP_ERR("Too many ML completion events: %u\n", num); + return ODP_POOL_INVALID; + } + + if (uarea_size > _odp_ml_glb->capa.pool.max_uarea_size) { + _ODP_ERR("Bad uarea size: %u\n", uarea_size); + return ODP_POOL_INVALID; + } + + if (cache_size < _odp_ml_glb->capa.pool.min_cache_size || + cache_size > _odp_ml_glb->capa.pool.max_cache_size) { + _ODP_ERR("Bad cache size: %u\n", cache_size); + return ODP_POOL_INVALID; + } + + odp_pool_param_init(&ml_pool_param); + ml_pool_param.type = ODP_POOL_BUFFER; + ml_pool_param.uarea_init.init_fn = pool_param->uarea_init.init_fn; + ml_pool_param.uarea_init.args = pool_param->uarea_init.args; + ml_pool_param.buf.num = num; + ml_pool_param.buf.cache_size = cache_size; + ml_pool_param.buf.size = buf_size; + ml_pool_param.buf.uarea_size = uarea_size; + + pool = _odp_pool_create(name, &ml_pool_param, ODP_POOL_ML_COMPL); + + return pool; +} + +odp_ml_compl_t odp_ml_compl_alloc(odp_pool_t pool) +{ + odp_buffer_t buf; + odp_event_t ev; + odp_ml_run_result_t *result; + uint32_t buf_size = _ODP_MAX(sizeof(odp_ml_run_result_t), + sizeof(odp_ml_load_result_t)); + + buf = odp_buffer_alloc(pool); + + if (odp_unlikely(buf == ODP_BUFFER_INVALID)) + return ODP_ML_COMPL_INVALID; + + result = odp_buffer_addr(buf); + memset(result, 0, buf_size); + + ev = odp_buffer_to_event(buf); + _odp_event_type_set(ev, ODP_EVENT_ML_COMPL); + + return (odp_ml_compl_t)(uintptr_t)buf; +} + +void odp_ml_compl_free(odp_ml_compl_t ml_compl) +{ + odp_event_t ev; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)ml_compl; + + if (odp_unlikely(ml_compl == ODP_ML_COMPL_INVALID)) { + _ODP_ERR("Bad ML job completion handle\n"); + return; + } + + ev = odp_buffer_to_event(buf); + _odp_event_type_set(ev, ODP_EVENT_BUFFER); + + odp_buffer_free(buf); +} + +int odp_ml_compl_run_result(odp_ml_compl_t ml_compl, odp_ml_run_result_t *result) +{ + odp_event_subtype_t subtype; + odp_ml_run_result_t *run_result; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)ml_compl; + odp_event_t ev = odp_buffer_to_event(buf); + + if (odp_unlikely(ml_compl == ODP_ML_COMPL_INVALID)) { + _ODP_ERR("Given ML completion event is invalid\n"); + return -2; + } + + if (odp_event_types(ev, &subtype) != ODP_EVENT_ML_COMPL || + subtype != ODP_EVENT_ML_COMPL_RUN) { + _ODP_ERR("Given completion event has wrong event type or subtype\n"); + return -2; + } + + run_result = odp_buffer_addr(buf); + if (result) + *result = *run_result; + + return run_result->error_code ? -1 : 0; +} + +int odp_ml_compl_load_result(odp_ml_compl_t ml_compl, odp_ml_load_result_t *result) +{ + odp_event_subtype_t subtype; + odp_ml_load_result_t *load_result; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)ml_compl; + odp_event_t ev = odp_buffer_to_event(buf); + + if (odp_unlikely(ml_compl == ODP_ML_COMPL_INVALID)) { + _ODP_ERR("Given ML completion event is invalid\n"); + return -2; + } + + if (odp_event_types(ev, &subtype) != ODP_EVENT_ML_COMPL || + subtype != ODP_EVENT_ML_COMPL_LOAD) { + _ODP_ERR("Given completion event has wrong event type or subtype\n"); + return -2; + } + + load_result = odp_buffer_addr(buf); + if (result) + *result = *load_result; + + return load_result->error_code ? -1 : 0; +} + +void *odp_ml_compl_user_area(odp_ml_compl_t ml_compl) +{ + return odp_buffer_user_area((odp_buffer_t)(uintptr_t)ml_compl); +} + +odp_ml_compl_t odp_ml_compl_from_event(odp_event_t event) +{ + _ODP_ASSERT(_odp_event_hdr_field(event, int8_t, event_type) == ODP_EVENT_ML_COMPL); + + return (odp_ml_compl_t)(uintptr_t)event; +} + +odp_event_t odp_ml_compl_to_event(odp_ml_compl_t ml_compl) +{ + return (odp_event_t)(uintptr_t)ml_compl; +} + +uint64_t odp_ml_compl_to_u64(odp_ml_compl_t ml_compl) +{ + return (uint64_t)(uintptr_t)ml_compl; +} + +void odp_ml_compl_param_init(odp_ml_compl_param_t *compl_param) +{ + memset(compl_param, 0, sizeof(odp_ml_compl_param_t)); + + compl_param->queue = ODP_QUEUE_INVALID; + compl_param->event = ODP_EVENT_INVALID; +} + +int odp_ml_model_load(odp_ml_model_t model, odp_ml_load_result_t *result) +{ + odp_ml_load_result_t result_local; + int ret = -1; + ml_model_t *mdl = ml_model_from_handle(model); + + memset(&result_local, 0, sizeof(result_local)); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad ML model handle\n"); + result_local.error_code = ML_BAD_HDL; + goto load_fail; + } + + odp_ticketlock_lock(&mdl->lock); + if (odp_unlikely(mdl->state != ML_STATE_CREATED)) { + _ODP_ERR("Model has not been created yet or is already loaded\n"); + odp_ticketlock_unlock(&mdl->lock); + result_local.error_code = ML_NOT_CREATED; + goto load_fail; + } + + mdl->state = ML_STATE_LOADED; + odp_ticketlock_unlock(&mdl->lock); + ret = 0; + +load_fail: + if (result) + *result = result_local; + + return ret; +} + +static inline int check_compl_param(const odp_ml_compl_param_t *compl_param, + uint32_t max_compl_id, odp_bool_t is_load) +{ + odp_ml_config_t *config = &_odp_ml_glb->ml_config; + + switch (compl_param->mode) { + case ODP_ML_COMPL_MODE_POLL: + if (is_load && !(config->load_mode_mask & ODP_ML_COMPL_MODE_POLL)) { + _ODP_ERR("Poll mode loading/unloading is not configured\n"); + return -1; + } + + if (!is_load && !(config->run_mode_mask & ODP_ML_COMPL_MODE_POLL)) { + _ODP_ERR("Poll mode run is not configured\n"); + return -1; + } + + if (compl_param->compl_id > max_compl_id) { + _ODP_ERR("Bad compl_id: %u, exceeding model max completion id %u\n", + compl_param->compl_id, max_compl_id); + return -1; + } + break; + case ODP_ML_COMPL_MODE_EVENT: + if (is_load && !(config->load_mode_mask & ODP_ML_COMPL_MODE_EVENT)) { + _ODP_ERR("Event mode loading/unloading is not configured\n"); + return -1; + } + + if (!is_load && !(config->run_mode_mask & ODP_ML_COMPL_MODE_EVENT)) { + _ODP_ERR("Event mode run is not configured\n"); + return -1; + } + + if (compl_param->event == ODP_EVENT_INVALID || + compl_param->queue == ODP_QUEUE_INVALID) { + _ODP_ERR("Bad event or queue\n"); + return -1; + } + + if (odp_event_type(compl_param->event) != ODP_EVENT_ML_COMPL) { + _ODP_ERR("Bad completion event type\n"); + return -1; + } + break; + default: + /* Including ODP_ML_COMPL_MODE_SYNC, which is not supported by + * asynchrous functions (e.g. *_start()) either. + */ + _ODP_ERR("Invalid completion mode %u\n", compl_param->mode); + return -1; + } + + return 0; +} + +int odp_ml_model_load_start(odp_ml_model_t model, const odp_ml_compl_param_t *compl_param) +{ + int ret; + ml_model_t *mdl = ml_model_from_handle(model); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad model handle\n"); + return -1; + } + + if (odp_unlikely(check_compl_param(compl_param, mdl->max_compl_id, true))) + return -1; + + if (compl_param->mode == ODP_ML_COMPL_MODE_POLL) + odp_atomic_store_rel_u32(&mdl->compl_status[compl_param->compl_id], 0); + + ret = odp_ml_model_load(model, NULL); + + if (odp_unlikely(ret)) + return -1; + + /* Send a completion event to the given queue */ + if (compl_param->mode == ODP_ML_COMPL_MODE_EVENT) { + odp_ml_load_result_t *result; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)compl_param->event; + + _odp_buffer_subtype_set(buf, ODP_EVENT_ML_COMPL_LOAD); + + result = odp_buffer_addr(buf); + result->error_code = 0; + result->user_ptr = compl_param->user_ptr; + + if (odp_unlikely(odp_queue_enq(compl_param->queue, compl_param->event))) { + _ODP_ERR("Completion event enqueue failed %" PRIu64 "\n", + odp_queue_to_u64(compl_param->queue)); + if (odp_ml_model_unload(model, NULL)) + _ODP_ERR("Failed to unload model\n"); + return -1; + } + + return 0; + } + + mdl->result[compl_param->compl_id].user_ptr = compl_param->user_ptr; + odp_atomic_store_rel_u32(&mdl->compl_status[compl_param->compl_id], 1); + return 0; +} + +int odp_ml_model_load_status(odp_ml_model_t model, uint32_t compl_id, odp_ml_load_result_t *result) +{ + int ret; + ml_model_t *mdl = ml_model_from_handle(model); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID || compl_id > mdl->max_compl_id)) { + _ODP_ERR("Invalid model or compl_id: %u\n", compl_id); + return -2; + } + + ret = odp_atomic_load_acq_u32(&mdl->compl_status[compl_id]); + + if (ret && result) { + result->error_code = 0; + result->user_ptr = mdl->result[compl_id].user_ptr; + } + + return ret; +} + +int odp_ml_model_unload(odp_ml_model_t model, odp_ml_load_result_t *result) +{ + odp_ml_load_result_t result_local; + int ret = -1; + ml_model_t *mdl = ml_model_from_handle(model); + + memset(&result_local, 0, sizeof(result_local)); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + result_local.error_code = ML_BAD_HDL; + _ODP_ERR("Bad ML model handle\n"); + goto unload_fail; + } + + odp_ticketlock_lock(&mdl->lock); + /* mdl->state == ML_STATE_FREE, ML_STATE_CREATED, ML_STATE_INFERENCING */ + if (odp_unlikely(mdl->state != ML_STATE_LOADED)) { + _ODP_ERR("Model has not been created/loaded or inferencing has not finished yet\n"); + odp_ticketlock_unlock(&mdl->lock); + result_local.error_code = ML_NOT_LOADED; + goto unload_fail; + } + + mdl->state = ML_STATE_CREATED; + odp_ticketlock_unlock(&mdl->lock); + + ret = 0; + +unload_fail: + if (result) + *result = result_local; + + return ret; +} + +int odp_ml_model_unload_start(odp_ml_model_t model, const odp_ml_compl_param_t *compl_param) +{ + int ret; + ml_model_t *mdl = ml_model_from_handle(model); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad model handle\n"); + return -1; + } + + if (odp_unlikely(check_compl_param(compl_param, mdl->max_compl_id, true))) + return -1; + + if (compl_param->mode == ODP_ML_COMPL_MODE_POLL) + odp_atomic_store_rel_u32(&mdl->compl_status[compl_param->compl_id], 0); + + ret = odp_ml_model_unload(model, NULL); + + if (odp_unlikely(ret)) + return -1; + + /* Upon successful unloading, send a completion event to the given queue */ + if (compl_param->mode == ODP_ML_COMPL_MODE_EVENT) { + odp_ml_load_result_t *result; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)compl_param->event; + + _odp_buffer_subtype_set(buf, ODP_EVENT_ML_COMPL_LOAD); + + result = odp_buffer_addr(buf); + result->error_code = 0; + result->user_ptr = compl_param->user_ptr; + + if (odp_unlikely(odp_queue_enq(compl_param->queue, compl_param->event))) { + _ODP_ERR("Completion event enqueue failed %" PRIu64 "\n", + odp_queue_to_u64(compl_param->queue)); + return -1; + } + + return 0; + } + + mdl->result[compl_param->compl_id].user_ptr = compl_param->user_ptr; + odp_atomic_store_rel_u32(&mdl->compl_status[compl_param->compl_id], 1); + return 0; +} + +int odp_ml_model_unload_status(odp_ml_model_t model, uint32_t compl_id, + odp_ml_load_result_t *result) +{ + return odp_ml_model_load_status(model, compl_id, result); +} + +void odp_ml_run_param_init(odp_ml_run_param_t *param) +{ + memset(param, 0, sizeof(odp_ml_run_param_t)); +} + +static void ml_shape_to_int64(const odp_ml_shape_info_t *shape, uint32_t batch_size, int64_t *array) +{ + for (uint32_t i = 0; i < shape->num_dim; i++) { + /* Replace dynamic dimension size with provided batch_size */ + if (shape->dim[i] == ODP_ML_DIM_DYNAMIC) + array[i] = batch_size; + else + array[i] = shape->dim[i]; + } +} + +/* Get the number of elements in given shape */ +static inline uint64_t get_num_elem(uint32_t batch_size, const odp_ml_shape_info_t *shape) +{ + uint64_t num_elements = 1; + int64_t dim[ODP_ML_MAX_DIMS] = {0}; + + ml_shape_to_int64(shape, batch_size, dim); + + for (uint32_t i = 0; i < shape->num_dim; i++) + num_elements *= (uint64_t)dim[i]; + + return num_elements; +} + +static inline uint32_t dyn_io_size(const odp_ml_shape_info_t *shape, uint32_t data_type_size, + const odp_ml_run_param_t *param) +{ + uint32_t size; + + if (!param || !param->batch_size) { + _ODP_ERR("Parameter 'param' must not be NULL and batch_size must be " + "provided when a input/output has dynamic dimension size\n"); + return 0; + } + + size = get_num_elem(param->batch_size, shape); + size *= data_type_size; + + return size; +} + +static int verify_run_params(odp_ml_model_t model, const odp_ml_data_t *data, + const odp_ml_run_param_t *param) +{ + const ml_model_t *mdl = ml_model_from_handle(model); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad ML model handle\n"); + return -1; + } + + if (odp_unlikely(!data)) { + _ODP_ERR("Parameter 'data' must not be NULL\n"); + return -1; + } + + /* Make sure that the number of input data segments equals or bigger than + * the number of model inputs. */ + if (mdl->info.num_inputs > data->num_input_seg) { + _ODP_ERR("The num of input data segments %u must not less than " + "the number of model inputs %u\n", data->num_input_seg, + mdl->info.num_inputs); + return -1; + } + + if (mdl->info.num_outputs > data->num_output_seg) { + _ODP_ERR("The num of output data segments %u must not less than " + "the number of model outputs %u\n", data->num_output_seg, + mdl->info.num_outputs); + return -1; + } + + if (data->num_input_seg > mdl->info.num_inputs && + (_odp_ml_glb->capa.max_segs_per_input == 1)) { + _ODP_ERR("Segmented input data is not supported\n"); + return -1; + } + + if (data->num_output_seg > mdl->info.num_outputs && + (_odp_ml_glb->capa.max_segs_per_output == 1)) { + _ODP_ERR("Segmented output data is not supported"); + return -1; + } + + uint32_t size = 0; + uint32_t input_index = 0; + uint32_t seg_size_sum = 0; + odp_bool_t index_new = true; + uint32_t segs_per_input = 1; + + for (uint32_t i = 0; i < data->num_input_seg; i++) { + if (data->input_seg[i].addr == NULL) { + _ODP_ERR("data->input_seg[%u].addr must not NULL\n", i); + return -1; + }; + + if (index_new) { + if (input_index > mdl->info.num_inputs - 1) { + _ODP_ERR("Too much number of input segments given\n"); + return -1; + } + + /* Input with dynamic batch size */ + if (mdl->input_info[input_index].shape.type == ODP_ML_SHAPE_BATCH) + size = dyn_io_size(&mdl->input_info[input_index].shape, + mdl->input_info[input_index].data_type_size, + param); + else + size = mdl->input_sizes[input_index]; + + if (!size) { + _ODP_ERR("Size for %uth input is 0\n", input_index); + return -1; + } + } + + seg_size_sum += data->input_seg[i].size; + + if (seg_size_sum > size) { + _ODP_ERR("Sum of segment sizes %u exceeds %uth input data size %u\n", + seg_size_sum, input_index, size); + return -1; + } + + if (seg_size_sum == size) { + if (segs_per_input > _odp_ml_glb->capa.max_segs_per_input) { + _ODP_ERR("Number of segments %u for input[%u] exceeds maximum" + " number of data segments per model input %u\n", + segs_per_input, input_index, + _odp_ml_glb->capa.max_segs_per_input); + return -1; + } + input_index++; + index_new = true; + seg_size_sum = 0; + segs_per_input = 1; + } else { + segs_per_input++; + index_new = false; + } + } + + if (input_index != mdl->info.num_inputs) { + _ODP_ERR("Data is not provided for all model inputs\n"); + return -1; + } + + seg_size_sum = 0; + index_new = true; + uint32_t output_index = 0; + uint32_t segs_per_output = 1; + + for (uint32_t i = 0; i < data->num_output_seg; i++) { + if (data->output_seg[i].addr == NULL) { + _ODP_ERR("data->output_seg[%u].addr must not NULL\n", i); + return -1; + } + + if (index_new) { + if (output_index > mdl->info.num_outputs - 1) { + _ODP_ERR("Too much number of output segments given\n"); + return -1; + } + + /* Output with dynamic batch size */ + if (mdl->output_info[output_index].shape.type == ODP_ML_SHAPE_BATCH) + size = dyn_io_size(&mdl->output_info[output_index].shape, + mdl->output_info[output_index].data_type_size, + param); + else + size = mdl->output_sizes[output_index]; + + if (!size) { + _ODP_ERR("Size for %uth output is 0\n", output_index); + return -1; + } + } + + seg_size_sum += data->output_seg[i].size; + + if (seg_size_sum > size) { + _ODP_ERR("Sum of segment sizes %u exceeds %uth output data size %u\n", + seg_size_sum, output_index, size); + return -1; + } + + if (seg_size_sum >= size) { + if (segs_per_output > _odp_ml_glb->capa.max_segs_per_output) { + _ODP_ERR("Number of segments %u for output[%u] exceeds maximum" + " number of data segments per model output %u\n", + segs_per_output, output_index, + _odp_ml_glb->capa.max_segs_per_output); + return -1; + } + output_index++; + index_new = true; + seg_size_sum = 0; + segs_per_output = 1; + } else { + segs_per_output++; + index_new = false; + } + } + + if (output_index != mdl->info.num_outputs) { + _ODP_ERR("Not enough output_segs to hold all output data\n"); + return -1; + } + + return 0; +} + +static ONNXTensorElementDataType onnx_dtype_from_odp_dtype(odp_ml_data_type_t data_type) +{ + switch (data_type) { + case ODP_ML_DATA_TYPE_NONE: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + case ODP_ML_DATA_TYPE_INT8: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8; + case ODP_ML_DATA_TYPE_UINT8: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8; + case ODP_ML_DATA_TYPE_INT16: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16; + case ODP_ML_DATA_TYPE_UINT16: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16; + case ODP_ML_DATA_TYPE_INT24: + /* Fall through*/ + case ODP_ML_DATA_TYPE_UINT24: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + case ODP_ML_DATA_TYPE_FP64: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE; + case ODP_ML_DATA_TYPE_INT32: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32; + case ODP_ML_DATA_TYPE_UINT32: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32; + case ODP_ML_DATA_TYPE_INT64: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64; + case ODP_ML_DATA_TYPE_UINT64: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; + case ODP_ML_DATA_TYPE_FP16: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; + case ODP_ML_DATA_TYPE_FP32: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; + case ODP_ML_DATA_TYPE_BFP16: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16; + default: + return ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + } +} + +static int verify_tensor(const OrtValue *tensor, odp_ml_data_type_t expected_type, + const odp_ml_shape_info_t *expected_shape, uint32_t batch_size) +{ + OrtTensorTypeAndShapeInfo *tensor_info; + ONNXTensorElementDataType tensor_type; + size_t dim_count; + OrtStatus *status = NULL; + int64_t dims[ODP_ML_MAX_DIMS] = {0}; + int64_t shape_arr[ODP_ML_MAX_DIMS] = {0}; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + + status = ort_api->GetTensorTypeAndShape(tensor, &tensor_info); + if (check_ortstatus(status)) { + _ODP_ERR("GetTensorTypeAndShape() failed\n"); + return -1; + } + + status = ort_api->GetTensorElementType(tensor_info, &tensor_type); + if (check_ortstatus(status)) { + ort_api->ReleaseTensorTypeAndShapeInfo(tensor_info); + _ODP_ERR("GetTensorElementType() failed\n"); + return -1; + } + + if (onnx_dtype_to_odp_dtype(tensor_type) != expected_type) { + ort_api->ReleaseTensorTypeAndShapeInfo(tensor_info); + _ODP_ERR("Tensor type does not match model type\n"); + return -1; + } + + status = ort_api->GetDimensionsCount(tensor_info, &dim_count); + if (check_ortstatus(status)) { + ort_api->ReleaseTensorTypeAndShapeInfo(tensor_info); + _ODP_ERR("GetDimensionsCount() failed\n"); + return -1; + } + + if (dim_count != expected_shape->num_dim) { + ort_api->ReleaseTensorTypeAndShapeInfo(tensor_info); + _ODP_ERR("Tensor dimension does not match shape_dim\n"); + return -1; + } + + status = ort_api->GetDimensions(tensor_info, dims, dim_count); + if (check_ortstatus(status)) { + ort_api->ReleaseTensorTypeAndShapeInfo(tensor_info); + _ODP_ERR("GetDimensions() failed\n"); + return -1; + } + + ml_shape_to_int64(expected_shape, batch_size, shape_arr); + + for (uint32_t i = 0; i < dim_count; i++) { + if (dims[i] != shape_arr[i]) { + ort_api->ReleaseTensorTypeAndShapeInfo(tensor_info); + _ODP_ERR("Shape[%u]: %" PRIu64 " does not match expected: %" PRIu64 "\n", + i, dims[i], shape_arr[i]); + return -1; + } + } + + ort_api->ReleaseTensorTypeAndShapeInfo(tensor_info); + return 0; +} + +static int input_data_to_tensor(const odp_ml_input_info_t *input_info, uint32_t num_seg, + const odp_ml_data_seg_t *input_seg, uint32_t *seg_idx, + uint32_t batch_size, OrtValue **input_tensor) +{ + int is_tensor; + uint64_t input_size; + OrtAllocator *allocator; + void *data = NULL; + OrtStatus *status = NULL; + int64_t shape[ODP_ML_MAX_DIMS] = {0}; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + ONNXTensorElementDataType onnx_dtype = ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + + ml_shape_to_int64(&input_info->shape, batch_size, shape); + + onnx_dtype = onnx_dtype_from_odp_dtype(input_info->data_type); + _ODP_ASSERT(onnx_dtype != ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED); + + status = ort_api->GetAllocatorWithDefaultOptions(&allocator); + if (check_ortstatus(status)) { + _ODP_ERR("GetAllocatorWithDefaultOptions() failed\n"); + return -1; + } + + status = ort_api->CreateTensorAsOrtValue(allocator, + shape, + input_info->shape.num_dim, + onnx_dtype, + input_tensor); + if (check_ortstatus(status) || !input_tensor[0]) { + _ODP_ERR("CreateTensorWithDataAsOrtValue() failed\n"); + return -1; + } + + input_size = input_info->data_type_size * get_num_elem(batch_size, &input_info->shape); + + status = ort_api->GetTensorMutableData(input_tensor[0], &data); + if (check_ortstatus(status) || !data) { + _ODP_ERR("GetTensorMutableData() failed\n"); + return -1; + } + + for (uint64_t i = 0; i < input_size; ) { + if (*seg_idx >= num_seg) { + _ODP_ERR("Insufficient input data\n"); + return -1; + } + + uint64_t seg_size = input_seg[*seg_idx].size; + + if (i + seg_size > input_size) { + _ODP_ERR("Excess input data in segment %" PRIu32 "\n", *seg_idx); + return -1; + } + + memcpy((uint8_t *)data + i, input_seg[(*seg_idx)++].addr, seg_size); + i += seg_size; + } + + if (!ODP_DEBUG) + return 0; + + status = ort_api->IsTensor(input_tensor[0], &is_tensor); + if (check_ortstatus(status) || !is_tensor) { + _ODP_ERR("input_tensor IsTensor failed\n"); + return -1; + } + + /* Make sure tensor shape matches input_shape */ + if (verify_tensor(input_tensor[0], input_info->data_type, + &input_info->shape, batch_size)) { + _ODP_ERR("Verify input_tensor failed\n"); + return -1; + } + + return 0; +} + +static int verify_output_tensor(OrtValue *output_tensor, odp_ml_data_type_t expected_type, + const odp_ml_shape_info_t *expected_shape, uint32_t batch_size) +{ + int is_tensor = 0; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + OrtStatus *status = ort_api->IsTensor(output_tensor, &is_tensor); + + if (check_ortstatus(status) || !is_tensor) { + _ODP_ERR("output_tensor IsTensor failed\n"); + return -1; + } + + /* Make sure tensor shape matches output_shape */ + if (verify_tensor(output_tensor, expected_type, expected_shape, batch_size)) { + _ODP_ERR("Verify output_tensor failed\n"); + return -1; + } + + return 0; +} + +static int get_tensor_data_size(OrtValue *tensor, uint32_t *size, uint32_t data_type_size) +{ + size_t num_elem; + OrtStatus *status; + OrtTensorTypeAndShapeInfo *tensor_info; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + + status = ort_api->GetTensorTypeAndShape(tensor, &tensor_info); + if (check_ortstatus(status)) { + _ODP_ERR("GetTensorTypeAndShape() failed\n"); + return -1; + } + + status = ort_api->GetTensorShapeElementCount(tensor_info, &num_elem); + if (check_ortstatus(status)) { + ort_api->ReleaseTensorTypeAndShapeInfo(tensor_info); + _ODP_ERR("GetTensorShapeElementCount() failed\n"); + return -1; + } + *size = data_type_size * num_elem; + + ort_api->ReleaseTensorTypeAndShapeInfo(tensor_info); + return 0; +} + +static int check_output_size(odp_bool_t is_segmented, uint32_t output_idx, uint32_t seg_idx, + uint64_t out_tensor_data_size, const odp_ml_data_t data[]) +{ + uint64_t output_size = 0; + + /* Output is not segmented */ + if (!is_segmented) { + /* Make sure tensor data size does not exceed size allocated for + * data->output_seg[seg_idx].addr */ + if (out_tensor_data_size > data->output_seg[seg_idx].size) { + _ODP_ERR("Malloc at least %" PRIu64 " bytes for %dth output tensor\n", + out_tensor_data_size, output_idx); + return -1; + } + + return 0; + } + + /* Output is segmented, first calculate total size for one tensor */ + for (; seg_idx < data->num_output_seg; seg_idx++) { + output_size += data->output_seg[seg_idx].size; + if (output_size >= out_tensor_data_size) + break; + } + + if (0 == output_size) { + _ODP_ERR("No output data segments for %uth output tensor\n", output_idx); + return -1; + } + + if (out_tensor_data_size > output_size) { + _ODP_ERR("Output segments (%" PRIu64 " bytes in total) for %uth output" + " is expected to be at least %" PRIu64 " bytes\n", + output_size, output_idx, out_tensor_data_size); + return -1; + } + + return 0; +} + +static int output_tensors_to_data(OrtValue **output_tensors, + uint32_t model_num_outputs, + const odp_ml_run_param_t *param, + const odp_ml_output_info_t *output_info, + const odp_ml_data_t *data, + odp_ml_run_result_t *result_local) +{ + uint32_t seg_idx; + uint64_t seg_size; + uint64_t cpy_size; + uint64_t left_size; + uint64_t output_val_offset; + uint32_t out_tensor_data_size; + void *output_val = NULL; /* Pointer to store one raw output value */ + OrtStatus *status = NULL; + uint32_t batch_size = (param && param->batch_size) ? param->batch_size : 0; + const OrtApi *ort_api = _odp_ml_glb->ort_api; + odp_bool_t is_segmented = (data->num_output_seg != model_num_outputs); + + seg_idx = 0; + for (uint32_t i = 0; i < model_num_outputs; i++) { + if (ODP_DEBUG && + verify_output_tensor(output_tensors[i], output_info[i].data_type, + &output_info[i].shape, batch_size)){ + result_local->error_code = ML_BAD_OUTPUT; + return -1; + } + + /* Get tensor data size */ + if (get_tensor_data_size(output_tensors[i], &out_tensor_data_size, + output_info[i].data_type_size)) { + result_local->error_code = ML_LIB_FAILED; + return -1; + } + + /* When output_tensor is an empty tensor [], skip getting data */ + if (out_tensor_data_size == 0) + continue; + + if (ODP_DEBUG && check_output_size(is_segmented, i, seg_idx, + out_tensor_data_size, data)) { + result_local->error_code = ML_BAD_OUTPUT; + return -1; + } + + /* Following assumes param and data->output_seg are valid */ + /* Get tensor data */ + output_val = NULL; + status = ort_api->GetTensorMutableData(output_tensors[i], &output_val); + if (check_ortstatus(status) || !output_val) { + result_local->error_code = ML_LIB_FAILED; + return -1; + } + + /* Output is not segmented */ + if (!is_segmented) { + /* Store output data to data->output_seg[i].addr */ + memcpy(data->output_seg[i].addr, output_val, out_tensor_data_size); + seg_idx++; + continue; + } + + /* Output is segmented */ + output_val_offset = 0; + left_size = out_tensor_data_size; + for (; seg_idx < data->num_output_seg; seg_idx++) { + seg_size = data->output_seg[seg_idx].size; + cpy_size = left_size > seg_size ? seg_size : left_size; + memcpy(data->output_seg[seg_idx].addr, + ((char *)output_val) + output_val_offset, cpy_size); + + output_val_offset += cpy_size; + left_size = out_tensor_data_size - output_val_offset; + + if (!left_size) { + seg_idx++; + break; + } + } + } + + return 0; +} + +int odp_ml_run(odp_ml_model_t model, const odp_ml_data_t *data, const odp_ml_run_param_t *param) +{ + odp_ml_run_result_t result_local; + + int retval = -1; /* Return value of this function */ + int ret = 0; + OrtStatus *status = NULL; + uint32_t batch_size = 0; + + OrtValue *input_tensor[CONFIG_ML_MAX_INPUTS] = {0}; + OrtValue *output_tensors[CONFIG_ML_MAX_OUTPUTS] = {0}; + const char *input_names[CONFIG_ML_MAX_INPUTS] = {0}; + const char *output_names[CONFIG_ML_MAX_OUTPUTS] = {0}; + + const OrtApi *ort_api = _odp_ml_glb->ort_api; + ml_model_t *mdl = ml_model_from_handle(model); + const odp_ml_model_info_t *ml_info = &mdl->info; + const odp_ml_input_info_t *input_info = mdl->input_info; + const odp_ml_output_info_t *output_info = mdl->output_info; + OrtSession *session = mdl->session; + + odp_ticketlock_lock(&mdl->lock); + if (odp_unlikely(mdl->state == ML_STATE_INFERENCING)) { + odp_ticketlock_unlock(&mdl->lock); + return 0; + } + if (odp_unlikely(mdl->state != ML_STATE_LOADED)) { + _ODP_ERR("Wrong model state: not created or not loaded\n"); + odp_ticketlock_unlock(&mdl->lock); + return -1; + } + mdl->state = ML_STATE_INFERENCING; + odp_ticketlock_unlock(&mdl->lock); + + memset(&result_local, 0, sizeof(result_local)); + + if (ODP_DEBUG && verify_run_params(model, data, param)) { + result_local.error_code = ML_BAD_INPUT; + goto init_fail; + } + + if (param && param->batch_size) + batch_size = param->batch_size; + + uint32_t seg_idx = 0; + + /* Transfer input data to tensor */ + for (uint32_t i = 0; i < ml_info->num_inputs; i++) { + ret = input_data_to_tensor(&input_info[i], + data->num_input_seg, + data->input_seg, + &seg_idx, + batch_size, + &input_tensor[i]); + if (ret) { + _ODP_ERR("%uth input data to tensor failed\n", i); + result_local.error_code = ML_LIB_FAILED; + goto release_input_tensors; + } + + _ODP_DBG("input_tensor[%u]: %p\n", i, input_tensor[i]); + + /* Model input names */ + input_names[i] = input_info[i].name; + } + + if (seg_idx < data->num_input_seg) { + _ODP_ERR("Excess input segments\n"); + ret = -1; + } + + for (uint32_t i = 0; i < ml_info->num_outputs; i++) + output_names[i] = output_info[i].name; + + /* Run inference */ + status = ort_api->Run(session, + NULL, + (const char * const *)input_names, + (const OrtValue * const*)input_tensor, + ml_info->num_inputs, + (const char * const *)output_names, + ml_info->num_outputs, + output_tensors); + + if (check_ortstatus(status)) { + _ODP_ERR("Run inference failed\n"); + result_local.error_code = ML_LIB_FAILED; + goto release_all_tensors; + } + + /* Verify output tensors and store them to output */ + if (output_tensors_to_data(output_tensors, ml_info->num_outputs, param, + output_info, data, &result_local)) { + _ODP_ERR("Output tensors to data failed\n"); + goto release_all_tensors; + } + + retval = 1; + +release_all_tensors: + for (uint32_t i = 0; i < ml_info->num_outputs; i++) + ort_api->ReleaseValue(output_tensors[i]); + +release_input_tensors: + for (uint32_t i = 0; i < ml_info->num_inputs; i++) + ort_api->ReleaseValue(input_tensor[i]); + +init_fail: + if (param && param->result) + *param->result = result_local; + + odp_ticketlock_lock(&mdl->lock); + mdl->state = ML_STATE_LOADED; + odp_ticketlock_unlock(&mdl->lock); + + return retval; +} + +int odp_ml_run_multi(odp_ml_model_t model, const odp_ml_data_t data[], + const odp_ml_run_param_t param[], int num) +{ + int i; + int ret; + + if (odp_unlikely(num < 1)) { + _ODP_ERR("Bad number of runs\n"); + return -1; + } + + for (i = 0; i < num; i++) { + if (param) + ret = odp_ml_run(model, &data[i], ¶m[i]); + else + ret = odp_ml_run(model, &data[i], NULL); + + if (odp_unlikely(ret != 1)) + break; + } + + if (odp_unlikely(i == 0)) + return ret; + + return i; +} + +int odp_ml_run_start(odp_ml_model_t model, const odp_ml_data_t *data, + const odp_ml_compl_param_t *compl_param, + const odp_ml_run_param_t *run_param) +{ + int ret; + ml_model_t *mdl = ml_model_from_handle(model); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID)) { + _ODP_ERR("Bad model handle\n"); + return -1; + } + + if (odp_unlikely(!compl_param)) { + _ODP_ERR("Completion parameter is NULL\n"); + return -1; + } + + /* Check completion mode */ + if (odp_unlikely(check_compl_param(compl_param, mdl->max_compl_id, false))) { + _ODP_ERR("Bad ML job completion parameter\n"); + return -1; + } + + if (compl_param->mode == ODP_ML_COMPL_MODE_POLL) + odp_atomic_store_rel_u32(&mdl->compl_status[compl_param->compl_id], 0); + + ret = odp_ml_run(model, data, run_param); + + if (odp_unlikely(ret < 1)) + return ret; + + /* Send a completion event to the given queue */ + if (compl_param->mode == ODP_ML_COMPL_MODE_EVENT) { + odp_ml_run_result_t *result; + odp_buffer_t buf = (odp_buffer_t)(uintptr_t)compl_param->event; + + _odp_buffer_subtype_set(buf, ODP_EVENT_ML_COMPL_RUN); + + result = odp_buffer_addr(buf); + result->error_code = 0; + result->user_ptr = compl_param->user_ptr; + + if (odp_unlikely(odp_queue_enq(compl_param->queue, compl_param->event))) { + _ODP_ERR("Completion event enqueue failed %" PRIu64 "\n", + odp_queue_to_u64(compl_param->queue)); + return -1; + } + + return 1; + } + + /* compl_param->mode == ODP_ML_COMPL_MODE_POLL */ + mdl->result[compl_param->compl_id].user_ptr = compl_param->user_ptr; + odp_atomic_store_rel_u32(&mdl->compl_status[compl_param->compl_id], 1); + + return 1; +} + +int odp_ml_run_start_multi(odp_ml_model_t model, const odp_ml_data_t data[], + const odp_ml_compl_param_t compl_param[], + const odp_ml_run_param_t run_param[], int num) +{ + int i; + int ret = 0; + + if (odp_unlikely(num < 1)) { + _ODP_ERR("Bad number of runs\n"); + return -1; + } + + for (i = 0; i < num; i++) { + if (run_param) + ret = odp_ml_run_start(model, &data[i], &compl_param[i], &run_param[i]); + else + ret = odp_ml_run_start(model, &data[i], &compl_param[i], NULL); + + if (odp_unlikely(ret != 1)) + break; + } + + if (odp_unlikely(i == 0)) + return ret; + + return i; +} + +int odp_ml_run_status(odp_ml_model_t model, uint32_t compl_id, odp_ml_run_result_t *result) +{ + int ret; + ml_model_t *mdl = ml_model_from_handle(model); + + if (odp_unlikely(model == ODP_ML_MODEL_INVALID || + compl_id > mdl->max_compl_id)) { + _ODP_ERR("Invalid model handle or completion id: %u\n", compl_id); + return -2; + } + + ret = odp_atomic_load_acq_u32(&mdl->compl_status[compl_id]); + + if (result) { + result->error_code = 0; + result->user_ptr = mdl->result[compl_id].user_ptr; + } + + return ret; +} + +static int opt_level_from_str(const char *level_str, GraphOptimizationLevel *level) +{ + if (strcmp(level_str, "DISABLE_ALL") == 0) + *level = ORT_DISABLE_ALL; + else if (strcmp(level_str, "ENABLE_BASIC") == 0) + *level = ORT_ENABLE_BASIC; + else if (strcmp(level_str, "ENABLE_EXTENDED") == 0) + *level = ORT_ENABLE_EXTENDED; + else if (strcmp(level_str, "ENABLE_ALL") == 0) + *level = ORT_ENABLE_ALL; + else + return -1; + + return 0; +} + +static int execution_mode_from_str(const char *mode_str, ExecutionMode *mode) +{ + if (strcmp(mode_str, "SEQUENTIAL") == 0) + *mode = ORT_SEQUENTIAL; + else if (strcmp(mode_str, "PARALLEL") == 0) + *mode = ORT_PARALLEL; + else + return -1; + + return 0; +} + +static int read_config_file(ort_run_opts_t *opts) +{ + const char *conf_str; + char mode_str[ML_MAX_CONFIG_STR_LEN]; + char opt_level_str[ML_MAX_CONFIG_STR_LEN]; + + _ODP_PRINT("ML config:\n"); + + conf_str = "ml.enable_profiling"; + if (!_odp_libconfig_lookup_int(conf_str, &opts->enable_profiling)) { + _ODP_ERR("Config option '%s' not found.\n", conf_str); + return -1; + } + _ODP_PRINT(" %s: %i\n", conf_str, opts->enable_profiling); + + conf_str = "ml.execution_mode"; + if (_odp_libconfig_lookup_str(conf_str, mode_str, ML_MAX_CONFIG_STR_LEN) < 0) { + _ODP_ERR("Config option '%s' not found.\n", conf_str); + return -1; + } + + if (execution_mode_from_str(mode_str, &opts->execution_mode)) { + _ODP_ERR("Unsupported execution mode: %s\n", mode_str); + return -1; + } + _ODP_PRINT(" %s: %s\n", conf_str, mode_str); + + conf_str = "ml.inter_op_num_threads"; + if (!_odp_libconfig_lookup_int(conf_str, &opts->inter_op_num_threads)) { + _ODP_ERR("Config option '%s' not found.\n", conf_str); + return -1; + } + _ODP_PRINT(" %s: %i\n", conf_str, opts->inter_op_num_threads); + + conf_str = "ml.intra_op_num_threads"; + if (!_odp_libconfig_lookup_int(conf_str, &opts->intra_op_num_threads)) { + _ODP_ERR("Config option '%s' not found.\n", conf_str); + return -1; + } + _ODP_PRINT(" %s: %i\n", conf_str, opts->intra_op_num_threads); + + conf_str = "ml.graph_optimization_level"; + if (_odp_libconfig_lookup_str(conf_str, opt_level_str, + ML_MAX_CONFIG_STR_LEN) < 0) { + _ODP_ERR("Config option '%s' not found.\n", conf_str); + return -1; + } + + if (opt_level_from_str(opt_level_str, &opts->graph_opt_level)) { + _ODP_ERR("Graph optimize level %s not supported\n", opt_level_str); + return -1; + } + _ODP_PRINT(" %s: %s\n", conf_str, opt_level_str); + + conf_str = "ml.optimized_model_filepath"; + if (_odp_libconfig_lookup_str(conf_str, opts->opt_model_filepath, + ML_MAX_CONFIG_STR_LEN) < 0) { + _ODP_ERR("Config option '%s' not found.\n", conf_str); + return -1; + } + _ODP_PRINT(" %s: %s\n", conf_str, opts->opt_model_filepath); + + return 0; +} + +int _odp_ml_init_global(void) +{ + int i; + OrtEnv *env; + odp_shm_t shm; + OrtStatus *status; + const OrtApi *ort_api; + + if (odp_global_ro.disable.ml) { + _ODP_ERR("ML is disabled\n"); + return 0; + } + + shm = odp_shm_reserve("_odp_ml_global", sizeof(ml_global_t), ODP_CACHE_LINE_SIZE, 0); + _odp_ml_glb = odp_shm_addr(shm); + + if (_odp_ml_glb == NULL) { + _ODP_ERR("SHM reserve failed for odp_ml\n"); + return -1; + } + + memset(_odp_ml_glb, 0, sizeof(ml_global_t)); + _odp_ml_glb->shm = shm; + + if (odp_ml_capability(&_odp_ml_glb->capa)) { + _ODP_ERR("ML capability failed\n"); + return -1; + } + + odp_pool_param_init(&_odp_ml_glb->pool_param); + + if (read_config_file(&_odp_ml_glb->ort_run_opts)) + return -1; + + ort_api = OrtGetApiBase()->GetApi(ORT_API_VERSION); + if (!ort_api) { + _ODP_ERR("Failed to init ONNX Runtime engine.\n"); + return -1; + } + _odp_ml_glb->ort_api = ort_api; + + status = ort_api->CreateEnv(ORT_LOGGING_LEVEL_WARNING, "Default", &env); + if (check_ortstatus(status) || !env) { + _ODP_ERR("ort_api->CreateEnv() failed.\n"); + return -1; + } + _odp_ml_glb->env = env; + + for (i = 0; i < ML_MAX_MODELS_CREATED; i++) + odp_ticketlock_init(&_odp_ml_glb->models[i].lock); + + return 0; +} + +int _odp_ml_term_global(void) +{ + if (odp_global_ro.disable.ml) + return 0; + + if (_odp_ml_glb == NULL) + return 0; + + if (_odp_ml_glb->env) + _odp_ml_glb->ort_api->ReleaseEnv(_odp_ml_glb->env); + + if (odp_shm_free(_odp_ml_glb->shm)) { + _ODP_ERR("Shm free failed for odp_ml\n"); + return -1; + } + + return 0; +} diff --git a/platform/linux-generic/odp_ml_fp16.c b/platform/linux-generic/odp_ml_fp16.c new file mode 100644 index 000000000..47b10f841 --- /dev/null +++ b/platform/linux-generic/odp_ml_fp16.c @@ -0,0 +1,425 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2022-2023 Marvell. + * Copyright (c) 2023 Nokia + * + * Based on + * - dpdk/lib/mldev/mldev_utils_scalar.h + * - dpdk/lib/mldev/mldev_utils_scalar.c + * - dpdk/lib/mldev/mldev_utils_scalar_bfloat16.c + */ + +#include <odp_ml_fp16.h> + +#include <errno.h> +#include <stdint.h> + +#ifndef BIT +#define BIT(nr) (1UL << (nr)) +#endif + +#ifndef BITS_PER_LONG +#define BITS_PER_LONG (__SIZEOF_LONG__ * 8) +#endif + +#ifndef GENMASK_U32 +#define GENMASK_U32(h, l) (((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h)))) +#endif + +/* float32: bit index of MSB & LSB of sign, exponent and mantissa */ +#define FP32_LSB_M 0 +#define FP32_MSB_M 22 +#define FP32_LSB_E 23 +#define FP32_MSB_E 30 +#define FP32_LSB_S 31 +#define FP32_MSB_S 31 + +/* float32: bitmask for sign, exponent and mantissa */ +#define FP32_MASK_S GENMASK_U32(FP32_MSB_S, FP32_LSB_S) +#define FP32_MASK_E GENMASK_U32(FP32_MSB_E, FP32_LSB_E) +#define FP32_MASK_M GENMASK_U32(FP32_MSB_M, FP32_LSB_M) + +/* float16: bit index of MSB & LSB of sign, exponent and mantissa */ +#define FP16_LSB_M 0 +#define FP16_MSB_M 9 +#define FP16_LSB_E 10 +#define FP16_MSB_E 14 +#define FP16_LSB_S 15 +#define FP16_MSB_S 15 + +/* float16: bitmask for sign, exponent and mantissa */ +#define FP16_MASK_S GENMASK_U32(FP16_MSB_S, FP16_LSB_S) +#define FP16_MASK_E GENMASK_U32(FP16_MSB_E, FP16_LSB_E) +#define FP16_MASK_M GENMASK_U32(FP16_MSB_M, FP16_LSB_M) + +/* bfloat16: bit index of MSB & LSB of sign, exponent and mantissa */ +#define BF16_LSB_M 0 +#define BF16_MSB_M 6 +#define BF16_LSB_E 7 +#define BF16_MSB_E 14 +#define BF16_LSB_S 15 +#define BF16_MSB_S 15 + +/* bfloat16: bitmask for sign, exponent and mantissa */ +#define BF16_MASK_S GENMASK_U32(BF16_MSB_S, BF16_LSB_S) +#define BF16_MASK_E GENMASK_U32(BF16_MSB_E, BF16_LSB_E) +#define BF16_MASK_M GENMASK_U32(BF16_MSB_M, BF16_LSB_M) + +/* Exponent bias */ +#define FP32_BIAS_E 127 +#define FP16_BIAS_E 15 +#define BF16_BIAS_E 127 + +#define FP32_PACK(sign, exponent, mantissa) \ + (((sign) << FP32_LSB_S) | ((exponent) << FP32_LSB_E) | (mantissa)) + +#define FP16_PACK(sign, exponent, mantissa) \ + (((sign) << FP16_LSB_S) | ((exponent) << FP16_LSB_E) | (mantissa)) + +#define BF16_PACK(sign, exponent, mantissa) \ + (((sign) << BF16_LSB_S) | ((exponent) << BF16_LSB_E) | (mantissa)) + +/* Represent float32 as float and uint32_t */ +union float32 { + float f; + uint32_t u; +}; + +/* Convert a single precision floating point number (float32) into a half precision + * floating point number (float16) using round to nearest rounding mode. + */ +static uint16_t +__float32_to_float16_scalar_rtn(float x) +{ + union float32 f32; /* float32 input */ + uint32_t f32_s; /* float32 sign */ + uint32_t f32_e; /* float32 exponent */ + uint32_t f32_m; /* float32 mantissa */ + uint16_t f16_s; /* float16 sign */ + uint16_t f16_e; /* float16 exponent */ + uint16_t f16_m; /* float16 mantissa */ + uint32_t tbits; /* number of truncated bits */ + uint32_t tmsb; /* MSB position of truncated bits */ + uint32_t m_32; /* temporary float32 mantissa */ + uint16_t m_16; /* temporary float16 mantissa */ + uint16_t u16; /* float16 output */ + int be_16; /* float16 biased exponent, signed */ + + f32.f = x; + f32_s = (f32.u & FP32_MASK_S) >> FP32_LSB_S; + f32_e = (f32.u & FP32_MASK_E) >> FP32_LSB_E; + f32_m = (f32.u & FP32_MASK_M) >> FP32_LSB_M; + + f16_s = f32_s; + f16_e = 0; + f16_m = 0; + + switch (f32_e) { + case (0): /* float32: zero or subnormal number */ + f16_e = 0; + f16_m = 0; /* convert to zero */ + break; + case (FP32_MASK_E >> FP32_LSB_E): /* float32: infinity or nan */ + f16_e = FP16_MASK_E >> FP16_LSB_E; + if (f32_m == 0) { /* infinity */ + f16_m = 0; + } else { /* nan, propagate mantissa and set MSB of mantissa to 1 */ + f16_m = f32_m >> (FP32_MSB_M - FP16_MSB_M); + f16_m |= BIT(FP16_MSB_M); + } + break; + default: /* float32: normal number */ + /* compute biased exponent for float16 */ + be_16 = (int)f32_e - FP32_BIAS_E + FP16_BIAS_E; + + /* overflow, be_16 = [31-INF], set to infinity */ + if (be_16 >= (int)(FP16_MASK_E >> FP16_LSB_E)) { + f16_e = FP16_MASK_E >> FP16_LSB_E; + f16_m = 0; + } else if ((be_16 >= 1) && (be_16 < (int)(FP16_MASK_E >> FP16_LSB_E))) { + /* normal float16, be_16 = [1:30]*/ + f16_e = be_16; + m_16 = f32_m >> (FP32_LSB_E - FP16_LSB_E); + tmsb = FP32_MSB_M - FP16_MSB_M - 1; + if ((f32_m & GENMASK_U32(tmsb, 0)) > BIT(tmsb)) { + /* round: non-zero truncated bits except MSB */ + m_16++; + + /* overflow into exponent */ + if (((m_16 & FP16_MASK_E) >> FP16_LSB_E) == 0x1) + f16_e++; + } else if ((f32_m & GENMASK_U32(tmsb, 0)) == BIT(tmsb)) { + /* round: MSB of truncated bits and LSB of m_16 is set */ + if ((m_16 & 0x1) == 0x1) { + m_16++; + + /* overflow into exponent */ + if (((m_16 & FP16_MASK_E) >> FP16_LSB_E) == 0x1) + f16_e++; + } + } + f16_m = m_16 & FP16_MASK_M; + } else if ((be_16 >= -(int)(FP16_MSB_M)) && (be_16 < 1)) { + /* underflow: zero / subnormal, be_16 = [-9:0] */ + f16_e = 0; + + /* add implicit leading zero */ + m_32 = f32_m | BIT(FP32_LSB_E); + tbits = FP32_LSB_E - FP16_LSB_E - be_16 + 1; + m_16 = m_32 >> tbits; + + /* if non-leading truncated bits are set */ + if ((f32_m & GENMASK_U32(tbits - 1, 0)) > BIT(tbits - 1)) { + m_16++; + + /* overflow into exponent */ + if (((m_16 & FP16_MASK_E) >> FP16_LSB_E) == 0x1) + f16_e++; + } else if ((f32_m & GENMASK_U32(tbits - 1, 0)) == BIT(tbits - 1)) { + /* if leading truncated bit is set */ + if ((m_16 & 0x1) == 0x1) { + m_16++; + + /* overflow into exponent */ + if (((m_16 & FP16_MASK_E) >> FP16_LSB_E) == 0x1) + f16_e++; + } + } + f16_m = m_16 & FP16_MASK_M; + } else if (be_16 == -(int)(FP16_MSB_M + 1)) { + /* underflow: zero, be_16 = [-10] */ + f16_e = 0; + if (f32_m != 0) + f16_m = 1; + else + f16_m = 0; + } else { + /* underflow: zero, be_16 = [-INF:-11] */ + f16_e = 0; + f16_m = 0; + } + + break; + } + + u16 = FP16_PACK(f16_s, f16_e, f16_m); + + return u16; +} + +/* Convert a half precision floating point number (float16) into a single precision + * floating point number (float32). + */ +static float +__float16_to_float32_scalar_rtx(uint16_t f16) +{ + union float32 f32; /* float32 output */ + uint16_t f16_s; /* float16 sign */ + uint16_t f16_e; /* float16 exponent */ + uint16_t f16_m; /* float16 mantissa */ + uint32_t f32_s; /* float32 sign */ + uint32_t f32_e; /* float32 exponent */ + uint32_t f32_m; /* float32 mantissa*/ + uint8_t shift; /* number of bits to be shifted */ + uint32_t clz; /* count of leading zeroes */ + int e_16; /* float16 exponent unbiased */ + + f16_s = (f16 & FP16_MASK_S) >> FP16_LSB_S; + f16_e = (f16 & FP16_MASK_E) >> FP16_LSB_E; + f16_m = (f16 & FP16_MASK_M) >> FP16_LSB_M; + + f32_s = f16_s; + switch (f16_e) { + case (FP16_MASK_E >> FP16_LSB_E): /* float16: infinity or nan */ + f32_e = FP32_MASK_E >> FP32_LSB_E; + if (f16_m == 0x0) { /* infinity */ + f32_m = f16_m; + } else { /* nan, propagate mantissa, set MSB of mantissa to 1 */ + f32_m = f16_m; + shift = FP32_MSB_M - FP16_MSB_M; + f32_m = (f32_m << shift) & FP32_MASK_M; + f32_m |= BIT(FP32_MSB_M); + } + break; + case 0: /* float16: zero or sub-normal */ + f32_m = f16_m; + if (f16_m == 0) { /* zero signed */ + f32_e = 0; + } else { /* subnormal numbers */ + clz = __builtin_clz((uint32_t)f16_m) - sizeof(uint32_t) * 8 + FP16_LSB_E; + e_16 = (int)f16_e - clz; + f32_e = FP32_BIAS_E + e_16 - FP16_BIAS_E; + + shift = clz + (FP32_MSB_M - FP16_MSB_M) + 1; + f32_m = (f32_m << shift) & FP32_MASK_M; + } + break; + default: /* normal numbers */ + f32_m = f16_m; + e_16 = (int)f16_e; + f32_e = FP32_BIAS_E + e_16 - FP16_BIAS_E; + + shift = (FP32_MSB_M - FP16_MSB_M); + f32_m = (f32_m << shift) & FP32_MASK_M; + } + + f32.u = FP32_PACK(f32_s, f32_e, f32_m); + + return f32.f; +} + +/* Convert a single precision floating point number (float32) into a + * brain float number (bfloat16) using round to nearest rounding mode. + */ +static uint16_t +__float32_to_bfloat16_scalar_rtn(float x) +{ + union float32 f32; /* float32 input */ + uint32_t f32_s; /* float32 sign */ + uint32_t f32_e; /* float32 exponent */ + uint32_t f32_m; /* float32 mantissa */ + uint16_t b16_s; /* float16 sign */ + uint16_t b16_e; /* float16 exponent */ + uint16_t b16_m; /* float16 mantissa */ + uint32_t tbits; /* number of truncated bits */ + uint16_t u16; /* float16 output */ + + f32.f = x; + f32_s = (f32.u & FP32_MASK_S) >> FP32_LSB_S; + f32_e = (f32.u & FP32_MASK_E) >> FP32_LSB_E; + f32_m = (f32.u & FP32_MASK_M) >> FP32_LSB_M; + + b16_s = f32_s; + b16_e = 0; + b16_m = 0; + + switch (f32_e) { + case (0): /* float32: zero or subnormal number */ + b16_e = 0; + if (f32_m == 0) /* zero */ + b16_m = 0; + else /* subnormal float32 number, normal bfloat16 */ + goto bf16_normal; + break; + case (FP32_MASK_E >> FP32_LSB_E): /* float32: infinity or nan */ + b16_e = BF16_MASK_E >> BF16_LSB_E; + if (f32_m == 0) { /* infinity */ + b16_m = 0; + } else { /* nan, propagate mantissa and set MSB of mantissa to 1 */ + b16_m = f32_m >> (FP32_MSB_M - BF16_MSB_M); + b16_m |= BIT(BF16_MSB_M); + } + break; + default: /* float32: normal number, normal bfloat16 */ + goto bf16_normal; + } + + goto bf16_pack; + +bf16_normal: + b16_e = f32_e; + tbits = FP32_MSB_M - BF16_MSB_M; + b16_m = f32_m >> tbits; + + /* if non-leading truncated bits are set */ + if ((f32_m & GENMASK_U32(tbits - 1, 0)) > BIT(tbits - 1)) { + b16_m++; + + /* if overflow into exponent */ + if (((b16_m & BF16_MASK_E) >> BF16_LSB_E) == 0x1) + b16_e++; + } else if ((f32_m & GENMASK_U32(tbits - 1, 0)) == BIT(tbits - 1)) { + /* if only leading truncated bit is set */ + if ((b16_m & 0x1) == 0x1) { + b16_m++; + + /* if overflow into exponent */ + if (((b16_m & BF16_MASK_E) >> BF16_LSB_E) == 0x1) + b16_e++; + } + } + b16_m = b16_m & BF16_MASK_M; + +bf16_pack: + u16 = BF16_PACK(b16_s, b16_e, b16_m); + + return u16; +} + +/* Convert a brain float number (bfloat16) into a + * single precision floating point number (float32). + */ +static float +__bfloat16_to_float32_scalar_rtx(uint16_t f16) +{ + union float32 f32; /* float32 output */ + uint16_t b16_s; /* float16 sign */ + uint16_t b16_e; /* float16 exponent */ + uint16_t b16_m; /* float16 mantissa */ + uint32_t f32_s; /* float32 sign */ + uint32_t f32_e; /* float32 exponent */ + uint32_t f32_m; /* float32 mantissa*/ + uint8_t shift; /* number of bits to be shifted */ + + b16_s = (f16 & BF16_MASK_S) >> BF16_LSB_S; + b16_e = (f16 & BF16_MASK_E) >> BF16_LSB_E; + b16_m = (f16 & BF16_MASK_M) >> BF16_LSB_M; + + f32_s = b16_s; + switch (b16_e) { + case (BF16_MASK_E >> BF16_LSB_E): /* bfloat16: infinity or nan */ + f32_e = FP32_MASK_E >> FP32_LSB_E; + if (b16_m == 0x0) { /* infinity */ + f32_m = 0; + } else { /* nan, propagate mantissa, set MSB of mantissa to 1 */ + f32_m = b16_m; + shift = FP32_MSB_M - BF16_MSB_M; + f32_m = (f32_m << shift) & FP32_MASK_M; + f32_m |= BIT(FP32_MSB_M); + } + break; + case 0: /* bfloat16: zero or subnormal */ + f32_m = b16_m; + if (b16_m == 0) { /* zero signed */ + f32_e = 0; + } else { /* subnormal numbers */ + goto fp32_normal; + } + break; + default: /* bfloat16: normal number */ + goto fp32_normal; + } + + goto fp32_pack; + +fp32_normal: + f32_m = b16_m; + f32_e = FP32_BIAS_E + b16_e - BF16_BIAS_E; + + shift = (FP32_MSB_M - BF16_MSB_M); + f32_m = (f32_m << shift) & FP32_MASK_M; + +fp32_pack: + f32.u = FP32_PACK(f32_s, f32_e, f32_m); + + return f32.f; +} + +uint16_t _odp_float32_to_float16(float x) +{ + return __float32_to_float16_scalar_rtn(x); +} + +float _odp_float16_to_float32(uint16_t f16) +{ + return __float16_to_float32_scalar_rtx(f16); +} + +uint16_t _odp_float32_to_bfloat16(float x) +{ + return __float32_to_bfloat16_scalar_rtn(x); +} + +float _odp_bfloat16_to_float32(uint16_t f16) +{ + return __bfloat16_to_float32_scalar_rtx(f16); +} diff --git a/platform/linux-generic/odp_ml_null.c b/platform/linux-generic/odp_ml_null.c new file mode 100644 index 000000000..718e80d76 --- /dev/null +++ b/platform/linux-generic/odp_ml_null.c @@ -0,0 +1,232 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#include <odp/api/hints.h> +#include <odp/api/ml.h> + +#include <odp_init_internal.h> + +#include <stdint.h> +#include <string.h> + +/* Dummy ML API implementation, no capability and just return error for + * other functions. + */ +int _odp_ml_init_global(void) +{ + return 0; +} + +int _odp_ml_term_global(void) +{ + return 0; +} + +int odp_ml_capability(odp_ml_capability_t *capa) +{ + memset(capa, 0, sizeof(odp_ml_capability_t)); + return 0; +} + +void odp_ml_config_init(odp_ml_config_t *config ODP_UNUSED) +{ +} + +int odp_ml_config(const odp_ml_config_t *config ODP_UNUSED) +{ + return -1; +} + +void odp_ml_model_param_init(odp_ml_model_param_t *param ODP_UNUSED) +{ +} + +odp_ml_model_t odp_ml_model_create(const char *name ODP_UNUSED, + const odp_ml_model_param_t *param ODP_UNUSED) +{ + return ODP_ML_MODEL_INVALID; +} + +int odp_ml_model_destroy(odp_ml_model_t model ODP_UNUSED) +{ + return -1; +} + +int odp_ml_model_info(odp_ml_model_t model ODP_UNUSED, odp_ml_model_info_t *info ODP_UNUSED) +{ + return -1; +} + +uint32_t odp_ml_model_input_info(odp_ml_model_t model ODP_UNUSED, + odp_ml_input_info_t info[] ODP_UNUSED, + uint32_t num ODP_UNUSED) +{ + return 0; +} + +uint32_t odp_ml_model_output_info(odp_ml_model_t model ODP_UNUSED, + odp_ml_output_info_t info[] ODP_UNUSED, + uint32_t num ODP_UNUSED) +{ + return 0; +} + +odp_ml_model_t odp_ml_model_lookup(const char *name ODP_UNUSED) +{ + return ODP_ML_MODEL_INVALID; +} + +uint64_t odp_ml_model_to_u64(odp_ml_model_t model ODP_UNUSED) +{ + return 0; +} + +void odp_ml_model_print(odp_ml_model_t model ODP_UNUSED) +{ +} + +void odp_ml_print(void) +{ +} + +void odp_ml_compl_pool_param_init(odp_ml_compl_pool_param_t *pool_param) +{ + memset(pool_param, 0, sizeof(odp_ml_compl_pool_param_t)); +} + +odp_pool_t odp_ml_compl_pool_create(const char *name ODP_UNUSED, + const odp_ml_compl_pool_param_t *pool_param ODP_UNUSED) +{ + return ODP_POOL_INVALID; +} + +odp_ml_compl_t odp_ml_compl_alloc(odp_pool_t pool ODP_UNUSED) +{ + return ODP_ML_COMPL_INVALID; +} + +void odp_ml_compl_free(odp_ml_compl_t ml_compl ODP_UNUSED) +{ +} + +int odp_ml_compl_run_result(odp_ml_compl_t ml_compl ODP_UNUSED, + odp_ml_run_result_t *result ODP_UNUSED) +{ + return -1; +} + +int odp_ml_compl_load_result(odp_ml_compl_t ml_compl ODP_UNUSED, + odp_ml_load_result_t *result ODP_UNUSED) +{ + return -1; +} + +void *odp_ml_compl_user_area(odp_ml_compl_t ml_compl ODP_UNUSED) +{ + return NULL; +} + +odp_ml_compl_t odp_ml_compl_from_event(odp_event_t event ODP_UNUSED) +{ + return ODP_ML_COMPL_INVALID; +} + +odp_event_t odp_ml_compl_to_event(odp_ml_compl_t ml_compl ODP_UNUSED) +{ + return ODP_EVENT_INVALID; +} + +uint64_t odp_ml_compl_to_u64(odp_ml_compl_t ml_compl ODP_UNUSED) +{ + return 0; +} + +void odp_ml_compl_param_init(odp_ml_compl_param_t *compl_param ODP_UNUSED) +{ +} + +int odp_ml_model_load(odp_ml_model_t model ODP_UNUSED, odp_ml_load_result_t *result ODP_UNUSED) +{ + return -1; +} + +int odp_ml_model_load_start(odp_ml_model_t model ODP_UNUSED, + const odp_ml_compl_param_t *compl_param ODP_UNUSED) +{ + return -1; +} + +int odp_ml_model_load_status(odp_ml_model_t model ODP_UNUSED, uint32_t compl_id ODP_UNUSED, + odp_ml_load_result_t *result ODP_UNUSED) +{ + return -1; +} + +int odp_ml_model_unload(odp_ml_model_t model ODP_UNUSED, odp_ml_load_result_t *result ODP_UNUSED) +{ + return -1; +} + +int odp_ml_model_unload_start(odp_ml_model_t model ODP_UNUSED, + const odp_ml_compl_param_t *compl_param ODP_UNUSED) +{ + return -1; +} + +int odp_ml_model_unload_status(odp_ml_model_t model ODP_UNUSED, uint32_t compl_id ODP_UNUSED, + odp_ml_load_result_t *result ODP_UNUSED) +{ + return -1; +} + +void odp_ml_run_param_init(odp_ml_run_param_t *param ODP_UNUSED) +{ +} + +int odp_ml_run(odp_ml_model_t model ODP_UNUSED, const odp_ml_data_t *data ODP_UNUSED, + const odp_ml_run_param_t *param ODP_UNUSED) +{ + return -1; +} + +int odp_ml_run_multi(odp_ml_model_t model ODP_UNUSED, const odp_ml_data_t data[] ODP_UNUSED, + const odp_ml_run_param_t param[] ODP_UNUSED, int num ODP_UNUSED) +{ + return -1; +} + +int odp_ml_run_start(odp_ml_model_t model ODP_UNUSED, const odp_ml_data_t *data ODP_UNUSED, + const odp_ml_compl_param_t *compl_param ODP_UNUSED, + const odp_ml_run_param_t *run_param ODP_UNUSED) +{ + return -1; +} + +int odp_ml_run_start_multi(odp_ml_model_t model ODP_UNUSED, + const odp_ml_data_t data[] ODP_UNUSED, + const odp_ml_compl_param_t compl_param[] ODP_UNUSED, + const odp_ml_run_param_t run_param[] ODP_UNUSED, + int num ODP_UNUSED) +{ + return -1; +} + +int odp_ml_run_status(odp_ml_model_t model ODP_UNUSED, uint32_t compl_id ODP_UNUSED, + odp_ml_run_result_t *result ODP_UNUSED) +{ + return -1; +} + +int odp_ml_model_extra_stat_info(odp_ml_model_t model ODP_UNUSED, + odp_ml_extra_stat_info_t info[] ODP_UNUSED, + int num ODP_UNUSED) +{ + return -1; +} + +int odp_ml_model_extra_stats(odp_ml_model_t model ODP_UNUSED, + uint64_t stats[] ODP_UNUSED, int num ODP_UNUSED) +{ + return -1; +} diff --git a/platform/linux-generic/odp_ml_quantize.c b/platform/linux-generic/odp_ml_quantize.c new file mode 100644 index 000000000..d3f3601e3 --- /dev/null +++ b/platform/linux-generic/odp_ml_quantize.c @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#include <odp/api/ml_quantize.h> + +#include <odp_debug_internal.h> +#include <odp_macros_internal.h> +#include <odp_ml_fp16.h> + +#include <math.h> +#include <stdint.h> + +void odp_ml_fp32_to_uint8(uint8_t *u8, const float *fp32, uint32_t num, float scale, + uint8_t zerop) +{ + float fval; + + _ODP_ASSERT(scale != 0); + + for (uint32_t i = 0; i < num; i++) { + /* Range mapping: map real values to signed integer */ + fval = nearbyintf(fp32[i] / scale) + (float)zerop; + + /* clip */ + fval = _ODP_MAX(fval, 0.f); + fval = _ODP_MIN(fval, 255.f); + u8[i] = (uint8_t)(int32_t)fval; + } +} + +void odp_ml_fp32_from_uint8(float *fp32, const uint8_t *u8, uint32_t num, float scale, + uint8_t zerop) +{ + for (uint32_t i = 0; i < num; i++) + fp32[i] = (float)(u8[i] - zerop) * scale; +} + +void odp_ml_fp32_to_int8(int8_t *i8, const float *fp32, uint32_t num, float scale, int8_t zerop) +{ + float fval; + + _ODP_ASSERT(scale != 0); + + for (uint32_t i = 0; i < num; i++) { + /* Range mapping: map real values to signed integer */ + fval = nearbyintf(fp32[i] / scale) + (float)zerop; + + /* NOTE: Clamps signed quantization values to [-127,127] instead of [-128,127]. + * This is to ensure that symmetric quantization results in a zero + * point of exactly 0 for signed 8 bit ints. + */ + fval = _ODP_MAX(fval, -127.f); + fval = _ODP_MIN(fval, 127.f); + i8[i] = (int8_t)(int32_t)fval; + } +} + +void odp_ml_fp32_from_int8(float *fp32, const int8_t *i8, uint32_t num, float scale, int8_t zerop) +{ + for (uint32_t i = 0; i < num; i++) + fp32[i] = (float)(i8[i] - zerop) * scale; +} + +void odp_ml_fp32_to_fp16(uint16_t *fp16, const float *fp32, uint32_t num) +{ + uint32_t i; + + for (i = 0; i < num; i++) + fp16[i] = _odp_float32_to_float16(fp32[i]); +} + +void odp_ml_fp32_from_fp16(float *fp32, const uint16_t *fp16, uint32_t num) +{ + uint32_t i; + + for (i = 0; i < num; i++) + fp32[i] = _odp_float16_to_float32(fp16[i]); +} diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 96fcd928a..17a4a9298 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -66,7 +66,6 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .timestamp = offsetof(odp_packet_hdr_t, timestamp), .input_flags = offsetof(odp_packet_hdr_t, p.input_flags), .flags = offsetof(odp_packet_hdr_t, p.flags), - .subtype = offsetof(odp_packet_hdr_t, subtype), .cls_mark = offsetof(odp_packet_hdr_t, cls_mark), .ipsec_ctx = offsetof(odp_packet_hdr_t, ipsec_ctx), .crypto_op = offsetof(odp_packet_hdr_t, crypto_op_result), @@ -1454,7 +1453,7 @@ void odp_packet_print(odp_packet_t pkt) len += _odp_snprint(&str[len], n - len, " pool index %u\n", hdr->event_hdr.index.pool); len += _odp_snprint(&str[len], n - len, " buf index %u\n", hdr->event_hdr.index.event); - len += _odp_snprint(&str[len], n - len, " ev subtype %i\n", hdr->subtype); + len += _odp_snprint(&str[len], n - len, " ev subtype %i\n", hdr->event_hdr.subtype); len += _odp_snprint(&str[len], n - len, " input_flags 0x%" PRIx64 "\n", hdr->p.input_flags.all); if (hdr->p.input_flags.all) { @@ -2401,7 +2400,7 @@ odp_packet_t odp_packet_reassemble(odp_pool_t pool_hdl, odp_packet_buf_t pkt_buf pkt_hdr->tailroom = tailroom; /* Reset metadata */ - pkt_hdr->subtype = ODP_EVENT_PACKET_BASIC; + pkt_hdr->event_hdr.subtype = ODP_EVENT_PACKET_BASIC; pkt_hdr->input = ODP_PKTIO_INVALID; packet_parse_reset(pkt_hdr, 1); diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 236813e80..8283c41e6 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -3015,8 +3015,15 @@ static int lso_update_custom(lso_profile_t *lso_prof, odp_packet_t pkt, int segn ptr = &u32; else if (size == 2) ptr = &u16; - else + else { + /* + * odp_lso_profile_create() ensures that size is one of the allowed values. + * But compiler doesn't know that, so set it here to avoid possibility of + * out of bounds warnings. + */ + size = 1; ptr = &u8; + } if (odp_packet_copy_to_mem(pkt, offset, size, ptr)) { _ODP_ERR("Read from packet failed at offset %u\n", offset); diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 94461e6b1..d3fde70f6 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -495,6 +495,7 @@ static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t e event_hdr->index.event = event_index; event_hdr->type = type; event_hdr->event_type = type; + event_hdr->subtype = ODP_EVENT_NO_SUBTYPE; event_hdr->pool = _odp_pool_handle(pool); /* Store base values for fast init */ @@ -542,7 +543,6 @@ static void init_event_hdr(pool_t *pool, _odp_event_hdr_t *event_hdr, uint32_t e static void init_buffers(pool_t *pool) { - uint64_t i; _odp_event_hdr_t *event_hdr; odp_buffer_hdr_t *buf_hdr; odp_packet_hdr_t *pkt_hdr; @@ -566,7 +566,7 @@ static void init_buffers(pool_t *pool) mask = pool->ring_mask; type = pool->type; - for (i = 0; i < pool->num + skipped_blocks ; i++) { + for (uint64_t i = 0; i < pool->num + skipped_blocks ; i++) { int skip = 0; addr = &pool->base_addr[i * pool->block_size]; @@ -1257,6 +1257,10 @@ int odp_pool_info(odp_pool_t pool_hdl, odp_pool_info_t *info) info->dma_pool_param.uarea_size = pool->params.buf.uarea_size; info->dma_pool_param.cache_size = pool->params.buf.cache_size; + } else if (pool->type_2 == ODP_POOL_ML_COMPL) { + info->ml_pool_param.num = pool->params.buf.num; + info->ml_pool_param.uarea_size = pool->params.buf.uarea_size; + info->ml_pool_param.cache_size = pool->params.buf.cache_size; } else { info->params = pool->params; } @@ -1371,11 +1375,11 @@ static inline void event_free_to_pool(pool_t *pool, if (odp_unlikely((uint32_t)num > cache_num)) burst = cache_num; - _odp_event_hdr_t *event_hdr[burst]; + _odp_event_hdr_t *ev_hdr[burst]; - cache_pop(cache, event_hdr, burst); + cache_pop(cache, ev_hdr, burst); - ring_ptr_enq_multi(ring, mask, (void **)event_hdr, burst); + ring_ptr_enq_multi(ring, mask, (void **)ev_hdr, burst); if (CONFIG_POOL_STATISTICS && pool->params.stats.bit.free_ops) odp_atomic_inc_u64(&pool->stats.free_ops); } @@ -1559,6 +1563,8 @@ static const char *get_long_type_str(odp_pool_type_t type) return "vector"; case ODP_POOL_DMA_COMPL: return "dma completion"; + case ODP_POOL_ML_COMPL: + return "ml completion"; default: return "unknown"; } @@ -1577,6 +1583,8 @@ static const char *get_short_type_str(odp_pool_type_t type) return "V"; case ODP_POOL_DMA_COMPL: return "D"; + case ODP_POOL_ML_COMPL: + return "M"; default: return "-"; } @@ -1875,6 +1883,7 @@ int odp_pool_ext_capability(odp_pool_type_t type, odp_pool_ext_capability_t *cap case ODP_POOL_TIMEOUT: case ODP_POOL_VECTOR: case ODP_POOL_DMA_COMPL: + case ODP_POOL_ML_COMPL: memset(capa, 0, sizeof(odp_pool_ext_capability_t)); return 0; default: diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c index c7040dd3c..bddaa532d 100644 --- a/platform/linux-generic/odp_queue_scalable.c +++ b/platform/linux-generic/odp_queue_scalable.c @@ -18,7 +18,6 @@ #include <odp_config_internal.h> #include <odp_debug_internal.h> - #include <odp_event_internal.h> #include <odp_packet_io_internal.h> #include <odp_pool_internal.h> @@ -472,12 +471,8 @@ static int queue_destroy(odp_queue_t handle) */ while (__atomic_load_n(&q->qschst.numevts, __ATOMIC_RELAXED) != 0 || __atomic_load_n(&q->qschst.cur_ticket, __ATOMIC_RELAXED) != - __atomic_load_n(&q->qschst.nxt_ticket, __ATOMIC_RELAXED)) { - sevl(); - while (wfe() && monitor32((uint32_t *)&q->qschst.numevts, - __ATOMIC_RELAXED) != 0) - odp_cpu_pause(); - } + __atomic_load_n(&q->qschst.nxt_ticket, __ATOMIC_RELAXED)) + _odp_wait_until_eq_u32((uint32_t *)&q->qschst.numevts, 0); if (q->schedq != NULL) { _odp_sched_queue_rem(q->sched_grp, q->sched_prio); @@ -596,13 +591,8 @@ static inline int _odp_queue_enq(sched_elem_t *q, __builtin_prefetch(&q->node, 1, 0); #endif /* Wait for our turn to signal consumers */ - if (odp_unlikely(__atomic_load_n(&q->cons_write, - __ATOMIC_RELAXED) != old_write)) { - sevl(); - while (wfe() && monitor32(&q->cons_write, - __ATOMIC_RELAXED) != old_write) - odp_cpu_pause(); - } + if (odp_unlikely(__atomic_load_n(&q->cons_write, __ATOMIC_RELAXED) != old_write)) + _odp_wait_until_eq_u32(&q->cons_write, old_write); /* Signal consumers that events are available (release events) * Enable other producers to continue @@ -824,13 +814,8 @@ int _odp_queue_deq(sched_elem_t *q, _odp_event_hdr_t *event_hdr[], int num) __builtin_prefetch(&q->node, 1, 0); #endif /* Wait for our turn to signal producers */ - if (odp_unlikely(__atomic_load_n(&q->prod_read, __ATOMIC_RELAXED) != - old_read)) { - sevl(); - while (wfe() && monitor32(&q->prod_read, - __ATOMIC_RELAXED) != old_read) - odp_cpu_pause(); - } + if (odp_unlikely(__atomic_load_n(&q->prod_read, __ATOMIC_RELAXED) != old_read)) + _odp_wait_until_eq_u32(&q->prod_read, old_read); /* Signal producers that empty slots are available * (release ring slots) diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 7bd8cbfed..379f1f828 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -44,6 +44,7 @@ #include <string.h> #include <time.h> +#include <inttypes.h> /* No synchronization context */ #define NO_SYNC_CONTEXT ODP_SCHED_SYNC_PARALLEL @@ -297,7 +298,7 @@ typedef struct { struct { uint32_t poll_time; - struct timespec sleep_time; + uint64_t sleep_time; } powersave; /* Scheduler interface config options (not used in fast path) */ @@ -545,8 +546,8 @@ static int read_config_file(sched_global_t *sched) } val = _ODP_MAX(0, val); - sched->powersave.sleep_time.tv_sec = val / 1000000000; - sched->powersave.sleep_time.tv_nsec = val % 1000000000; + val = _ODP_MIN((int)ODP_TIME_SEC_IN_NS - 1, val); + sched->powersave.sleep_time = val; _ODP_PRINT(" %s: %i\n", str, val); _ODP_PRINT(" dynamic load balance: %s\n", sched->load_balance ? "ON" : "OFF"); @@ -1672,7 +1673,7 @@ static inline int schedule_loop_sleep(odp_queue_t *out_queue, uint64_t wait, timer_run(2); break; } - timer_run(1); + uint64_t next = timer_run(sleep ? TIMER_SCAN_FORCE : 1); if (first) { start = odp_time_local(); @@ -1683,19 +1684,27 @@ static inline int schedule_loop_sleep(odp_queue_t *out_queue, uint64_t wait, continue; } - if (sleep) - nanosleep(&sched->powersave.sleep_time, NULL); + if (sleep && next) { + uint64_t sleep_nsec = _ODP_MIN(sched->powersave.sleep_time, next); - if (wait != ODP_SCHED_WAIT || !sleep) { - current = odp_time_local(); - if (odp_time_cmp(start_sleep, current) < 0) - sleep = 1; + if (wait != ODP_SCHED_WAIT) { + uint64_t nsec_to_end = odp_time_diff_ns(end, current); + + sleep_nsec = _ODP_MIN(sleep_nsec, nsec_to_end); + } + + struct timespec ts = { 0, sleep_nsec }; + + nanosleep(&ts, NULL); } - if (wait == ODP_SCHED_WAIT) - continue; + if (!sleep || wait != ODP_SCHED_WAIT) + current = odp_time_local(); + + if (!sleep && odp_time_cmp(start_sleep, current) < 0) + sleep = 1; - if (odp_time_cmp(end, current) < 0) + if (wait != ODP_SCHED_WAIT && odp_time_cmp(end, current) < 0) break; } diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c index 6d60c048f..5166fb6d0 100644 --- a/platform/linux-generic/odp_schedule_scalable.c +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -223,13 +223,9 @@ void _odp_sched_update_enq(sched_elem_t *q, uint32_t actual) if (odp_unlikely(ticket != TICKET_INVALID)) { /* Wait for our turn to update schedq. */ if (odp_unlikely(__atomic_load_n(&q->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ticket)) { - sevl(); - while (wfe() && - monitor8(&q->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ticket) - odp_cpu_pause(); - } + __ATOMIC_ACQUIRE) != ticket)) + _odp_wait_until_eq_acq_u8(&q->qschst.cur_ticket, ticket); + /* Enqueue at end of scheduler queue */ /* We are here because of empty-to-non-empty transition * This means queue must be pushed to schedq if possible @@ -366,13 +362,9 @@ sched_update_deq(sched_elem_t *q, _ODP_ASSERT(q->qschst_type != ODP_SCHED_SYNC_ATOMIC); /* Wait for our turn to update schedq. */ if (odp_unlikely(__atomic_load_n(&q->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ticket)) { - sevl(); - while (wfe() && - monitor8(&q->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ticket) - odp_cpu_pause(); - } + __ATOMIC_ACQUIRE) != ticket)) + _odp_wait_until_eq_acq_u8(&q->qschst.cur_ticket, ticket); + /* We are here because of non-empty-to-empty transition or * WRR budget exhausted * This means the queue must be popped from the schedq, now or @@ -494,12 +486,9 @@ static inline void sched_update_popd(sched_elem_t *elem) 1, __ATOMIC_RELAXED); if (odp_unlikely(__atomic_load_n(&elem->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ticket)) { - sevl(); - while (wfe() && monitor8(&elem->qschst.cur_ticket, - __ATOMIC_ACQUIRE) != ticket) - odp_cpu_pause(); - } + __ATOMIC_ACQUIRE) != ticket)) + _odp_wait_until_eq_acq_u8(&elem->qschst.cur_ticket, ticket); + sched_update_popd_sc(elem); atomic_store_release(&elem->qschst.cur_ticket, ticket + 1, /*readonly=*/false); @@ -1054,15 +1043,8 @@ restart_same: continue; } /* Wait for our turn to dequeue */ - if (odp_unlikely(__atomic_load_n(&rwin->turn, - __ATOMIC_ACQUIRE) - != sn)) { - sevl(); - while (wfe() && - monitor32(&rwin->turn, __ATOMIC_ACQUIRE) - != sn) - odp_cpu_pause(); - } + if (odp_unlikely(__atomic_load_n(&rwin->turn, __ATOMIC_ACQUIRE) != sn)) + _odp_wait_until_eq_acq_u32(&rwin->turn, sn); #ifdef CONFIG_QSCHST_LOCK LOCK(&elem->qschlock); #endif @@ -1143,13 +1125,8 @@ static void schedule_order_lock(uint32_t lock_index) return; } if (odp_unlikely(__atomic_load_n(&rctx->rwin->olock[lock_index], - __ATOMIC_ACQUIRE) != rctx->sn)) { - sevl(); - while (wfe() && - monitor32(&rctx->rwin->olock[lock_index], - __ATOMIC_ACQUIRE) != rctx->sn) - odp_cpu_pause(); - } + __ATOMIC_ACQUIRE) != rctx->sn)) + _odp_wait_until_eq_acq_u32(&rctx->rwin->olock[lock_index], rctx->sn); } static void schedule_order_unlock(uint32_t lock_index) @@ -1555,12 +1532,7 @@ static int schedule_group_destroy(odp_schedule_group_t group) if (sg->xcount[p] != 0) { bitset_t wanted = atom_bitset_load(&sg->thr_wanted, __ATOMIC_RELAXED); - sevl(); - while (wfe() && - !bitset_is_eql(wanted, - bitset_monitor(&sg->thr_actual[p], - __ATOMIC_RELAXED))) - odp_cpu_pause(); + _odp_wait_until_eq_bitset(&sg->thr_actual[p], wanted); } /* Else ignore because no ODP queues on this prio */ } @@ -2127,13 +2099,10 @@ static void order_lock(void) _ODP_ASSERT(ts->rctx != NULL); rwin = ts->rctx->rwin; sn = ts->rctx->sn; - sevl(); /* Use acquire ordering to be on the safe side even if * this isn't an acquire/release situation (aka lock). */ - while (wfe() && - monitor32(&rwin->hc.head, __ATOMIC_ACQUIRE) != sn) - odp_cpu_pause(); + _odp_wait_until_eq_acq_u32(&rwin->hc.head, sn); } } diff --git a/platform/linux-generic/odp_schedule_scalable_ordered.c b/platform/linux-generic/odp_schedule_scalable_ordered.c index f6655d7fa..f8568ce53 100644 --- a/platform/linux-generic/odp_schedule_scalable_ordered.c +++ b/platform/linux-generic/odp_schedule_scalable_ordered.c @@ -123,8 +123,6 @@ static void rwin_insert(reorder_window_t *rwin, /*readonly=*/false); rctx = NULL; do { - hc_t new; - new.head = old.head; new.chgi = old.chgi + 1; /* Changed value */ /* Update head & chgi, fail if any has changed */ diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index 52f1000f1..a2593b531 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -26,7 +26,6 @@ #include <odp/api/cpu.h> #include <errno.h> -#include <pthread.h> #include <string.h> #include <stdio.h> #include <inttypes.h> @@ -386,8 +385,9 @@ int _odp_system_info_init(void) num_cpus); /* Read and save all CPU frequencies for static mode */ - for (i = 0; i < CONFIG_NUM_CPU_IDS; i++) - odp_global_ro.system_info.cpu_hz[i] = cpu_hz_current(i); + if (odp_global_ro.system_info.cpu_hz_static) + for (i = 0; i < CONFIG_NUM_CPU_IDS; i++) + odp_global_ro.system_info.cpu_hz[i] = cpu_hz_current(i); /* By default, read max frequency from a cpufreq file */ for (i = 0; i < CONFIG_NUM_CPU_IDS; i++) { @@ -627,5 +627,8 @@ void odp_sys_config_print(void) _ODP_PRINT("CONFIG_IPSEC_MAX_NUM_SA: %i\n", CONFIG_IPSEC_MAX_NUM_SA); _ODP_PRINT("CONFIG_TIMER_128BIT_ATOMICS: %i\n", CONFIG_TIMER_128BIT_ATOMICS); _ODP_PRINT("CONFIG_TIMER_PROFILE_INLINE: %i\n", CONFIG_TIMER_PROFILE_INLINE); + _ODP_PRINT("CONFIG_ML_MAX_MODELS: %i\n", CONFIG_ML_MAX_MODELS); + _ODP_PRINT("CONFIG_ML_MAX_INPUTS: %i\n", CONFIG_ML_MAX_INPUTS); + _ODP_PRINT("CONFIG_ML_MAX_OUTPUTS: %i\n", CONFIG_ML_MAX_OUTPUTS); _ODP_PRINT("\n"); } diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index daf187390..c8ea31078 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -761,11 +761,12 @@ static inline void timer_expire(timer_pool_t *tp, uint32_t idx, uint64_t tick) } } -static inline void timer_pool_scan(timer_pool_t *tp, uint64_t tick) +static inline uint64_t timer_pool_scan(timer_pool_t *tp, uint64_t tick) { tick_buf_t *array = &tp->tick_buf[0]; uint32_t high_wm = odp_atomic_load_acq_u32(&tp->high_wm); uint32_t i; + uint64_t min = UINT64_MAX; _ODP_ASSERT(high_wm <= tp->param.num_timers); for (i = 0; i < high_wm; i++) { @@ -780,18 +781,23 @@ static inline void timer_pool_scan(timer_pool_t *tp, uint64_t tick) if (odp_unlikely(exp_tck <= tick)) { /* Attempt to expire timer */ timer_expire(tp, i, tick); + min = 0; + } else { + min = _ODP_MIN(min, exp_tck - tick); } } + + return min; } /****************************************************************************** * Inline timer processing *****************************************************************************/ -static inline void timer_pool_scan_inline(int num, odp_time_t now) +static inline uint64_t timer_pool_scan_inline(int num, odp_time_t now, int force) { timer_pool_t *tp; - uint64_t new_tick, old_tick, nsec; + uint64_t new_tick, old_tick, ticks_to_next_expire, nsec, min = UINT64_MAX; int64_t diff; int i; @@ -817,7 +823,7 @@ static inline void timer_pool_scan_inline(int num, odp_time_t now) old_tick = odp_atomic_load_u64(&tp->cur_tick); diff = new_tick - old_tick; - if (diff < 1) + if (diff < 1 && !force) continue; if (odp_atomic_cas_u64(&tp->cur_tick, &old_tick, new_tick)) { @@ -832,26 +838,30 @@ static inline void timer_pool_scan_inline(int num, odp_time_t now) odp_atomic_store_u32(&tp->notify_overrun, 2); } } - timer_pool_scan(tp, nsec); + ticks_to_next_expire = timer_pool_scan(tp, nsec); + min = _ODP_MIN(min, ticks_to_next_expire); } } + + return min; } -void _odp_timer_run_inline(int dec) +uint64_t _odp_timer_run_inline(int dec) { odp_time_t now; int num = timer_global->highest_tp_idx + 1; - int poll_interval = timer_global->poll_interval; + int force = (dec == TIMER_SCAN_FORCE); + int poll_interval = force ? 0 : timer_global->poll_interval; if (num == 0) - return; + return UINT64_MAX; /* Rate limit how often this thread checks the timer pools. */ if (poll_interval > 1) { timer_local.run_cnt -= dec; if (timer_local.run_cnt > 0) - return; + return UINT64_MAX; timer_local.run_cnt = poll_interval; } @@ -862,7 +872,12 @@ void _odp_timer_run_inline(int dec) if (odp_time_cmp(period, timer_global->poll_interval_time) < 0) - return; + return UINT64_MAX; + timer_local.last_run = now; + } + + if (force) { + timer_local.run_cnt = poll_interval; timer_local.last_run = now; } @@ -870,13 +885,14 @@ void _odp_timer_run_inline(int dec) if (CONFIG_TIMER_PROFILE_INLINE) { odp_time_t t1 = odp_time_local_strict(); - timer_pool_scan_inline(num, now); + uint64_t ret = timer_pool_scan_inline(num, now, force); odp_time_t t2 = odp_time_local_strict(); timer_local.prof_nsec += odp_time_diff_ns(t2, t1); timer_local.prof_rounds++; + return ret; } else { - timer_pool_scan_inline(num, now); + return timer_pool_scan_inline(num, now, force); } } diff --git a/platform/linux-generic/test/Makefile.am b/platform/linux-generic/test/Makefile.am index 30ef26078..7aca5fd3f 100644 --- a/platform/linux-generic/test/Makefile.am +++ b/platform/linux-generic/test/Makefile.am @@ -21,6 +21,11 @@ SUBDIRS += validation/api/pktio \ example \ performance +if WITH_ML +TESTS += validation/api/ml/ml_linux$(EXEEXT) +SUBDIRS += validation/api/ml +endif + if ODP_PKTIO_PCAP TESTS += validation/api/pktio/pktio_run_pcap.sh endif diff --git a/platform/linux-generic/test/example/ipsec_api/Makefile.am b/platform/linux-generic/test/example/ipsec_api/Makefile.am index 101c97cdf..2535ad466 100644 --- a/platform/linux-generic/test/example/ipsec_api/Makefile.am +++ b/platform/linux-generic/test/example/ipsec_api/Makefile.am @@ -19,5 +19,3 @@ clean-local: rm -f $(builddir)/$$f; \ done \ fi - -.NOTPARALLEL: diff --git a/platform/linux-generic/test/example/ipsec_crypto/Makefile.am b/platform/linux-generic/test/example/ipsec_crypto/Makefile.am index 101c97cdf..2535ad466 100644 --- a/platform/linux-generic/test/example/ipsec_crypto/Makefile.am +++ b/platform/linux-generic/test/example/ipsec_crypto/Makefile.am @@ -19,5 +19,3 @@ clean-local: rm -f $(builddir)/$$f; \ done \ fi - -.NOTPARALLEL: diff --git a/platform/linux-generic/test/inline-timer.conf b/platform/linux-generic/test/inline-timer.conf index d645bef3c..fa3b6982f 100644 --- a/platform/linux-generic/test/inline-timer.conf +++ b/platform/linux-generic/test/inline-timer.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.27" +config_file_version = "0.1.28" timer: { # Enable inline timer implementation diff --git a/platform/linux-generic/test/packet_align.conf b/platform/linux-generic/test/packet_align.conf index 427674bb2..fb1418348 100644 --- a/platform/linux-generic/test/packet_align.conf +++ b/platform/linux-generic/test/packet_align.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.27" +config_file_version = "0.1.28" pool: { pkt: { diff --git a/platform/linux-generic/test/pktio_ipc/ipc_common.c b/platform/linux-generic/test/pktio_ipc/ipc_common.c index f693feeb2..128a7c6e1 100644 --- a/platform/linux-generic/test/pktio_ipc/ipc_common.c +++ b/platform/linux-generic/test/pktio_ipc/ipc_common.c @@ -1,11 +1,12 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2018 Linaro Limited + * Copyright (c) 2023 Nokia */ #include "ipc_common.h" +/** Start time in seconds */ +int start_time_sec; /** Run time in seconds */ int run_time_sec; /** Pid of the master process */ @@ -97,23 +98,28 @@ void parse_args(int argc, char *argv[]) int opt; int long_index; static struct option longopts[] = { - {"time", required_argument, NULL, 't'}, + {"start-timeout", required_argument, NULL, 's'}, + {"run-time", required_argument, NULL, 't'}, {"pid", required_argument, NULL, 'p'}, /* master process pid */ {"help", no_argument, NULL, 'h'}, /* return 'h' */ {NULL, 0, NULL, 0} }; + start_time_sec = 0; /* wait forever if time is 0 */ run_time_sec = 0; /* loop forever if time to run is 0 */ master_pid = 0; while (1) { - opt = getopt_long(argc, argv, "+t:p:h", + opt = getopt_long(argc, argv, "+s:t:p:h", longopts, &long_index); if (opt == -1) break; /* No more options */ switch (opt) { + case 's': + start_time_sec = atoi(optarg); + break; case 't': run_time_sec = atoi(optarg); break; @@ -151,15 +157,14 @@ void usage(char *progname) { printf("\n" "Usage: %s OPTIONS\n" - " E.g. -n ipc_name_space %s -t seconds\n" "\n" "OpenDataPlane odp-linux ipc test application.\n" "\n" - "Mandatory OPTIONS:\n" - " -n, --ns IPC name space ID /dev/shm/odp-<ns>-objname.\n" "Optional OPTIONS\n" " -h, --help Display help and exit.\n" - " -t, --time Time to run in seconds.\n" - "\n", NO_PATH(progname), NO_PATH(progname) + " -p, --pid PID of the master process.\n" + " -t, --run-time Time to run in seconds.\n" + " -s, --start-timeout Maximum time for pktio startup.\n" + "\n", NO_PATH(progname) ); } diff --git a/platform/linux-generic/test/pktio_ipc/ipc_common.h b/platform/linux-generic/test/pktio_ipc/ipc_common.h index b2b469553..94ec21460 100644 --- a/platform/linux-generic/test/pktio_ipc/ipc_common.h +++ b/platform/linux-generic/test/pktio_ipc/ipc_common.h @@ -1,7 +1,6 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2018 Linaro Limited + * Copyright (c) 2023 Nokia */ #define _POSIX_C_SOURCE 200809L @@ -64,6 +63,9 @@ typedef struct ODP_PACKED { odp_u32be_t magic; } pkt_tail_t; +/** Start time in seconds */ +extern int start_time_sec; + /** Run time in seconds */ extern int run_time_sec; diff --git a/platform/linux-generic/test/pktio_ipc/pktio_ipc1.c b/platform/linux-generic/test/pktio_ipc/pktio_ipc1.c index 6c71e18da..df7a5ca3f 100644 --- a/platform/linux-generic/test/pktio_ipc/pktio_ipc1.c +++ b/platform/linux-generic/test/pktio_ipc/pktio_ipc1.c @@ -1,7 +1,6 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2018 Linaro Limited + * Copyright (c) 2023 Nokia */ #include "ipc_common.h" @@ -49,17 +48,17 @@ static int pktio_run_loop(odp_pool_t pool) else sprintf(name, TEST_IPC_PKTIO_NAME); - wait = odp_time_local_from_ns(run_time_sec * ODP_TIME_SEC_IN_NS); + wait = odp_time_local_from_ns(start_time_sec * ODP_TIME_SEC_IN_NS); start_cycle = odp_time_local(); current_cycle = start_cycle; for (;;) { - if (run_time_sec) { + if (start_time_sec) { cycle = odp_time_local(); diff = odp_time_diff(cycle, start_cycle); if (odp_time_cmp(wait, diff) < 0) { - printf("timeout exit, run_time_sec %d\n", - run_time_sec); + printf("timeout exit 1, start_time_sec %d\n", + start_time_sec); return -1; } } @@ -83,12 +82,12 @@ static int pktio_run_loop(odp_pool_t pool) /* start ipc pktio, i.e. wait until other process connects */ for (;;) { - if (run_time_sec) { + if (start_time_sec) { cycle = odp_time_local(); diff = odp_time_diff(cycle, start_cycle); if (odp_time_cmp(wait, diff) < 0) { - printf("timeout exit, run_time_sec %d\n", - run_time_sec); + printf("timeout exit 2, start_time_sec %d\n", + start_time_sec); goto exit; } } @@ -102,6 +101,8 @@ static int pktio_run_loop(odp_pool_t pool) } /* packets loop */ + wait = odp_time_local_from_ns(run_time_sec * ODP_TIME_SEC_IN_NS); + start_cycle = odp_time_local(); for (;;) { int i; diff --git a/platform/linux-generic/test/pktio_ipc/pktio_ipc2.c b/platform/linux-generic/test/pktio_ipc/pktio_ipc2.c index e6ca5f5e5..fc3b6833a 100644 --- a/platform/linux-generic/test/pktio_ipc/pktio_ipc2.c +++ b/platform/linux-generic/test/pktio_ipc/pktio_ipc2.c @@ -1,7 +1,6 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2018 Linaro Limited + * Copyright (c) 2023 Nokia */ /** @@ -46,17 +45,17 @@ static int ipc_second_process(int master_pid) exit(EXIT_FAILURE); } - wait = odp_time_local_from_ns(run_time_sec * ODP_TIME_SEC_IN_NS); + wait = odp_time_local_from_ns(start_time_sec * ODP_TIME_SEC_IN_NS); start_cycle = odp_time_local(); for (;;) { /* exit loop if time specified */ - if (run_time_sec) { + if (start_time_sec) { cycle = odp_time_local(); diff = odp_time_diff(cycle, start_cycle); if (odp_time_cmp(wait, diff) < 0) { - printf("timeout exit, run_time_sec %d\n", - run_time_sec); + printf("timeout exit 1, start_time_sec %d\n", + start_time_sec); goto not_started; } } @@ -85,12 +84,12 @@ static int ipc_second_process(int master_pid) /* start ipc pktio, i.e. wait until other process connects */ for (;;) { /* 1. exit loop if time specified */ - if (run_time_sec) { + if (start_time_sec) { cycle = odp_time_local(); diff = odp_time_diff(cycle, start_cycle); if (odp_time_cmp(wait, diff) < 0) { - printf("timeout exit, run_time_sec %d\n", - run_time_sec); + printf("timeout exit 2, start_time_sec %d\n", + start_time_sec); goto not_started; } } @@ -103,6 +102,8 @@ static int ipc_second_process(int master_pid) odp_time_wait_ns(50 * ODP_TIME_MSEC_IN_NS); } + wait = odp_time_local_from_ns(run_time_sec * ODP_TIME_SEC_IN_NS); + start_cycle = odp_time_local(); for (;;) { /* exit loop if time specified */ if (run_time_sec) { diff --git a/platform/linux-generic/test/pktio_ipc/pktio_ipc_run.sh b/platform/linux-generic/test/pktio_ipc/pktio_ipc_run.sh index bad2626bd..b181668e8 100755 --- a/platform/linux-generic/test/pktio_ipc/pktio_ipc_run.sh +++ b/platform/linux-generic/test/pktio_ipc/pktio_ipc_run.sh @@ -1,9 +1,8 @@ #!/bin/sh # -# Copyright (c) 2015-2018, Linaro Limited -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2015-2018 Linaro Limited +# Copyright (c) 2023 Nokia # # directories where test binary can be found: @@ -17,31 +16,23 @@ PATH=$(dirname $0):$PATH PATH=$(dirname $0)/../../../../platform/linux-generic/test/pktio_ipc:$PATH PATH=.:$PATH -RUNTIME1=3 -RUNTIME2=1 -TIMEOUT=3 -if [ "${TEST}" = "coverage" ]; then - RUNTIME1=30 - RUNTIME2=15 - TIMEOUT=20 -fi +STARTTIME=30 +RUNTIME=1 run() { local ret=0 echo "==== run pktio_ipc1 then pktio_ipc2 ====" - pktio_ipc1${EXEEXT} -t ${RUNTIME1} & + pktio_ipc1${EXEEXT} -s ${STARTTIME} -t ${RUNTIME} & IPC_PID=$! - pktio_ipc2${EXEEXT} -p ${IPC_PID} -t ${RUNTIME2} + pktio_ipc2${EXEEXT} -p ${IPC_PID} -s ${STARTTIME} -t ${RUNTIME} ret=$? - # pktio_ipc1 should do clean up and exit just - # after pktio_ipc2 exited. If it does not happen - # kill him in test. - sleep ${TIMEOUT} + (kill ${IPC_PID} 2>&1 > /dev/null ) > /dev/null if [ $? -eq 0 ]; then + wait $IPC_PID echo "pktio_ipc1${EXEEXT} was killed" ls -l /dev/shm/${UID}/odp* 2> /dev/null rm -rf /dev/shm/${UID}/odp-${IPC_PID}* 2>&1 > /dev/null @@ -58,16 +49,15 @@ run() fi echo "==== run pktio_ipc2 then pktio_ipc1 ====" - pktio_ipc2${EXEEXT} -t ${RUNTIME1} & + pktio_ipc2${EXEEXT} -s ${STARTTIME} -t ${RUNTIME} & IPC_PID=$! - pktio_ipc1${EXEEXT} -p ${IPC_PID} -t ${RUNTIME2} + pktio_ipc1${EXEEXT} -p ${IPC_PID} -s ${STARTTIME} -t ${RUNTIME} ret=$? - # pktio_ipc2 do not exit on pktio_ipc1 disconnect - # wait until it exits cleanly - sleep ${TIMEOUT} + (kill ${IPC_PID} 2>&1 > /dev/null ) > /dev/null if [ $? -eq 0 ]; then + wait $IPC_PID echo "pktio_ipc2${EXEEXT} was killed" ls -l /dev/shm/${UID}/odp* 2> /dev/null rm -rf /dev/shm/${UID}/odp-${IPC_PID}* 2>&1 > /dev/null diff --git a/platform/linux-generic/test/process-mode.conf b/platform/linux-generic/test/process-mode.conf index 5bfcb9f2f..f4c6f7952 100644 --- a/platform/linux-generic/test/process-mode.conf +++ b/platform/linux-generic/test/process-mode.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.27" +config_file_version = "0.1.28" # Shared memory options shm: { diff --git a/platform/linux-generic/test/sched-basic.conf b/platform/linux-generic/test/sched-basic.conf index 1a401298e..8a6d0ac98 100644 --- a/platform/linux-generic/test/sched-basic.conf +++ b/platform/linux-generic/test/sched-basic.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.27" +config_file_version = "0.1.28" # Test scheduler with an odd spread value and without dynamic load balance sched_basic: { diff --git a/platform/linux-generic/test/stash-custom.conf b/platform/linux-generic/test/stash-custom.conf index b96c1cf45..6a2496303 100644 --- a/platform/linux-generic/test/stash-custom.conf +++ b/platform/linux-generic/test/stash-custom.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.27" +config_file_version = "0.1.28" # Test overflow safe stash variant stash: { diff --git a/platform/linux-generic/test/validation/api/ml/.gitignore b/platform/linux-generic/test/validation/api/ml/.gitignore new file mode 100644 index 000000000..e31f902c4 --- /dev/null +++ b/platform/linux-generic/test/validation/api/ml/.gitignore @@ -0,0 +1 @@ +ml_linux diff --git a/platform/linux-generic/test/validation/api/ml/Makefile.am b/platform/linux-generic/test/validation/api/ml/Makefile.am new file mode 100644 index 000000000..f4b9e9755 --- /dev/null +++ b/platform/linux-generic/test/validation/api/ml/Makefile.am @@ -0,0 +1,34 @@ +include ../Makefile.inc + +test_PROGRAMS = ml_linux +ml_linux_SOURCES = ml_linux.c + +EXTRA_DIST = \ + batch_add_gen.py \ + batch_add.onnx \ + gen_models.sh \ + README.md \ + requirements.txt \ + simple_linear_gen.py \ + simple_linear.onnx + +# If building out-of-tree, make check will not copy the scripts and data to the +# $(builddir) assuming that all commands are run locally. However this prevents +# running tests on a remote target using LOG_COMPILER. +# So copy all script and data files explicitly here. +all-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + if [ -e $(srcdir)/$$f ]; then \ + mkdir -p $(builddir)/$$(dirname $$f); \ + cp -f $(srcdir)/$$f $(builddir)/$$f; \ + fi \ + done \ + fi + +clean-local: + if [ "x$(srcdir)" != "x$(builddir)" ]; then \ + for f in $(EXTRA_DIST); do \ + rm -f $(builddir)/$$f; \ + done \ + fi diff --git a/platform/linux-generic/test/validation/api/ml/README.md b/platform/linux-generic/test/validation/api/ml/README.md new file mode 100644 index 000000000..80ad30e96 --- /dev/null +++ b/platform/linux-generic/test/validation/api/ml/README.md @@ -0,0 +1,23 @@ +# How to run ML validation test + +Simple onnx models are used to test ML API. + +## Generate models + +### Install python requirements + +```bash +python3 -m pip install -r <this directory>/requirements.txt +``` + +### Generate models for validation tests + +```bash +<this directory>/gen_models.sh +``` + +## Run ML validation tests + +```bash +<this directory>/ml_linux +``` diff --git a/platform/linux-generic/test/validation/api/ml/batch_add.onnx b/platform/linux-generic/test/validation/api/ml/batch_add.onnx Binary files differnew file mode 100644 index 000000000..43485f463 --- /dev/null +++ b/platform/linux-generic/test/validation/api/ml/batch_add.onnx diff --git a/platform/linux-generic/test/validation/api/ml/batch_add_gen.py b/platform/linux-generic/test/validation/api/ml/batch_add_gen.py new file mode 100644 index 000000000..33515bd2f --- /dev/null +++ b/platform/linux-generic/test/validation/api/ml/batch_add_gen.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Nokia +# + +import onnx +from onnx import helper +from onnx import TensorProto + +graph = helper.make_graph( + [ # nodes + helper.make_node("Add", ["x1", "x2"], ["y"], "Batch Add"), + ], + "Batch Add", # name + [ # inputs + helper.make_tensor_value_info('x1', TensorProto.DOUBLE, ["c", 3]), + helper.make_tensor_value_info('x2', TensorProto.DOUBLE, ["c", 3]), + ], + [ # outputs + helper.make_tensor_value_info('y', TensorProto.DOUBLE, ["c", 3]), + ] +) + +model = helper.make_model( + graph, + opset_imports=[helper.make_opsetid("", 14)], + producer_name='ODP validation tests', + model_version=1, + doc_string="y = x1 + x2", + ir_version = 8 +) + +onnx.save(model, 'batch_add.onnx') diff --git a/platform/linux-generic/test/validation/api/ml/gen_models.sh b/platform/linux-generic/test/validation/api/ml/gen_models.sh new file mode 100755 index 000000000..d88f3c432 --- /dev/null +++ b/platform/linux-generic/test/validation/api/ml/gen_models.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Nokia +# + +set -e + +# cd to the directory where this script is in +cd "$( dirname "${BASH_SOURCE[0]}" )" + +python3 simple_linear_gen.py + +python3 batch_add_gen.py diff --git a/platform/linux-generic/test/validation/api/ml/ml_linux.c b/platform/linux-generic/test/validation/api/ml/ml_linux.c new file mode 100644 index 000000000..28e18fbb5 --- /dev/null +++ b/platform/linux-generic/test/validation/api/ml/ml_linux.c @@ -0,0 +1,1167 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include <unistd.h> +#include <string.h> +#include <libgen.h> +#include <odp_api.h> +#include <odp/helper/odph_api.h> +#include "odp_cunit_common.h" + +#define TIMEOUT 5 +#define MODEL_NAME "Test" +#define NUM_INPUTS 1 +#define NUM_OUTPUTS 1 +#define RUN_NUM 2 +#define BUF_LEN 256 +#define CONFIG_MAX_MODEL_SIZE 500 + +#define COMPL_POOL_NAME "ML compl pool" +#define NUM_COMPL 10 + +/** + * About model simple_linear.onnx being tested in this suite + * + * Model info: + * Version: 1 + * Inputs: name: x, type: int32, shape: [1] + * Outputs: name: y, type: int32, shape: [1] + * + * The model is of form y = 3 * x + 4 + * Thus when x = 5, the output y should be 19. + */ +typedef struct global_t { + int disabled; + odp_ml_capability_t ml_capa; + odp_ml_config_t ml_config; + odp_ml_model_param_t model_param; + odp_ml_model_t ml_model; + odp_pool_t compl_pool; + odp_queue_t queue; + odp_ml_data_t data; + odp_ml_data_seg_t input_seg; + odp_ml_data_seg_t output_seg; + odp_ml_run_param_t run_param; + uint64_t wait_ns; + int32_t x; + int32_t y; + int32_t y_expected; + +} global_t; + +static global_t global; + +static int fill_model_param(const char *model_name, odp_ml_model_param_t *model_param) +{ + size_t size; + char *pos; + char *exe_dir; + size_t exe_dir_len; + FILE *model_file; + char exe_path[BUF_LEN]; + ssize_t exe_path_len; + char model_path[BUF_LEN]; + + /* Model file is placed in the same directory as the executable ml_linux */ + exe_path_len = readlink("/proc/self/exe", exe_path, BUF_LEN - 1); + if (exe_path_len != -1) { + exe_path[exe_path_len] = '\0'; + + pos = strstr(exe_path, ".libs"); + if (pos) + *(pos + 5) = '\0'; + + exe_dir = dirname(exe_path); + exe_dir_len = strlen(exe_dir); + + memcpy(model_path, exe_dir, exe_dir_len); + model_path[exe_dir_len] = '/'; + model_path[exe_dir_len + 1] = '\0'; + + strncat(model_path, model_name, BUF_LEN - strlen(model_path) - 1); + ODPH_DBG("model_path: %s\n", model_path); + model_file = fopen(model_path, "rb"); + } else { /* Can't get executable path, try to find model file at current dir*/ + model_file = fopen(model_name, "rb"); + } + + if (model_file == NULL) { + perror("Failed to open model file"); + return -1; + } + + /* Get the model file size in bytes */ + fseek(model_file, 0, SEEK_END); + model_param->size = ftell(model_file); + rewind(model_file); + + model_param->model = malloc(model_param->size); + if (!model_param->model) { + ODPH_ERR("\n\nMemory allocation failed\n"); + fclose(model_file); + return -1; + } + size = fread(model_param->model, model_param->size, 1, model_file); + + fclose(model_file); + if (size != 1) { + ODPH_ERR("\n\nRead model file failed\n"); + return -1; + } + + model_param->max_compl_id = 0; + + return 0; +} + +static int ml_suite_init(void) +{ + odp_ml_capability_t *ml_capa = &global.ml_capa; + odp_queue_param_t queue_param; + odp_ml_compl_pool_param_t ml_pool_param; + + memset(&global, 0, sizeof(global_t)); + global.queue = ODP_QUEUE_INVALID; + global.compl_pool = ODP_POOL_INVALID; + + if (odp_ml_capability(ml_capa)) { + ODPH_ERR("ML capability failed\n"); + return -1; + } + + if (ml_capa->max_models == 0) { + global.disabled = 1; + ODPH_DBG("ML test disabled\n"); + return 0; + } + + /* Configure ML */ + odp_ml_config_init(&global.ml_config); + global.ml_config.max_models_created = ml_capa->max_models; + global.ml_config.max_models_loaded = ml_capa->max_models_loaded; + global.ml_config.max_model_size = CONFIG_MAX_MODEL_SIZE; + + if (ml_capa->load.compl_mode_mask & ODP_ML_COMPL_MODE_SYNC) + global.ml_config.load_mode_mask |= ODP_ML_COMPL_MODE_SYNC; + + if (ml_capa->load.compl_mode_mask & ODP_ML_COMPL_MODE_POLL) + global.ml_config.load_mode_mask |= ODP_ML_COMPL_MODE_POLL; + + if (ml_capa->load.compl_mode_mask & ODP_ML_COMPL_MODE_EVENT) + global.ml_config.load_mode_mask |= ODP_ML_COMPL_MODE_EVENT; + + if (ml_capa->run.compl_mode_mask & ODP_ML_COMPL_MODE_SYNC) + global.ml_config.run_mode_mask |= ODP_ML_COMPL_MODE_SYNC; + + if (ml_capa->run.compl_mode_mask & ODP_ML_COMPL_MODE_POLL) + global.ml_config.run_mode_mask |= ODP_ML_COMPL_MODE_POLL; + + if (ml_capa->run.compl_mode_mask & ODP_ML_COMPL_MODE_EVENT) + global.ml_config.run_mode_mask |= ODP_ML_COMPL_MODE_EVENT; + + if (odp_ml_config(&global.ml_config)) { + ODPH_ERR("\n\nConfiguring ML failed\n"); + return -1; + } + + global.x = 5; + global.wait_ns = 500 * ODP_TIME_MSEC_IN_NS; + global.y_expected = 19; /* y = 3 * x + 4 = 3 * 5 + 4 = 19 */ + + /* Prepare data for running model inference */ + odp_ml_run_param_init(&global.run_param); + + global.data.num_input_seg = NUM_INPUTS; + global.data.input_seg = &global.input_seg; + global.input_seg.size = sizeof(int32_t); + global.input_seg.addr = &global.x; + + global.data.num_output_seg = NUM_OUTPUTS; + global.data.output_seg = &global.output_seg; + global.output_seg.size = sizeof(int32_t); + global.output_seg.addr = &global.y; + + if (fill_model_param("simple_linear.onnx", &global.model_param)) + return -1; + + /* Create ML model */ + global.ml_model = odp_ml_model_create(MODEL_NAME, &global.model_param); + if (global.ml_model == ODP_ML_MODEL_INVALID) { + ODPH_ERR("Create ML model failed\n"); + goto error; + } + + /* Asynchronous mode with event completion is not supported */ + if (!((ml_capa->load.compl_mode_mask & ODP_ML_COMPL_MODE_EVENT) || + (ml_capa->run.compl_mode_mask & ODP_ML_COMPL_MODE_EVENT))) + return 0; + + /* Create a queue for sending ML completion event to */ + odp_queue_param_init(&queue_param); + queue_param.type = ODP_QUEUE_TYPE_SCHED; + queue_param.sched.sync = ODP_SCHED_SYNC_PARALLEL; + queue_param.sched.prio = odp_schedule_default_prio(); + queue_param.sched.group = ODP_SCHED_GROUP_ALL; + + global.queue = odp_queue_create("ML compl queue", &queue_param); + if (global.queue == ODP_QUEUE_INVALID) { + ODPH_ERR("Queue create failed\n"); + goto error; + } + + /* Create an ML job completion pool */ + if (ml_capa->pool.max_num < NUM_COMPL) { + ODPH_ERR("Too small ML compl pool %u\n", ml_capa->pool.max_num); + goto error; + } + + odp_ml_compl_pool_param_init(&ml_pool_param); + ml_pool_param.num = NUM_COMPL; + + global.compl_pool = odp_ml_compl_pool_create(COMPL_POOL_NAME, &ml_pool_param); + if (global.compl_pool == ODP_POOL_INVALID) { + ODPH_ERR("Create ML completion pool failed\n"); + goto error; + } + + return 0; + +error: + free(global.model_param.model); + return -1; +} + +static int ml_suite_term(void) +{ + if (global.compl_pool != ODP_POOL_INVALID && + odp_pool_destroy(global.compl_pool)) { + ODPH_ERR("Completion pool destroy failed\n"); + return -1; + } + + if (global.ml_model && odp_ml_model_destroy(global.ml_model)) { + ODPH_ERR("Destroy ML model failed\n"); + return -1; + } + + if (global.queue != ODP_QUEUE_INVALID && + odp_queue_destroy(global.queue)) { + ODPH_ERR("Destroy ML queue failed\n"); + return -1; + } + + free(global.model_param.model); + + return 0; +} + +static int check_ml_support(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + return ODP_TEST_ACTIVE; +} + +static int check_load_sync(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + if (global.ml_config.load_mode_mask & ODP_ML_COMPL_MODE_SYNC) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + +static int check_load_poll(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + if (global.ml_config.load_mode_mask & ODP_ML_COMPL_MODE_POLL) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + +static int check_load_event(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + if (global.ml_config.load_mode_mask & ODP_ML_COMPL_MODE_EVENT) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + +static int check_run_sync(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + /* Model run test uses synchronous load */ + if ((global.ml_config.run_mode_mask & ODP_ML_COMPL_MODE_SYNC) && + (global.ml_config.load_mode_mask & ODP_ML_COMPL_MODE_SYNC)) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + +static int check_run_poll(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + /* Poll mode model run test uses synchronous load */ + if ((global.ml_config.run_mode_mask & ODP_ML_COMPL_MODE_POLL) && + (global.ml_config.load_mode_mask & ODP_ML_COMPL_MODE_SYNC)) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + +static int check_run_event(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + /* Poll mode model run test uses synchronous load */ + if ((global.ml_config.run_mode_mask & ODP_ML_COMPL_MODE_EVENT) && + (global.ml_config.load_mode_mask & ODP_ML_COMPL_MODE_SYNC)) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + +static int check_run_poll_event(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + /* test_ml_run_start_multi uses synchronous load, poll mode and event mode run */ + if ((global.ml_config.run_mode_mask & ODP_ML_COMPL_MODE_EVENT) && + (global.ml_config.run_mode_mask & ODP_ML_COMPL_MODE_POLL) && + (global.ml_config.load_mode_mask & ODP_ML_COMPL_MODE_SYNC)) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + +static void test_ml_debug(void) +{ + uint64_t u64; + + u64 = odp_ml_model_to_u64(global.ml_model); + CU_ASSERT(u64 != odp_ml_model_to_u64(ODP_ML_MODEL_INVALID)); + printf("\n ML model handle: 0x%" PRIx64 "\n", u64); + + odp_ml_model_print(global.ml_model); +} + +static void test_ml_model_create(void) +{ + uint32_t i; + /* One for global.ml_model */ + uint32_t max_models = global.ml_config.max_models_created - 1; + odp_ml_model_t models[max_models]; + + for (i = 0; i < max_models; i++) { + models[i] = odp_ml_model_create(NULL, &global.model_param); + + if (models[i] == ODP_ML_MODEL_INVALID) { + ODPH_ERR("ML model create failed: %u / %u\n", i, max_models); + break; + } + } + + CU_ASSERT(i == max_models); + max_models = i; + + /* Destroy valid models */ + for (i = 0; i < max_models; i++) + CU_ASSERT_FATAL(odp_ml_model_destroy(models[i]) == 0); +} + +static void test_ml_model_lookup(void) +{ + odp_ml_model_t model2; + odp_ml_model_t model_lookup; + + /* Look up model with the same name, should find one with equal handle */ + model_lookup = odp_ml_model_lookup(MODEL_NAME); + CU_ASSERT_FATAL(model_lookup != ODP_ML_MODEL_INVALID); + CU_ASSERT(odp_ml_model_to_u64(global.ml_model) == odp_ml_model_to_u64(model_lookup)); + + /* Look up model with a different name, should return invalid handle */ + model_lookup = odp_ml_model_lookup("diff"); + CU_ASSERT_FATAL(model_lookup == ODP_ML_MODEL_INVALID); + + model2 = odp_ml_model_create(MODEL_NAME, &global.model_param); + CU_ASSERT_FATAL(model2 != ODP_ML_MODEL_INVALID); + CU_ASSERT(odp_ml_model_to_u64(global.ml_model) != odp_ml_model_to_u64(model2)); + + model_lookup = odp_ml_model_lookup(MODEL_NAME); + CU_ASSERT(odp_ml_model_to_u64(model_lookup) == odp_ml_model_to_u64(global.ml_model) || + odp_ml_model_to_u64(model_lookup) == odp_ml_model_to_u64(model2)); + + CU_ASSERT(odp_ml_model_destroy(model2) == 0); +} + +static void test_ml_model_info(void) +{ + int ret; + uint32_t num_ret; + odp_ml_model_info_t ml_info; + odp_ml_input_info_t input_info[2]; + odp_ml_output_info_t output_info[2]; + + /* Verify model info about global.ml_model, namely, simple_linear.onnx */ + memset(&ml_info, 0x88, sizeof(odp_ml_model_info_t)); + ret = odp_ml_model_info(global.ml_model, &ml_info); + CU_ASSERT(ret == 0); + CU_ASSERT(!strcmp(ml_info.name, MODEL_NAME)); + CU_ASSERT(ml_info.model_version == 1); + CU_ASSERT(ml_info.num_inputs == NUM_INPUTS); + CU_ASSERT(ml_info.num_outputs == NUM_OUTPUTS); + + num_ret = odp_ml_model_input_info(global.ml_model, input_info, NUM_INPUTS); + CU_ASSERT(num_ret == NUM_INPUTS); + CU_ASSERT(!strcmp(input_info[0].name, "x")); + CU_ASSERT(input_info[0].shape.num_dim == 1); + CU_ASSERT(input_info[0].shape.dim[0] == 1); + CU_ASSERT((int)input_info[0].data_type == ODP_ML_DATA_TYPE_INT32); + + /* When num is 0, return normally, and input_info is ignored */ + num_ret = odp_ml_model_input_info(global.ml_model, input_info, 0); + CU_ASSERT(num_ret == NUM_INPUTS); + + /* When num is bigger than actual number of inputs, extra input_info is left untouched */ + input_info[1].data_type = (odp_ml_data_type_t)-1; + num_ret = odp_ml_model_input_info(global.ml_model, input_info, NUM_INPUTS + 1); + CU_ASSERT(num_ret == NUM_INPUTS); + CU_ASSERT(!strcmp(input_info[0].name, "x")); + CU_ASSERT(input_info[0].shape.num_dim == 1); + CU_ASSERT(input_info[0].shape.dim[0] == 1); + CU_ASSERT((int)input_info[0].data_type == ODP_ML_DATA_TYPE_INT32); + /* input_info[1] is left untouched */ + CU_ASSERT(input_info[1].data_type == (odp_ml_data_type_t)-1); + + num_ret = odp_ml_model_output_info(global.ml_model, output_info, NUM_OUTPUTS); + CU_ASSERT(num_ret == NUM_OUTPUTS); + CU_ASSERT(!strcmp(output_info[0].name, "y")); + CU_ASSERT(output_info[0].shape.num_dim == 1); + CU_ASSERT(output_info[0].shape.dim[0] == 1); + CU_ASSERT((int)output_info[0].data_type == ODP_ML_DATA_TYPE_INT32); + + /* When num is 0, return normally, and input_info is ignored */ + num_ret = odp_ml_model_output_info(global.ml_model, output_info, 0); + CU_ASSERT(num_ret == NUM_OUTPUTS); + + /* When num is bigger than actual number of inputs, extra output_info is left untouched */ + num_ret = odp_ml_model_output_info(global.ml_model, output_info, NUM_OUTPUTS + 1); + output_info[1].shape.num_dim = 98876; + CU_ASSERT(num_ret == NUM_OUTPUTS); + CU_ASSERT(!strcmp(output_info[0].name, "y")); + CU_ASSERT(output_info[0].shape.num_dim == 1); + CU_ASSERT(output_info[0].shape.dim[0] == 1); + CU_ASSERT((int)output_info[0].data_type == ODP_ML_DATA_TYPE_INT32); + /* output_info[1] is left untouched */ + CU_ASSERT(output_info[1].shape.num_dim == 98876); +} + +static void test_ml_model_load(void) +{ + int ret; + odp_ml_model_t test_model; + odp_ml_load_result_t result; + + test_model = odp_ml_model_create(NULL, &global.model_param); + CU_ASSERT_FATAL(test_model != ODP_ML_MODEL_INVALID); + + ret = odp_ml_model_load(test_model, &result); + CU_ASSERT(ret == 0); + CU_ASSERT(result.error_code == 0); + + ret = odp_ml_model_unload(test_model, NULL); + CU_ASSERT(ret == 0); + + CU_ASSERT(odp_ml_model_destroy(test_model) == 0); +} + +/* Test asynchronous model loading in ODP_ML_COMPL_MODE_POLL mode */ +static void test_ml_model_load_async_poll(void) +{ + int ret; + odp_ml_load_result_t result; + odp_ml_compl_param_t compl_param; + int dummy = 6; + void *user_ptr = &dummy; + uint64_t wait_ns = 500 * ODP_TIME_MSEC_IN_NS; + + memset(&result, 0, sizeof(result)); + odp_ml_compl_param_init(&compl_param); + compl_param.mode = ODP_ML_COMPL_MODE_POLL; + compl_param.compl_id = 0; + compl_param.user_ptr = user_ptr; + + ret = odp_ml_model_load_start(global.ml_model, &compl_param); + CU_ASSERT_FATAL(ret == 0); + + /* When odp_ml_model_load_start() succeeded, continue to check completion status */ + for (int i = 0; i < TIMEOUT; i++) { + ret = odp_ml_model_load_status(global.ml_model, 0, &result); + if (ret) + break; + + /* ret = 0 meaning run has not finished, continue to check status */ + odp_time_wait_ns(wait_ns); + } + + CU_ASSERT(ret > 0); + CU_ASSERT(result.error_code == 0); + CU_ASSERT(result.user_ptr == user_ptr); + /* odp_ml_model_load does not modify data in user_ptr */ + if (result.user_ptr) + CU_ASSERT(*(int *)result.user_ptr == dummy); + + ret = odp_ml_model_unload_start(global.ml_model, &compl_param); + CU_ASSERT_FATAL(ret == 0); + + /* When odp_ml_model_unload_start() succeeded, continue to check completion + * status */ + for (int i = 0; i < TIMEOUT; i++) { + ret = odp_ml_model_unload_status(global.ml_model, 0, &result); + if (ret) + break; + + /* ret = 0 meaning run has not finished, continue to check status */ + odp_time_wait_ns(wait_ns); + } + + CU_ASSERT_FATAL(ret > 0); + CU_ASSERT(result.error_code == 0); + CU_ASSERT(result.user_ptr == user_ptr); + + /* odp_ml_model_unload does not modify data in user_ptr */ + if (result.user_ptr) + CU_ASSERT(*(int *)result.user_ptr == dummy); +} + +static int +get_result_from_ml_compl_event(odp_ml_load_result_t *load_result, odp_ml_run_result_t *run_result) +{ + int ret; + odp_event_t ev; + odp_ml_compl_t compl; + odp_event_type_t ev_type; + odp_queue_t from_queue = ODP_QUEUE_INVALID; + uint64_t sched_wait = odp_schedule_wait_time(global.wait_ns); + + /* Run event scheduler to find the ml completion event */ + for (int i = 0; i < TIMEOUT; i++) { + ev = odp_schedule(&from_queue, sched_wait); + if (ev != ODP_EVENT_INVALID) + break; + } + + CU_ASSERT(ev != ODP_EVENT_INVALID); + if (ev == ODP_EVENT_INVALID) { + ODPH_ERR("Timeout while waiting for completion event\n"); + return -1; + } + + ev_type = odp_event_type(ev); + CU_ASSERT(from_queue == global.queue); + CU_ASSERT(ev_type == ODP_EVENT_ML_COMPL); + if (from_queue != global.queue || ev_type != ODP_EVENT_ML_COMPL) { + odp_event_free(ev); + ODPH_ERR("Received unexpected event while waiting for completion\n"); + return -1; + } + + compl = odp_ml_compl_from_event(ev); + CU_ASSERT(compl != ODP_ML_COMPL_INVALID); + + if (load_result) { + CU_ASSERT(odp_ml_compl_load_result(compl, NULL) == 0); + ret = odp_ml_compl_load_result(compl, load_result); + } else { + CU_ASSERT(odp_ml_compl_run_result(compl, NULL) == 0); + ret = odp_ml_compl_run_result(compl, run_result); + } + + CU_ASSERT(ret == 0); + odp_ml_compl_free(compl); + + return ret; +} + +/* Test asynchronous model loading in ODP_ML_COMPL_MODE_EVENT mode */ +static void test_ml_model_load_async_event(void) +{ + int ret; + odp_ml_compl_t compl; + odp_ml_load_result_t result; + odp_ml_compl_param_t compl_param; + int dummy = 6; + void *user_ptr = &dummy; + + compl = odp_ml_compl_alloc(global.compl_pool); + CU_ASSERT_FATAL(compl != ODP_ML_COMPL_INVALID); + + odp_ml_compl_param_init(&compl_param); + compl_param.mode = ODP_ML_COMPL_MODE_EVENT; + compl_param.event = odp_ml_compl_to_event(compl); + compl_param.queue = global.queue; + compl_param.user_ptr = user_ptr; + + ret = odp_ml_model_load_start(global.ml_model, &compl_param); + CU_ASSERT(ret == 0); + + /* Return when odp_ml_model_load_start() failed */ + if (ret) { + odp_ml_compl_free(compl); + ODPH_ERR("ML model odp_ml_model_load_start() failed\n"); + return; + } + + /* Run event scheduler to find the ml completion event and verify it */ + if (get_result_from_ml_compl_event(&result, NULL)) + return; + + CU_ASSERT(result.error_code == 0); + CU_ASSERT(result.user_ptr == user_ptr); + + /* Model load does not modify data in user_ptr */ + if (result.user_ptr) + CU_ASSERT(*(int *)result.user_ptr == dummy); + + compl = odp_ml_compl_alloc(global.compl_pool); + CU_ASSERT(compl != ODP_ML_COMPL_INVALID); + + if (compl == ODP_ML_COMPL_INVALID) + return; + + compl_param.event = odp_ml_compl_to_event(compl); + ret = odp_ml_model_unload_start(global.ml_model, &compl_param); + CU_ASSERT_FATAL(ret == 0); + + /* Run event scheduler to find the ml completion event and verify it */ + if (get_result_from_ml_compl_event(&result, NULL)) + return; + + CU_ASSERT(result.error_code == 0); + CU_ASSERT(result.user_ptr == user_ptr); + + /* odp_ml_model_unload does not modify data in user_ptr */ + if (result.user_ptr) + CU_ASSERT(*(int *)result.user_ptr == dummy); +} + +/* About model batch_add.onnx being tested in this function + * + * Model info: + * Version: 1 + * Inputs: + * inputs[0]: name: x1, type: double, shape: [c, 3] + * inputs[1]: name: x2, type: double, shape: [c, 3] + * Outputs: + * Outputs[0]: name: y, type: double, shape: [c, 3] + * + * The model computes element-wise sum of input tensors x1 and x2 and stores them + * in y. The first dimension of input and output tensors represent batch size, + * thus it must be the same for all tensors here. The dynamic dimension size + * in the output tensor here can be deduced from the given batch size, thus no + * need for the implementation to fill it. + */ +#define NUM_COLUMN 3 +#define MAX_BATCH_SIZE 4 +#define SIZE (NUM_COLUMN * MAX_BATCH_SIZE * sizeof(double)) +static void run_model_batch_add(void) +{ + int ret; + odp_ml_data_t data; + odp_ml_model_t model; + odp_ml_data_seg_t input_segs[SIZE * 2]; + odp_ml_data_seg_t output_segs[SIZE]; + odp_ml_run_result_t result; + odp_ml_run_param_t run_param; + odp_ml_model_param_t model_param; + + double y[12]; + double y_expected[12]; + uint32_t batch_size = MAX_BATCH_SIZE; + double x1[12] = {97, 47, 62, 19, 93, 59, 67, 42, 28, 55, 46, 31}; + double x2[12] = {81, 56, 27, 4, 69, 12, 91, 98, 23, 90, 52, 64}; + + for (int i = 0; i < 12; i++) + y_expected[i] = x1[i] + x2[i]; + + odp_ml_model_param_init(&model_param); + + odp_ml_data_format_t input_format[2] = { + { + .data_type = ODP_ML_DATA_TYPE_FP64, + .data_type_size = 8, + .shape.type = ODP_ML_SHAPE_BATCH, + .shape.num_dim = 2, + .shape.dim = {ODP_ML_DIM_DYNAMIC, NUM_COLUMN}, + .shape.dim_max = {MAX_BATCH_SIZE, NUM_COLUMN} + }, + { + .data_type = ODP_ML_DATA_TYPE_FP64, + .data_type_size = 8, + .shape.type = ODP_ML_SHAPE_BATCH, + .shape.num_dim = 2, + .shape.dim = {ODP_ML_DIM_DYNAMIC, NUM_COLUMN}, + .shape.dim_max = {MAX_BATCH_SIZE, NUM_COLUMN} + } + }; + + model_param.extra_info.num_inputs = 2; + model_param.extra_info.input_format = input_format; + + /* Verify model info about matrix_mul.onnx */ + if (fill_model_param("batch_add.onnx", &model_param)) + return; + + model = odp_ml_model_create("batch_add", &model_param); + free(model_param.model); + CU_ASSERT(model != ODP_ML_MODEL_INVALID); + if (!model) + return; + + if (odp_ml_model_load(model, NULL)) { + CU_ASSERT(odp_ml_model_destroy(model) == 0); + return; + } + + odp_ml_model_print(model); + + /* Prepare parameters for running inference */ + odp_ml_run_param_init(&run_param); + run_param.result = &result; + + data.num_input_seg = 2; + data.input_seg = input_segs; + input_segs[0].addr = x1; + input_segs[1].addr = x2; + + data.num_output_seg = 1; + data.output_seg = output_segs; + output_segs[0].size = sizeof(y); + output_segs[0].addr = y; + + /* Test different batch sizes */ + for (int i = 0; i < MAX_BATCH_SIZE; i++) { + run_param.batch_size = batch_size; + input_segs[0].size = sizeof(double) * NUM_COLUMN * batch_size; + input_segs[1].size = sizeof(double) * NUM_COLUMN * batch_size; + ret = odp_ml_run(model, &data, &run_param); + CU_ASSERT(ret == 1); + if (ret != 1) + goto fail; + + for (uint32_t j = 0; j < batch_size * NUM_COLUMN; j++) + CU_ASSERT(y[j] == y_expected[j]); + + batch_size--; + } + + /* Test also without run results */ + run_param.result = NULL; + ret = odp_ml_run(model, &data, &run_param); + CU_ASSERT(ret == 1); + + /* Test different segment sizes */ + batch_size = MAX_BATCH_SIZE; + odp_ml_run_param_init(&run_param); + run_param.result = &result; + run_param.batch_size = batch_size; + data.input_seg = input_segs; + data.output_seg = output_segs; + + for (int seg_size = SIZE; seg_size > 0; seg_size--) { + int num_seg = (SIZE + seg_size - 1) / seg_size; + + if ((uint32_t)num_seg > global.ml_capa.max_segs_per_input || + (uint32_t)num_seg > global.ml_capa.max_segs_per_output) + break; + + data.num_input_seg = num_seg * 2; + data.num_output_seg = num_seg; + + for (int seg = 0; seg < num_seg; seg++) { + int size = seg_size; + + if (seg == num_seg - 1) + size = SIZE - seg * seg_size; + + input_segs[seg].addr = (char *)x1 + seg * seg_size; + input_segs[seg].size = size; + input_segs[seg + num_seg].addr = (char *)x2 + seg * seg_size; + input_segs[seg + num_seg].size = size; + output_segs[seg].addr = (char *)y + seg * seg_size; + output_segs[seg].size = size; + } + + memset(y, 0, sizeof(y)); + ret = odp_ml_run(model, &data, &run_param); + CU_ASSERT(ret == 1); + if (ret != 1) + goto fail; + + for (uint32_t j = 0; j < batch_size * NUM_COLUMN; j++) + CU_ASSERT(y[j] == y_expected[j]); + } + +fail: + CU_ASSERT_FATAL(odp_ml_model_unload(model, NULL) == 0); + CU_ASSERT(odp_ml_model_destroy(model) == 0); +} + +static void run_global_ml_model(void) +{ + int ret = 0; + odp_ml_run_result_t result; + + ret = odp_ml_model_load(global.ml_model, NULL); + CU_ASSERT_FATAL(ret == 0); + + global.run_param.result = &result; + + ret = odp_ml_run(global.ml_model, &global.data, &global.run_param); + CU_ASSERT(ret == 1); + CU_ASSERT(!result.error_code); + CU_ASSERT(*(int32_t *)global.output_seg.addr == global.y_expected); + + ret = odp_ml_model_unload(global.ml_model, NULL); + CU_ASSERT_FATAL(ret == 0); + global.run_param.result = NULL; +} + +static void test_ml_run(void) +{ + run_global_ml_model(); + run_model_batch_add(); +} + +static void test_ml_run_multi(void) +{ + int ret; + int32_t y; + int32_t x = 8; + int32_t y_expected = 28; + odp_ml_data_t data[RUN_NUM]; + odp_ml_data_seg_t input_seg; + odp_ml_data_seg_t output_seg; + odp_ml_run_param_t param[RUN_NUM]; + odp_ml_run_result_t result[RUN_NUM]; + uint64_t wait_ns = 500 * ODP_TIME_MSEC_IN_NS; + + ret = odp_ml_model_load(global.ml_model, NULL); + CU_ASSERT_FATAL(ret == 0); + + param[0] = global.run_param; + param[0].result = &result[0]; + odp_ml_run_param_init(¶m[1]); + param[1].result = &result[1]; + + /* Prepare data for running model inference */ + data[0] = global.data; + data[1].num_input_seg = NUM_INPUTS; + data[1].input_seg = &input_seg; + input_seg.size = sizeof(int32_t); + input_seg.addr = &x; + + data[1].num_output_seg = NUM_OUTPUTS; + data[1].output_seg = &output_seg; + output_seg.size = sizeof(int32_t); + output_seg.addr = &y; + + int num_completed = 0; + + for (int i = 0; i < TIMEOUT; i++) { + ret = odp_ml_run_multi(global.ml_model, data + num_completed, param + num_completed, + RUN_NUM - num_completed); + CU_ASSERT(ret >= 0); + if (ret < 0) + break; + + num_completed += ret; + + if (num_completed >= RUN_NUM) + break; + + odp_time_wait_ns(wait_ns); + } + + CU_ASSERT(num_completed == RUN_NUM); + CU_ASSERT(!result[0].error_code); + CU_ASSERT(!result[1].error_code); + CU_ASSERT(*(int32_t *)global.output_seg.addr == global.y_expected); + CU_ASSERT(*(int32_t *)output_seg.addr == y_expected); + + ret = odp_ml_model_unload(global.ml_model, NULL); + CU_ASSERT_FATAL(ret == 0); +} + +/* Test asynchronous inference running in ODP_ML_COMPL_MODE_EVENT mode */ +static void test_ml_model_run_async_event(void) +{ + int ret; + void *user_ptr; + odp_ml_compl_t compl; + odp_ml_run_result_t result; + odp_ml_data_seg_t *outputs; + odp_ml_compl_param_t compl_param; + + /* Load model in order to run inference */ + ret = odp_ml_model_load(global.ml_model, NULL); + CU_ASSERT_FATAL(ret == 0); + + compl = odp_ml_compl_alloc(global.compl_pool); + CU_ASSERT_FATAL(compl != ODP_ML_COMPL_INVALID); + + odp_ml_compl_param_init(&compl_param); + compl_param.mode = ODP_ML_COMPL_MODE_EVENT; + compl_param.event = odp_ml_compl_to_event(compl); + compl_param.queue = global.queue; + + /* user_ptr structure maintains the output data pointer for output retrieval */ + user_ptr = &global.output_seg; + compl_param.user_ptr = user_ptr; + + memset(global.output_seg.addr, 0, global.output_seg.size); + ret = odp_ml_run_start(global.ml_model, &global.data, &compl_param, NULL); + CU_ASSERT_FATAL(ret == 1); + + /* Run event scheduler to find the ml completion event and verify it */ + if (get_result_from_ml_compl_event(NULL, &result)) + return; + + CU_ASSERT(!result.error_code); + CU_ASSERT(result.user_ptr == user_ptr); + + outputs = (odp_ml_data_seg_t *)result.user_ptr; + CU_ASSERT(*(int32_t *)outputs[0].addr == global.y_expected); + + /* Unload model */ + ret = odp_ml_model_unload(global.ml_model, NULL); + CU_ASSERT_FATAL(ret == 0); +} + +/* Test asynchronous inference running in ODP_ML_COMPL_MODE_POLL mode */ +static void test_ml_model_run_async_poll(void) +{ + int ret; + void *user_ptr; + odp_ml_run_result_t result; + odp_ml_data_seg_t *outputs; + odp_ml_compl_param_t compl_param; + uint64_t wait_ns = 500 * ODP_TIME_MSEC_IN_NS; + + memset(&result, 0, sizeof(result)); + /* Load model in order to run inference */ + ret = odp_ml_model_load(global.ml_model, NULL); + CU_ASSERT_FATAL(ret == 0); + + odp_ml_compl_param_init(&compl_param); + compl_param.mode = ODP_ML_COMPL_MODE_POLL; + compl_param.compl_id = 0; + + /* user_ptr structure maintains the output data pointer for output retrieval */ + user_ptr = &global.output_seg; + compl_param.user_ptr = user_ptr; + + memset(global.output_seg.addr, 0, global.output_seg.size); + ret = odp_ml_run_start(global.ml_model, &global.data, &compl_param, NULL); + CU_ASSERT_FATAL(ret == 1); + + /* When odp_ml_run_start() succeeded, continue to check completion status */ + for (int i = 0; i < TIMEOUT; i++) { + ret = odp_ml_run_status(global.ml_model, 0, &result); + if (ret) + break; + + /* ret = 0 meaning run has not finished, continue to check status */ + odp_time_wait_ns(wait_ns); + } + + outputs = (odp_ml_data_seg_t *)result.user_ptr; + + CU_ASSERT(ret > 0); + CU_ASSERT(!result.error_code); + CU_ASSERT(result.user_ptr == user_ptr); + CU_ASSERT(*(int32_t *)outputs[0].addr == global.y_expected); + + /* Unload model */ + ret = odp_ml_model_unload(global.ml_model, NULL); + CU_ASSERT_FATAL(ret == 0); +} + +static void test_ml_run_start_multi(void) +{ + int ret; + int32_t y; + odp_ml_compl_t compl; + odp_ml_data_t data[RUN_NUM]; + odp_ml_data_seg_t input_seg; + odp_ml_data_seg_t output_seg; + odp_ml_data_seg_t *outputs[RUN_NUM]; + odp_ml_compl_param_t compl_param[RUN_NUM]; + odp_ml_run_result_t run_result[RUN_NUM]; + int32_t x = 5; + int32_t y_expected = 19; + uint64_t wait_ns = 500 * ODP_TIME_MSEC_IN_NS; + + /* Load model in order to run inference */ + ret = odp_ml_model_load(global.ml_model, NULL); + CU_ASSERT_FATAL(ret == 0); + + compl = odp_ml_compl_alloc(global.compl_pool); + CU_ASSERT_FATAL(compl != ODP_ML_COMPL_INVALID); + + /* Prepare data for running model inference */ + data[0] = global.data; + + data[1].num_input_seg = NUM_INPUTS; + data[1].input_seg = &input_seg; + input_seg.size = sizeof(int32_t); + input_seg.addr = &x; + + data[1].num_output_seg = NUM_OUTPUTS; + data[1].output_seg = &output_seg; + output_seg.size = sizeof(int32_t); + output_seg.addr = &y; + + /* Two completion parameters: one use event mode, another poll mode */ + odp_ml_compl_param_init(&compl_param[0]); + compl_param[0].mode = ODP_ML_COMPL_MODE_EVENT; + compl_param[0].event = odp_ml_compl_to_event(compl); + compl_param[0].queue = global.queue; + /* user_ptr structure maintains the output data pointer for output retrieval */ + compl_param[0].user_ptr = &global.output_seg; + + odp_ml_compl_param_init(&compl_param[1]); + compl_param[1].mode = ODP_ML_COMPL_MODE_POLL; + compl_param[1].compl_id = 0; + /* user_ptr structure maintains the output data pointer for output retrieval */ + compl_param[1].user_ptr = &output_seg; + + memset(global.output_seg.addr, 0, sizeof(int32_t)); + + int num_completed = 0; + + for (int i = 0; i < TIMEOUT; i++) { + ret = odp_ml_run_start_multi(global.ml_model, data + num_completed, + compl_param + num_completed, NULL, + RUN_NUM - num_completed); + CU_ASSERT(ret >= 0); + if (ret < 0) + break; + + num_completed += ret; + + if (num_completed >= RUN_NUM) + break; + + odp_time_wait_ns(wait_ns); + } + + CU_ASSERT(num_completed == RUN_NUM); + + /* Run event scheduler to find the ml completion event and verify it */ + if (get_result_from_ml_compl_event(NULL, &run_result[0])) { + ret = odp_ml_model_unload(global.ml_model, NULL); + return; + } + + CU_ASSERT(!run_result[0].error_code); + CU_ASSERT(run_result[0].user_ptr == &global.output_seg); + outputs[0] = (odp_ml_data_seg_t *)run_result[0].user_ptr; + CU_ASSERT(*(int32_t *)outputs[0][0].addr == global.y_expected); + + /* Check completion status for the poll mode */ + for (int i = 0; i < TIMEOUT; i++) { + ret = odp_ml_run_status(global.ml_model, 0, &run_result[1]); + if (ret) + break; + + /* ret = 0 meaning run has not finished, continue to check status */ + odp_time_wait_ns(wait_ns); + } + + outputs[1] = (odp_ml_data_seg_t *)run_result[1].user_ptr; + CU_ASSERT(ret > 0); + CU_ASSERT(!run_result[1].error_code); + CU_ASSERT(run_result[1].user_ptr == &output_seg); + CU_ASSERT(*(int32_t *)outputs[1][0].addr == y_expected); + + /* Unload model */ + ret = odp_ml_model_unload(global.ml_model, NULL); + CU_ASSERT_FATAL(ret == 0); +} + +static void test_ml_model_extra_stat_info(void) +{ + int ret; + + ret = odp_ml_model_extra_stat_info(global.ml_model, NULL, 0); + CU_ASSERT(ret >= 0); +} + +static void test_ml_model_extra_stats(void) +{ + int ret; + + ret = odp_ml_model_extra_stats(global.ml_model, NULL, 0); + CU_ASSERT(ret >= 0); +} + +odp_testinfo_t ml_suite[] = { + ODP_TEST_INFO_CONDITIONAL(test_ml_debug, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_model_create, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_model_lookup, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_model_info, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_model_load, check_load_sync), + ODP_TEST_INFO_CONDITIONAL(test_ml_model_load_async_poll, check_load_poll), + ODP_TEST_INFO_CONDITIONAL(test_ml_model_load_async_event, check_load_event), + /* Synchronous load/unload is used load/unload model before/after model run */ + ODP_TEST_INFO_CONDITIONAL(test_ml_run, check_run_sync), + ODP_TEST_INFO_CONDITIONAL(test_ml_run_multi, check_run_sync), + ODP_TEST_INFO_CONDITIONAL(test_ml_model_run_async_event, check_run_event), + ODP_TEST_INFO_CONDITIONAL(test_ml_model_run_async_poll, check_run_poll), + ODP_TEST_INFO_CONDITIONAL(test_ml_run_start_multi, check_run_poll_event), + ODP_TEST_INFO_CONDITIONAL(test_ml_model_extra_stat_info, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_model_extra_stats, check_ml_support), + ODP_TEST_INFO_NULL +}; + +odp_suiteinfo_t ml_suites[] = { + {"ML", ml_suite_init, ml_suite_term, ml_suite}, + ODP_SUITE_INFO_NULL +}; + +int main(int argc, char *argv[]) +{ + int ret; + + /* parse common options: */ + if (odp_cunit_parse_options(&argc, argv)) + return -1; + + ret = odp_cunit_register(ml_suites); + + if (ret == 0) + ret = odp_cunit_run(); + + return ret; +} diff --git a/platform/linux-generic/test/validation/api/ml/requirements.txt b/platform/linux-generic/test/validation/api/ml/requirements.txt new file mode 100644 index 000000000..2dcba7a3a --- /dev/null +++ b/platform/linux-generic/test/validation/api/ml/requirements.txt @@ -0,0 +1,2 @@ +onnx +numpy diff --git a/platform/linux-generic/test/validation/api/ml/simple_linear.onnx b/platform/linux-generic/test/validation/api/ml/simple_linear.onnx Binary files differnew file mode 100644 index 000000000..45c4b95b9 --- /dev/null +++ b/platform/linux-generic/test/validation/api/ml/simple_linear.onnx diff --git a/platform/linux-generic/test/validation/api/ml/simple_linear_gen.py b/platform/linux-generic/test/validation/api/ml/simple_linear_gen.py new file mode 100644 index 000000000..b3e6124cd --- /dev/null +++ b/platform/linux-generic/test/validation/api/ml/simple_linear_gen.py @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Nokia +# + +import onnx +from onnx import helper +from onnx import TensorProto + +weight = helper.make_tensor(name='w', data_type=TensorProto.INT32, dims=[1], vals=[3]) +w = helper.make_node('Constant', inputs=[], outputs=['w'], name='weight', value=weight) + +bias = helper.make_tensor(name='b', data_type=TensorProto.INT32, dims=[1], vals=[4]) +b = helper.make_node('Constant', inputs=[], outputs=['b'], name='bias', value=bias) + +# The functional nodes: +mul = helper.make_node('Mul', inputs=['x', 'w'], outputs=['wx'], name='Mul') +add = helper.make_node('Add', inputs=['wx', 'b'], outputs=['y'], name='Add') + +# Create the graph +g = helper.make_graph([w, mul, b, add], 'linear', + [helper.make_tensor_value_info('x', TensorProto.INT32, [1])], + [helper.make_tensor_value_info('y', TensorProto.INT32, [1])] +) + +model = helper.make_model( + producer_name='ODP validation tests', + model_version=1, + doc_string="y = 3x + 4", + graph=g, + opset_imports=[helper.make_opsetid("", 13)] +) + +# Save the model +onnx.save(model, 'simple_linear.onnx') diff --git a/platform/linux-generic/test/validation/api/shmem/shmem_odp1.c b/platform/linux-generic/test/validation/api/shmem/shmem_odp1.c index 3b4ba819c..98148d6c7 100644 --- a/platform/linux-generic/test/validation/api/shmem/shmem_odp1.c +++ b/platform/linux-generic/test/validation/api/shmem/shmem_odp1.c @@ -79,7 +79,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(shmem_suites); diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 83bdf378c..71af1c682 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -23,6 +23,9 @@ my $V = '0.32'; use Getopt::Long qw(:config no_auto_abbrev); my $quiet = 0; +my $verbose = 0; +my %verbose_messages = (); +my %verbose_emitted = (); my $tree = 1; my $chk_signoff = 1; my $chk_patch = 1; @@ -60,7 +63,9 @@ my $min_conf_desc_length = 4; my $spelling_file = "$D/spelling.txt"; my $codespell = 0; my $codespellfile = "/usr/share/codespell/dictionary.txt"; +my $user_codespellfile = ""; my $conststructsfile = "$D/const_structs.checkpatch"; +my $docsfile = "$D/../Documentation/dev-tools/checkpatch.rst"; my $typedefsfile; my $color = "auto"; my $allow_c99_comments = 1; # Can be overridden by --ignore C99_COMMENT_TOLERANCE @@ -69,6 +74,8 @@ my $git_command ='export LANGUAGE=en_US.UTF-8; git'; my $tabsize = 8; my ${CONFIG_} = "CONFIG_"; +my %maybe_linker_symbol; # for externs in c exceptions, when seen in *vmlinux.lds.h + sub help { my ($exitcode) = @_; @@ -78,6 +85,7 @@ Version: $V Options: -q, --quiet quiet + -v, --verbose verbose mode --no-tree run without a kernel tree --no-signoff do not check for 'Signed-off-by' line --patch treat FILE as patchfile (default) @@ -125,7 +133,7 @@ Options: --ignore-perl-version override checking of perl version. expect runtime errors. --codespell Use the codespell dictionary for spelling/typos - (default:/usr/share/codespell/dictionary.txt) + (default:$codespellfile) --codespellfile Use this codespell dictionary --typedefsfile Read additional types from this file --color[=WHEN] Use colors 'always', 'never', or only when output @@ -158,15 +166,51 @@ sub list_types { my $text = <$script>; close($script); - my @types = (); + my %types = (); # Also catch when type or level is passed through a variable - for ($text =~ /(?:(?:\bCHK|\bWARN|\bERROR|&\{\$msg_level})\s*\(|\$msg_type\s*=)\s*"([^"]+)"/g) { - push (@types, $_); + while ($text =~ /(?:(\bCHK|\bWARN|\bERROR|&\{\$msg_level})\s*\(|\$msg_type\s*=)\s*"([^"]+)"/g) { + if (defined($1)) { + if (exists($types{$2})) { + $types{$2} .= ",$1" if ($types{$2} ne $1); + } else { + $types{$2} = $1; + } + } else { + $types{$2} = "UNDETERMINED"; + } } - @types = sort(uniq(@types)); + print("#\tMessage type\n\n"); - foreach my $type (@types) { + if ($color) { + print(" ( Color coding: "); + print(RED . "ERROR" . RESET); + print(" | "); + print(YELLOW . "WARNING" . RESET); + print(" | "); + print(GREEN . "CHECK" . RESET); + print(" | "); + print("Multiple levels / Undetermined"); + print(" )\n\n"); + } + + foreach my $type (sort keys %types) { + my $orig_type = $type; + if ($color) { + my $level = $types{$type}; + if ($level eq "ERROR") { + $type = RED . $type . RESET; + } elsif ($level eq "WARN") { + $type = YELLOW . $type . RESET; + } elsif ($level eq "CHK") { + $type = GREEN . $type . RESET; + } + } print(++$count . "\t" . $type . "\n"); + if ($verbose && exists($verbose_messages{$orig_type})) { + my $message = $verbose_messages{$orig_type}; + $message =~ s/\n/\n\t/g; + print("\t" . $message . "\n\n"); + } } exit($exitcode); @@ -198,6 +242,46 @@ if (-f $conf) { unshift(@ARGV, @conf_args) if @conf_args; } +sub load_docs { + open(my $docs, '<', "$docsfile") + or warn "$P: Can't read the documentation file $docsfile $!\n"; + + my $type = ''; + my $desc = ''; + my $in_desc = 0; + + while (<$docs>) { + chomp; + my $line = $_; + $line =~ s/\s+$//; + + if ($line =~ /^\s*\*\*(.+)\*\*$/) { + if ($desc ne '') { + $verbose_messages{$type} = trim($desc); + } + $type = $1; + $desc = ''; + $in_desc = 1; + } elsif ($in_desc) { + if ($line =~ /^(?:\s{4,}|$)/) { + $line =~ s/^\s{4}//; + $desc .= $line; + $desc .= "\n"; + } else { + $verbose_messages{$type} = trim($desc); + $type = ''; + $desc = ''; + $in_desc = 0; + } + } + } + + if ($desc ne '') { + $verbose_messages{$type} = trim($desc); + } + close($docs); +} + # Perl's Getopt::Long allows options to take optional arguments after a space. # Prevent --color by itself from consuming other arguments foreach (@ARGV) { @@ -208,6 +292,7 @@ foreach (@ARGV) { GetOptions( 'q|quiet+' => \$quiet, + 'v|verbose!' => \$verbose, 'tree!' => \$tree, 'signoff!' => \$chk_signoff, 'patch!' => \$chk_patch, @@ -235,7 +320,7 @@ GetOptions( 'debug=s' => \%debug, 'test-only=s' => \$tst_only, 'codespell!' => \$codespell, - 'codespellfile=s' => \$codespellfile, + 'codespellfile=s' => \$user_codespellfile, 'typedefsfile=s' => \$typedefsfile, 'color=s' => \$color, 'no-color' => \$color, #keep old behaviors of -nocolor @@ -243,17 +328,54 @@ GetOptions( 'kconfig-prefix=s' => \${CONFIG_}, 'h|help' => \$help, 'version' => \$help -) or help(1); +) or $help = 2; + +if ($user_codespellfile) { + # Use the user provided codespell file unconditionally + $codespellfile = $user_codespellfile; +} elsif (!(-f $codespellfile)) { + # If /usr/share/codespell/dictionary.txt is not present, try to find it + # under codespell's install directory: <codespell_root>/data/dictionary.txt + if (($codespell || $help) && which("python3") ne "") { + my $python_codespell_dict = << "EOF"; + +import os.path as op +import codespell_lib +codespell_dir = op.dirname(codespell_lib.__file__) +codespell_file = op.join(codespell_dir, 'data', 'dictionary.txt') +print(codespell_file, end='') +EOF + + my $codespell_dict = `python3 -c "$python_codespell_dict" 2> /dev/null`; + $codespellfile = $codespell_dict if (-f $codespell_dict); + } +} + +# $help is 1 if either -h, --help or --version is passed as option - exitcode: 0 +# $help is 2 if invalid option is passed - exitcode: 1 +help($help - 1) if ($help); -help(0) if ($help); +die "$P: --git cannot be used with --file or --fix\n" if ($git && ($file || $fix)); +die "$P: --verbose cannot be used with --terse\n" if ($verbose && $terse); +if ($color =~ /^[01]$/) { + $color = !$color; +} elsif ($color =~ /^always$/i) { + $color = 1; +} elsif ($color =~ /^never$/i) { + $color = 0; +} elsif ($color =~ /^auto$/i) { + $color = (-t STDOUT); +} else { + die "$P: Invalid color mode: $color\n"; +} + +load_docs() if ($verbose); list_types(0) if ($list_types); $fix = 1 if ($fix_inplace); $check_orig = $check; -die "$P: --git cannot be used with --file or --fix\n" if ($git && ($file || $fix)); - my $exit = 0; my $perl_version_ok = 1; @@ -268,18 +390,6 @@ if ($#ARGV < 0) { push(@ARGV, '-'); } -if ($color =~ /^[01]$/) { - $color = !$color; -} elsif ($color =~ /^always$/i) { - $color = 1; -} elsif ($color =~ /^never$/i) { - $color = 0; -} elsif ($color =~ /^auto$/i) { - $color = (-t STDOUT); -} else { - die "$P: Invalid color mode: $color\n"; -} - # skip TAB size 1 to avoid additional checks on $tabsize - 1 die "$P: Invalid TAB size: $tabsize\n" if ($tabsize < 2); @@ -402,10 +512,12 @@ our $Attribute = qr{ __ro_after_init| __kprobes| $InitAttribute| + __aligned\s*\(.*\)| ____cacheline_aligned| ____cacheline_aligned_in_smp| ____cacheline_internodealigned_in_smp| - __weak + __weak| + __alloc_size\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\) }x; our $Modifier; our $Inline = qr{inline|__always_inline|noinline|__inline|__inline__}; @@ -417,7 +529,7 @@ our $Binary = qr{(?i)0b[01]+$Int_type?}; our $Hex = qr{(?i)0x[0-9a-f]+$Int_type?}; our $Int = qr{[0-9]+$Int_type?}; our $Octal = qr{0[0-7]+$Int_type?}; -our $String = qr{"[X\t]*"}; +our $String = qr{(?:\b[Lu])?"[X\t]*"}; our $Float_hex = qr{(?i)0x[0-9a-f]+p-?[0-9]+[fl]?}; our $Float_dec = qr{(?i)(?:[0-9]+\.[0-9]*|[0-9]*\.[0-9]+)(?:e-?[0-9]+)?[fl]?}; our $Float_int = qr{(?i)[0-9]+e-?[0-9]+[fl]?}; @@ -467,10 +579,14 @@ our $typeKernelTypedefs = qr{(?x: (?:__)?(?:u|s|be|le)(?:8|16|32|64)| atomic_t )}; +our $typeStdioTypedefs = qr{(?x: + FILE +)}; our $typeTypedefs = qr{(?x: $typeC99Typedefs\b| $typeOtherOSTypedefs\b| - $typeKernelTypedefs\b + $typeKernelTypedefs\b| + $typeStdioTypedefs\b )}; our $zero_initializer = qr{(?:(?:0[xX])?0+$Int_type?|NULL|false)\b}; @@ -507,6 +623,22 @@ our $signature_tags = qr{(?xi: Cc: )}; +our @link_tags = qw(Link Closes); + +#Create a search and print patterns for all these strings to be used directly below +our $link_tags_search = ""; +our $link_tags_print = ""; +foreach my $entry (@link_tags) { + if ($link_tags_search ne "") { + $link_tags_search .= '|'; + $link_tags_print .= ' or '; + } + $entry .= ':'; + $link_tags_search .= $entry; + $link_tags_print .= "'$entry'"; +} +$link_tags_search = "(?:${link_tags_search})"; + our $tracing_logging_tags = qr{(?xi: [=-]*> | <[=-]* | @@ -589,6 +721,17 @@ sub find_standard_signature { return ""; } +our $obsolete_archives = qr{(?xi: + \Qfreedesktop.org/archives/dri-devel\E | + \Qlists.infradead.org\E | + \Qlkml.org\E | + \Qmail-archive.com\E | + \Qmailman.alsa-project.org/pipermail\E | + \Qmarc.info\E | + \Qozlabs.org/pipermail\E | + \Qspinics.net\E +)}; + our @typeListMisordered = ( qr{char\s+(?:un)?signed}, qr{int\s+(?:(?:un)?signed\s+)?short\s}, @@ -698,6 +841,10 @@ our %deprecated_apis = ( "rcu_barrier_sched" => "rcu_barrier", "get_state_synchronize_sched" => "get_state_synchronize_rcu", "cond_synchronize_sched" => "cond_synchronize_rcu", + "kmap" => "kmap_local_page", + "kunmap" => "kunmap_local", + "kmap_atomic" => "kmap_local_page", + "kunmap_atomic" => "kunmap_local", ); #Create a search pattern for all these strings to speed up a loop below @@ -933,7 +1080,8 @@ our $FuncArg = qr{$Typecast{0,1}($LvalOrFunc|$Constant|$String)}; our $declaration_macros = qr{(?x: (?:$Storage\s+)?(?:[A-Z_][A-Z0-9]*_){0,2}(?:DEFINE|DECLARE)(?:_[A-Z0-9]+){1,6}\s*\(| (?:$Storage\s+)?[HLP]?LIST_HEAD\s*\(| - (?:SKCIPHER_REQUEST|SHASH_DESC|AHASH_REQUEST)_ON_STACK\s*\( + (?:SKCIPHER_REQUEST|SHASH_DESC|AHASH_REQUEST)_ON_STACK\s*\(| + (?:$Storage\s+)?(?:XA_STATE|XA_STATE_ORDER)\s*\( )}; our %allow_repeated_words = ( @@ -1000,10 +1148,10 @@ sub is_maintained_obsolete { sub is_SPDX_License_valid { my ($license) = @_; - return 1 if (!$tree || which("python") eq "" || !(-e "$root/scripts/spdxcheck.py") || !(-e "$gitroot")); + return 1 if (!$tree || which("python3") eq "" || !(-x "$root/scripts/spdxcheck.py") || !(-e "$gitroot")); my $root_path = abs_path($root); - my $status = `cd "$root_path"; echo "$license" | python scripts/spdxcheck.py -`; + my $status = `cd "$root_path"; echo "$license" | scripts/spdxcheck.py -`; return 0 if ($status ne ""); return 1; } @@ -1097,7 +1245,8 @@ sub git_commit_info { # git log --format='%H %s' -1 $line | # echo "commit $(cut -c 1-12,41-)" # done - } elsif ($lines[0] =~ /^fatal: ambiguous argument '$commit': unknown revision or path not in the working tree\./) { + } elsif ($lines[0] =~ /^fatal: ambiguous argument '$commit': unknown revision or path not in the working tree\./ || + $lines[0] =~ /^fatal: bad object $commit/) { $id = undef; } else { $id = substr($lines[0], 0, 12); @@ -2209,7 +2358,16 @@ sub report { splice(@lines, 1, 1); $output = join("\n", @lines); } - $output = (split('\n', $output))[0] . "\n" if ($terse); + + if ($terse) { + $output = (split('\n', $output))[0] . "\n"; + } + + if ($verbose && exists($verbose_messages{$type}) && + !exists($verbose_emitted{$type})) { + $output .= $verbose_messages{$type} . "\n\n"; + $verbose_emitted{$type} = 1; + } push(our @report, $output); @@ -2494,6 +2652,8 @@ sub process { my $reported_maintainer_file = 1; # ODP has no MAINTAINERS file my $non_utf8_charset = 0; + my $last_git_commit_id_linenr = -1; + my $last_blank_line = 0; my $last_coalesced_string_linenr = -1; @@ -2816,10 +2976,10 @@ sub process { my ($email_name, $email_comment, $email_address, $comment1) = parse_email($ctx); my ($author_name, $author_comment, $author_address, $comment2) = parse_email($author); - if ($email_address eq $author_address && $email_name eq $author_name) { + if (lc $email_address eq lc $author_address && $email_name eq $author_name) { $author_sob = $ctx; $authorsignoff = 2; - } elsif ($email_address eq $author_address) { + } elsif (lc $email_address eq lc $author_address) { $author_sob = $ctx; $authorsignoff = 3; } elsif ($email_name eq $author_name) { @@ -3003,17 +3163,73 @@ sub process { if ($sign_off =~ /^co-developed-by:$/i) { if ($email eq $author) { WARN("BAD_SIGN_OFF", - "Co-developed-by: should not be used to attribute nominal patch author '$author'\n" . "$here\n" . $rawline); + "Co-developed-by: should not be used to attribute nominal patch author '$author'\n" . $herecurr); } if (!defined $lines[$linenr]) { WARN("BAD_SIGN_OFF", - "Co-developed-by: must be immediately followed by Signed-off-by:\n" . "$here\n" . $rawline); - } elsif ($rawlines[$linenr] !~ /^\s*signed-off-by:\s*(.*)/i) { + "Co-developed-by: must be immediately followed by Signed-off-by:\n" . $herecurr); + } elsif ($rawlines[$linenr] !~ /^signed-off-by:\s*(.*)/i) { WARN("BAD_SIGN_OFF", - "Co-developed-by: must be immediately followed by Signed-off-by:\n" . "$here\n" . $rawline . "\n" .$rawlines[$linenr]); + "Co-developed-by: must be immediately followed by Signed-off-by:\n" . $herecurr . $rawlines[$linenr] . "\n"); } elsif ($1 ne $email) { WARN("BAD_SIGN_OFF", - "Co-developed-by and Signed-off-by: name/email do not match \n" . "$here\n" . $rawline . "\n" .$rawlines[$linenr]); + "Co-developed-by and Signed-off-by: name/email do not match\n" . $herecurr . $rawlines[$linenr] . "\n"); + } + } + +# check if Reported-by: is followed by a Closes: tag + if ($sign_off =~ /^reported(?:|-and-tested)-by:$/i) { + if (!defined $lines[$linenr]) { + WARN("BAD_REPORTED_BY_LINK", + "Reported-by: should be immediately followed by Closes: with a URL to the report\n" . $herecurr . "\n"); + } elsif ($rawlines[$linenr] !~ /^closes:\s*/i) { + WARN("BAD_REPORTED_BY_LINK", + "Reported-by: should be immediately followed by Closes: with a URL to the report\n" . $herecurr . $rawlines[$linenr] . "\n"); + } + } + } + + +# Check Fixes: styles is correct + if (!$in_header_lines && + $line =~ /^\s*fixes:?\s*(?:commit\s*)?[0-9a-f]{5,}\b/i) { + my $orig_commit = ""; + my $id = "0123456789ab"; + my $title = "commit title"; + my $tag_case = 1; + my $tag_space = 1; + my $id_length = 1; + my $id_case = 1; + my $title_has_quotes = 0; + + if ($line =~ /(\s*fixes:?)\s+([0-9a-f]{5,})\s+($balanced_parens)/i) { + my $tag = $1; + $orig_commit = $2; + $title = $3; + + $tag_case = 0 if $tag eq "Fixes:"; + $tag_space = 0 if ($line =~ /^fixes:? [0-9a-f]{5,} ($balanced_parens)/i); + + $id_length = 0 if ($orig_commit =~ /^[0-9a-f]{12}$/i); + $id_case = 0 if ($orig_commit !~ /[A-F]/); + + # Always strip leading/trailing parens then double quotes if existing + $title = substr($title, 1, -1); + if ($title =~ /^".*"$/) { + $title = substr($title, 1, -1); + $title_has_quotes = 1; + } + } + + my ($cid, $ctitle) = git_commit_info($orig_commit, $id, + $title); + + if ($ctitle ne $title || $tag_case || $tag_space || + $id_length || $id_case || !$title_has_quotes) { + if (WARN("BAD_FIXES_TAG", + "Please use correct Fixes: style 'Fixes: <12 chars of sha1> (\"<title line>\")' - ie: 'Fixes: $cid (\"$ctitle\")'\n" . $herecurr) && + $fix) { + $fixed[$fixlinenr] = "Fixes: $cid (\"$ctitle\")"; } } } @@ -3051,13 +3267,13 @@ sub process { length($line) > 75 && !($line =~ /^\s*[a-zA-Z0-9_\/\.]+\s+\|\s+\d+/ || # file delta changes - $line =~ /^\s*(?:[\w\.\-]+\/)++[\w\.\-]+:/ || + $line =~ /^\s*(?:[\w\.\-\+]*\/)++[\w\.\-\+]+:/ || # filename then : - $line =~ /^\s*(?:Fixes:|Link:|$signature_tags)/i || - # A Fixes: or Link: line or signature tag line + $line =~ /^\s*(?:Fixes:|$link_tags_search|$signature_tags)/i || + # A Fixes:, link or signature tag line $commit_log_possible_stack_dump)) { WARN("COMMIT_LOG_LONG_LINE", - "Possible unwrapped commit description (prefer a maximum 75 chars per line)\n" . $herecurr); + "Prefer a maximum 75 chars per line (possible unwrapped commit description?)\n" . $herecurr); $commit_log_long_line = 1; } @@ -3067,6 +3283,29 @@ sub process { $commit_log_possible_stack_dump = 0; } +# Check for odd tags before a URI/URL + if ($in_commit_log && + $line =~ /^\s*(\w+:)\s*http/ && $1 !~ /^$link_tags_search$/) { + if ($1 =~ /^v(?:ersion)?\d+/i) { + WARN("COMMIT_LOG_VERSIONING", + "Patch version information should be after the --- line\n" . $herecurr); + } else { + WARN("COMMIT_LOG_USE_LINK", + "Unknown link reference '$1', use $link_tags_print instead\n" . $herecurr); + } + } + +# Check for misuse of the link tags + if ($in_commit_log && + $line =~ /^\s*(\w+:)\s*(\S+)/) { + my $tag = $1; + my $value = $2; + if ($tag =~ /^$link_tags_search$/ && $value !~ m{^https?://}) { + WARN("COMMIT_LOG_WRONG_LINK", + "'$tag' should be followed by a public http(s) link\n" . $herecurr); + } + } + # Check for lines starting with a # if ($in_commit_log && $line =~ /^#/) { if (WARN("COMMIT_COMMENT_SYMBOL", @@ -3077,10 +3316,20 @@ sub process { } # Check for git id commit length and improperly formed commit descriptions - if ($in_commit_log && !$commit_log_possible_stack_dump && +# A correctly formed commit description is: +# commit <SHA-1 hash length 12+ chars> ("Complete commit subject") +# with the commit subject '("' prefix and '")' suffix +# This is a fairly compilicated block as it tests for what appears to be +# bare SHA-1 hash with minimum length of 5. It also avoids several types of +# possible SHA-1 matches. +# A commit match can span multiple lines so this block attempts to find a +# complete typical commit on a maximum of 3 lines + if ($perl_version_ok && + $in_commit_log && !$commit_log_possible_stack_dump && $line !~ /^\s*(?:Link|Patchwork|http|https|BugLink|base-commit):/i && $line !~ /^This reverts commit [0-9a-f]{7,40}/ && - ($line =~ /\bcommit\s+[0-9a-f]{5,}\b/i || + (($line =~ /\bcommit\s+[0-9a-f]{5,}\b/i || + ($line =~ /\bcommit\s*$/i && defined($rawlines[$linenr]) && $rawlines[$linenr] =~ /^\s*[0-9a-f]{5,}\b/i)) || ($line =~ /(?:\s|^)[0-9a-f]{12,40}(?:[\s"'\(\[]|$)/i && $line !~ /[\<\[][0-9a-f]{12,40}[\>\]]/i && $line !~ /\bfixes:\s*[0-9a-f]{12,40}/i))) { @@ -3090,49 +3339,62 @@ sub process { my $long = 0; my $case = 1; my $space = 1; - my $hasdesc = 0; - my $hasparens = 0; my $id = '0123456789ab'; my $orig_desc = "commit description"; my $description = ""; + my $herectx = $herecurr; + my $has_parens = 0; + my $has_quotes = 0; + + my $input = $line; + if ($line =~ /(?:\bcommit\s+[0-9a-f]{5,}|\bcommit\s*$)/i) { + for (my $n = 0; $n < 2; $n++) { + if ($input =~ /\bcommit\s+[0-9a-f]{5,}\s*($balanced_parens)/i) { + $orig_desc = $1; + $has_parens = 1; + # Always strip leading/trailing parens then double quotes if existing + $orig_desc = substr($orig_desc, 1, -1); + if ($orig_desc =~ /^".*"$/) { + $orig_desc = substr($orig_desc, 1, -1); + $has_quotes = 1; + } + last; + } + last if ($#lines < $linenr + $n); + $input .= " " . trim($rawlines[$linenr + $n]); + $herectx .= "$rawlines[$linenr + $n]\n"; + } + $herectx = $herecurr if (!$has_parens); + } - if ($line =~ /\b(c)ommit\s+([0-9a-f]{5,})\b/i) { + if ($input =~ /\b(c)ommit\s+([0-9a-f]{5,})\b/i) { $init_char = $1; $orig_commit = lc($2); - } elsif ($line =~ /\b([0-9a-f]{12,40})\b/i) { + $short = 0 if ($input =~ /\bcommit\s+[0-9a-f]{12,40}/i); + $long = 1 if ($input =~ /\bcommit\s+[0-9a-f]{41,}/i); + $space = 0 if ($input =~ /\bcommit [0-9a-f]/i); + $case = 0 if ($input =~ /\b[Cc]ommit\s+[0-9a-f]{5,40}[^A-F]/); + } elsif ($input =~ /\b([0-9a-f]{12,40})\b/i) { $orig_commit = lc($1); } - $short = 0 if ($line =~ /\bcommit\s+[0-9a-f]{12,40}/i); - $long = 1 if ($line =~ /\bcommit\s+[0-9a-f]{41,}/i); - $space = 0 if ($line =~ /\bcommit [0-9a-f]/i); - $case = 0 if ($line =~ /\b[Cc]ommit\s+[0-9a-f]{5,40}[^A-F]/); - if ($line =~ /\bcommit\s+[0-9a-f]{5,}\s+\("([^"]+)"\)/i) { - $orig_desc = $1; - $hasparens = 1; - } elsif ($line =~ /\bcommit\s+[0-9a-f]{5,}\s*$/i && - defined $rawlines[$linenr] && - $rawlines[$linenr] =~ /^\s*\("([^"]+)"\)/) { - $orig_desc = $1; - $hasparens = 1; - } elsif ($line =~ /\bcommit\s+[0-9a-f]{5,}\s+\("[^"]+$/i && - defined $rawlines[$linenr] && - $rawlines[$linenr] =~ /^\s*[^"]+"\)/) { - $line =~ /\bcommit\s+[0-9a-f]{5,}\s+\("([^"]+)$/i; - $orig_desc = $1; - $rawlines[$linenr] =~ /^\s*([^"]+)"\)/; - $orig_desc .= " " . $1; - $hasparens = 1; - } - ($id, $description) = git_commit_info($orig_commit, $id, $orig_desc); if (defined($id) && - ($short || $long || $space || $case || ($orig_desc ne $description) || !$hasparens)) { + ($short || $long || $space || $case || ($orig_desc ne $description) || !$has_quotes) && + $last_git_commit_id_linenr != $linenr - 1) { ERROR("GIT_COMMIT_ID", - "Please use git commit description style 'commit <12+ chars of sha1> (\"<title line>\")' - ie: '${init_char}ommit $id (\"$description\")'\n" . $herecurr); + "Please use git commit description style 'commit <12+ chars of sha1> (\"<title line>\")' - ie: '${init_char}ommit $id (\"$description\")'\n" . $herectx); } + #don't report the next line if this line ends in commit and the sha1 hash is the next line + $last_git_commit_id_linenr = $linenr if ($line =~ /\bcommit\s*$/i); + } + +# Check for mailing list archives other than lore.kernel.org + if ($rawline =~ m{http.*\b$obsolete_archives}) { + WARN("PREFER_LORE_ARCHIVE", + "Use lore.kernel.org archive links when possible - see https://lore.kernel.org/lists.html\n" . $herecurr); } # Check for added, moved or deleted files @@ -3152,7 +3414,7 @@ sub process { ($line =~ /^new file mode\s*\d+\s*$/) && ($realfile =~ m@^Documentation/devicetree/bindings/.*\.txt$@)) { WARN("DT_SCHEMA_BINDING_PATCH", - "DT bindings should be in DT schema format. See: Documentation/devicetree/writing-schema.rst\n"); + "DT bindings should be in DT schema format. See: Documentation/devicetree/bindings/writing-schema.rst\n"); } # Check for wrappage within a valid hunk of the file @@ -3341,47 +3603,47 @@ sub process { # Kconfig supports named choices), so use a word boundary # (\b) rather than a whitespace character (\s) $line =~ /^\+\s*(?:config|menuconfig|choice)\b/) { - my $length = 0; - my $cnt = $realcnt; - my $ln = $linenr + 1; - my $f; - my $is_start = 0; - my $is_end = 0; - for (; $cnt > 0 && defined $lines[$ln - 1]; $ln++) { - $f = $lines[$ln - 1]; - $cnt-- if ($lines[$ln - 1] !~ /^-/); - $is_end = $lines[$ln - 1] =~ /^\+/; + my $ln = $linenr; + my $needs_help = 0; + my $has_help = 0; + my $help_length = 0; + while (defined $lines[$ln]) { + my $f = $lines[$ln++]; next if ($f =~ /^-/); - last if (!$file && $f =~ /^\@\@/); + last if ($f !~ /^[\+ ]/); # !patch context - if ($lines[$ln - 1] =~ /^\+\s*(?:bool|tristate|prompt)\s*["']/) { - $is_start = 1; - } elsif ($lines[$ln - 1] =~ /^\+\s*(?:---)?help(?:---)?$/) { - $length = -1; + if ($f =~ /^\+\s*(?:bool|tristate|prompt)\s*["']/) { + $needs_help = 1; + next; + } + if ($f =~ /^\+\s*help\s*$/) { + $has_help = 1; + next; } - $f =~ s/^.//; - $f =~ s/#.*//; - $f =~ s/^\s+//; - next if ($f =~ /^$/); + $f =~ s/^.//; # strip patch context [+ ] + $f =~ s/#.*//; # strip # directives + $f =~ s/^\s+//; # strip leading blanks + next if ($f =~ /^$/); # skip blank lines + # At the end of this Kconfig block: # This only checks context lines in the patch # and so hopefully shouldn't trigger false # positives, even though some of these are # common words in help texts - if ($f =~ /^\s*(?:config|menuconfig|choice|endchoice| - if|endif|menu|endmenu|source)\b/x) { - $is_end = 1; + if ($f =~ /^(?:config|menuconfig|choice|endchoice| + if|endif|menu|endmenu|source)\b/x) { last; } - $length++; + $help_length++ if ($has_help); } - if ($is_start && $is_end && $length < $min_conf_desc_length) { + if ($needs_help && + $help_length < $min_conf_desc_length) { + my $stat_real = get_stat_real($linenr, $ln - 1); WARN("CONFIG_DESCRIPTION", - "please write a paragraph that describes the config symbol fully\n" . $herecurr); + "please write a help paragraph that fully describes the config symbol\n" . "$here\n$stat_real\n"); } - #print "is_start<$is_start> is_end<$is_end> length<$length>\n"; } # check MAINTAINERS entries @@ -3477,7 +3739,7 @@ sub process { my $comment = ""; if ($realfile =~ /\.(h|s|S)$/) { $comment = '/*'; - } elsif ($realfile =~ /\.(c|dts|dtsi)$/) { + } elsif ($realfile =~ /\.(c|rs|dts|dtsi)$/) { $comment = '//'; } elsif (($checklicenseline == 2) || $realfile =~ /\.(sh|pl|py|awk|tc|yaml)$/) { $comment = '#'; @@ -3504,7 +3766,7 @@ sub process { "'$spdx_license' is not supported in LICENSES/...\n" . $herecurr); } if ($realfile =~ m@^Documentation/devicetree/bindings/@ && - not $spdx_license =~ /GPL-2\.0.*BSD-2-Clause/) { + $spdx_license !~ /GPL-2\.0(?:-only)? OR BSD-2-Clause/) { my $msg_level = \&WARN; $msg_level = \&CHK if ($file); if (&{$msg_level}("SPDX_LICENSE_TAG", @@ -3514,18 +3776,23 @@ sub process { $fixed[$fixlinenr] =~ s/SPDX-License-Identifier: .*/SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)/; } } + if ($realfile =~ m@^include/dt-bindings/@ && + $spdx_license !~ /GPL-2\.0(?:-only)? OR \S+/) { + WARN("SPDX_LICENSE_TAG", + "DT binding headers should be licensed (GPL-2.0-only OR .*)\n" . $herecurr); + } } } } # check for embedded filenames - if ($rawline =~ /^\+.*\Q$realfile\E/) { + if ($rawline =~ /^\+.*\b\Q$realfile\E\b/) { WARN("EMBEDDED_FILENAME", "It's generally not useful to have the filename in the file\n" . $herecurr); } # check we are in a valid source file if not then ignore this hunk - next if ($realfile !~ /\.(h|c|s|S|sh|dtsi|dts)$/); + next if ($realfile !~ /\.(h|c|rs|s|S|sh|dtsi|dts)$/); # check for using SPDX-License-Identifier on the wrong line number if ($realline != $checklicenseline && @@ -3612,7 +3879,7 @@ sub process { if ($realfile =~ /\.S$/ && $line =~ /^\+\s*(?:[A-Z]+_)?SYM_[A-Z]+_(?:START|END)(?:_[A-Z_]+)?\s*\(\s*\.L/) { WARN("AVOID_L_PREFIX", - "Avoid using '.L' prefixed local symbol names for denoting a range of code via 'SYM_*_START/END' annotations; see Documentation/asm-annotations.rst\n" . $herecurr); + "Avoid using '.L' prefixed local symbol names for denoting a range of code via 'SYM_*_START/END' annotations; see Documentation/core-api/asm-annotations.rst\n" . $herecurr); } # check we are in a valid source file C or perl if not then ignore this hunk @@ -3788,7 +4055,7 @@ sub process { if ($prevline =~ /^[\+ ]};?\s*$/ && $line =~ /^\+/ && !($line =~ /^\+\s*$/ || - $line =~ /^\+\s*EXPORT_SYMBOL/ || + $line =~ /^\+\s*(?:EXPORT_SYMBOL|early_param|ALLOW_ERROR_INJECTION)/ || $line =~ /^\+\s*MODULE_/i || $line =~ /^\+\s*\#\s*(?:end|elif|else)/ || $line =~ /^\+[a-z_]*init/ || @@ -4335,6 +4602,7 @@ sub process { # XXX(foo); # EXPORT_SYMBOL(something_foo); my $name = $1; + $name =~ s/^\s*($Ident).*/$1/; if ($stat =~ /^(?:.\s*}\s*\n)?.([A-Z_]+)\s*\(\s*($Ident)/ && $name =~ /^${Ident}_$2/) { #print "FOO C name<$name>\n"; @@ -4555,12 +4823,12 @@ sub process { } } -# avoid BUG() or BUG_ON() - if ($line =~ /\b(?:BUG|BUG_ON)\b/) { +# do not use BUG() or variants + if ($line =~ /\b(?!AA_|BUILD_|DCCP_|IDA_|KVM_|RWLOCK_|snd_|SPIN_)(?:[a-zA-Z_]*_)?BUG(?:_ON)?(?:_[A-Z_]+)?\s*\(/) { my $msg_level = \&WARN; $msg_level = \&CHK if ($file); &{$msg_level}("AVOID_BUG", - "Avoid crashing the kernel - try using WARN_ON & recovery code rather than BUG() or BUG_ON()\n" . $herecurr); + "Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants\n" . $herecurr); } # avoid LINUX_VERSION_CODE @@ -4781,7 +5049,7 @@ sub process { if|for|while|switch|return|case| volatile|__volatile__| __attribute__|format|__extension__| - asm|__asm__)$/x) + asm|__asm__|scoped_guard)$/x) { # cpp #define statements have non-optional spaces, ie # if there is a space between the name and the open @@ -5268,9 +5536,13 @@ sub process { } } -#goto labels aren't indented, allow a single space however - if ($line=~/^.\s+[A-Za-z\d_]+:(?![0-9]+)/ and - !($line=~/^. [A-Za-z\d_]+:/) and !($line=~/^.\s+default:/)) { +# check that goto labels aren't indented (allow a single space indentation) +# and ignore bitfield definitions like foo:1 +# Strictly, labels can have whitespace after the identifier and before the : +# but this is not allowed here as many ?: uses would appear to be labels + if ($sline =~ /^.\s+[A-Za-z_][A-Za-z\d_]*:(?!\s*\d+)/ && + $sline !~ /^. [A-Za-z\d_][A-Za-z\d_]*:/ && + $sline !~ /^.\s+default:/) { if (WARN("INDENTED_LABEL", "labels should not be indented\n" . $herecurr) && $fix) { @@ -5365,7 +5637,7 @@ sub process { # Return of what appears to be an errno should normally be negative if ($sline =~ /\breturn(?:\s*\(+\s*|\s+)(E[A-Z]+)(?:\s*\)+\s*|\s*)[;:,]/) { my $name = $1; - if ($name ne 'EOF' && $name ne 'ERROR') { + if ($name ne 'EOF' && $name ne 'ERROR' && $name !~ /^EPOLL/) { WARN("USE_NEGATIVE_ERRNO", "return of an errno should typically be negative (ie: return -$1)\n" . $herecurr); } @@ -5408,6 +5680,7 @@ sub process { defined($stat) && defined($cond) && $line =~ /\b(?:if|while|for)\s*\(/ && $line !~ /^.\s*#/) { my ($s, $c) = ($stat, $cond); + my $fixed_assign_in_if = 0; if ($c =~ /\bif\s*\(.*[^<>!=]=[^=].*/s) { if (ERROR("ASSIGN_IN_IF", @@ -5432,6 +5705,7 @@ sub process { $newline .= ')'; $newline .= " {" if (defined($brace)); fix_insert_line($fixlinenr + 1, $newline); + $fixed_assign_in_if = 1; } } } @@ -5455,8 +5729,20 @@ sub process { $stat_real = "[...]\n$stat_real"; } - ERROR("TRAILING_STATEMENTS", - "trailing statements should be on next line\n" . $herecurr . $stat_real); + if (ERROR("TRAILING_STATEMENTS", + "trailing statements should be on next line\n" . $herecurr . $stat_real) && + !$fixed_assign_in_if && + $cond_lines == 0 && + $fix && $perl_version_ok && + $fixed[$fixlinenr] =~ /^\+(\s*)((?:if|while|for)\s*$balanced_parens)\s*(.*)$/) { + my $indent = $1; + my $test = $2; + my $rest = rtrim($4); + if ($rest =~ /;$/) { + $fixed[$fixlinenr] = "\+$indent$test"; + fix_insert_line($fixlinenr + 1, "$indent\t$rest"); + } + } } } @@ -5569,12 +5855,17 @@ sub process { $var !~ /\bSCN[diux]16/ && $var !~ /\bSCN[diux]32/ && $var !~ /\bSCN[diux]64/ && +#Ignore onnxruntime CamelCase usage + $var !~ /\bOrt*/ && + $var !~ /\bort_api->*/ && +#Ignore ETHTOOL_LINK_MODE_<foo> variants + $var !~ /^ETHTOOL_LINK_MODE_/ && #Ignore SI style variants like nS, mV and dB #(ie: max_uV, regulator_min_uA_show, RANGE_mA_VALUE) $var !~ /^(?:[a-z0-9_]*|[A-Z0-9_]*)?_?[a-z][A-Z](?:_[a-z0-9_]+|_[A-Z0-9_]+)?$/ && #Ignore some three character SI units explicitly, like MiB and KHz $var !~ /^(?:[a-z_]*?)_?(?:[KMGT]iB|[KMGT]?Hz)(?:_[a-z_]+)?$/) { - while ($var =~ m{($Ident)}g) { + while ($var =~ m{\b($Ident)}g) { my $word = $1; next if ($word !~ /[A-Z][a-z]|[a-z][A-Z]/); if ($check) { @@ -5704,6 +5995,7 @@ sub process { $dstat !~ /$exceptions/ && $dstat !~ /^\.$Ident\s*=/ && # .foo = $dstat !~ /^(?:\#\s*$Ident|\#\s*$Constant)\s*$/ && # stringification #foo + $dstat !~ /^case\b/ && # case ... $dstat !~ /^do\s*$Constant\s*while\s*$Constant;?$/ && # do {...} while (...); // do {...} while (...) $dstat !~ /^while\s*$Constant\s*$Constant\s*$/ && # while (...) {...} $dstat !~ /^for\s*$Constant$/ && # for (...) @@ -5747,7 +6039,7 @@ sub process { next if ($arg =~ /\.\.\./); next if ($arg =~ /^type$/i); my $tmp_stmt = $define_stmt; - $tmp_stmt =~ s/\b(sizeof|typeof|__typeof__|__builtin\w+|typecheck\s*\(\s*$Type\s*,|\#+)\s*\(*\s*$arg\s*\)*\b//g; + $tmp_stmt =~ s/\b(__must_be_array|offsetof|sizeof|sizeof_field|__stringify|typeof|__typeof__|__builtin\w+|typecheck\s*\(\s*$Type\s*,|\#+)\s*\(*\s*$arg\s*\)*\b//g; $tmp_stmt =~ s/\#+\s*$arg\b//g; $tmp_stmt =~ s/\b$arg\s*\#\#//g; my $use_cnt = () = $tmp_stmt =~ /\b$arg\b/g; @@ -5776,6 +6068,9 @@ sub process { # check for line continuations outside of #defines, preprocessor #, and asm + } elsif ($realfile =~ m@/vmlinux.lds.h$@) { + $line =~ s/(\w+)/$maybe_linker_symbol{$1}++/ge; + #print "REAL: $realfile\nln: $line\nkeys:", sort keys %maybe_linker_symbol; } else { if ($prevline !~ /^..*\\$/ && $line !~ /^\+\s*\#.*\\$/ && # preprocessor @@ -6046,7 +6341,8 @@ sub process { } # concatenated string without spaces between elements - if ($line =~ /$String[A-Za-z0-9_]/ || $line =~ /[A-Za-z0-9_]$String/) { + if ($line =~ /$String[A-Z_]/ || + ($line =~ /([A-Za-z0-9_]+)$String/ && $1 !~ /^[Lu]$/)) { if (CHK("CONCATENATED_STRING", "Concatenated strings should use spaces between elements\n" . $herecurr) && $fix) { @@ -6059,7 +6355,7 @@ sub process { } # uncoalesced string fragments - if ($line =~ /$String\s*"/) { + if ($line =~ /$String\s*[Lu]?"/) { if (WARN("STRING_FRAGMENTS", "Consecutive strings are generally better as a single string\n" . $herecurr) && $fix) { @@ -6617,9 +6913,11 @@ sub process { $specifier = $1; $extension = $2; $qualifier = $3; - if ($extension !~ /[SsBKRraEehMmIiUDdgVCbGNOxtf]/ || + if ($extension !~ /[4SsBKRraEehMmIiUDdgVCbGNOxtf]/ || ($extension eq "f" && - defined $qualifier && $qualifier !~ /^w/)) { + defined $qualifier && $qualifier !~ /^w/) || + ($extension eq "4" && + defined $qualifier && $qualifier !~ /^cc/)) { $bad_specifier = $specifier; last; } @@ -6633,15 +6931,19 @@ sub process { } if ($bad_specifier ne "") { my $stat_real = get_stat_real($linenr, $lc); + my $msg_level = \&WARN; my $ext_type = "Invalid"; my $use = ""; if ($bad_specifier =~ /p[Ff]/) { $use = " - use %pS instead"; $use =~ s/pS/ps/ if ($bad_specifier =~ /pf/); + } elsif ($bad_specifier =~ /pA/) { + $use = " - '%pA' is only intended to be used from Rust code"; + $msg_level = \&ERROR; } - WARN("VSPRINTF_POINTER_EXTENSION", - "$ext_type vsprintf pointer extension '$bad_specifier'$use\n" . "$here\n$stat_real\n"); + &{$msg_level}("VSPRINTF_POINTER_EXTENSION", + "$ext_type vsprintf pointer extension '$bad_specifier'$use\n" . "$here\n$stat_real\n"); } } } @@ -6706,12 +7008,43 @@ sub process { # } # } +# strcpy uses that should likely be strscpy + if ($line =~ /\bstrcpy\s*\(/) { + WARN("STRCPY", + "Prefer strscpy over strcpy - see: https://github.com/KSPP/linux/issues/88\n" . $herecurr); + } + # strlcpy uses that should likely be strscpy if ($line =~ /\bstrlcpy\s*\(/) { WARN("STRLCPY", - "Prefer strscpy over strlcpy - see: https://lore.kernel.org/r/CAHk-=wgfRnXz0W3D37d01q3JFkr_i_uTL=V6A6G1oUZcprmknw\@mail.gmail.com/\n" . $herecurr); + "Prefer strscpy over strlcpy - see: https://github.com/KSPP/linux/issues/89\n" . $herecurr); + } + +# strncpy uses that should likely be strscpy or strscpy_pad + if ($line =~ /\bstrncpy\s*\(/) { + WARN("STRNCPY", + "Prefer strscpy, strscpy_pad, or __nonstring over strncpy - see: https://github.com/KSPP/linux/issues/90\n" . $herecurr); + } + +# ethtool_sprintf uses that should likely be ethtool_puts + if ($line =~ /\bethtool_sprintf\s*\(\s*$FuncArg\s*,\s*$FuncArg\s*\)/) { + if (WARN("PREFER_ETHTOOL_PUTS", + "Prefer ethtool_puts over ethtool_sprintf with only two arguments\n" . $herecurr) && + $fix) { + $fixed[$fixlinenr] =~ s/\bethtool_sprintf\s*\(\s*($FuncArg)\s*,\s*($FuncArg)/ethtool_puts($1, $7)/; + } } + # use $rawline because $line loses %s via sanitization and thus we can't match against it. + if ($rawline =~ /\bethtool_sprintf\s*\(\s*$FuncArg\s*,\s*\"\%s\"\s*,\s*$FuncArg\s*\)/) { + if (WARN("PREFER_ETHTOOL_PUTS", + "Prefer ethtool_puts over ethtool_sprintf with standalone \"%s\" specifier\n" . $herecurr) && + $fix) { + $fixed[$fixlinenr] =~ s/\bethtool_sprintf\s*\(\s*($FuncArg)\s*,\s*"\%s"\s*,\s*($FuncArg)/ethtool_puts($1, $7)/; + } + } + + # typecasts on min/max could be min_t/max_t if ($perl_version_ok && defined $stat && @@ -6817,6 +7150,21 @@ sub process { } } elsif ($realfile =~ /\.c$/ && defined $stat && + $stat =~ /^\+extern struct\s+(\w+)\s+(\w+)\[\];/) + { + my ($st_type, $st_name) = ($1, $2); + + for my $s (keys %maybe_linker_symbol) { + #print "Linker symbol? $st_name : $s\n"; + goto LIKELY_LINKER_SYMBOL + if $st_name =~ /$s/; + } + WARN("AVOID_EXTERNS", + "found a file-scoped extern type:$st_type name:$st_name in .c file\n" + . "is this a linker symbol ?\n" . $herecurr); + LIKELY_LINKER_SYMBOL: + + } elsif ($realfile =~ /\.c$/ && defined $stat && $stat =~ /^.\s*extern\s+/) { WARN("AVOID_EXTERNS", @@ -6884,14 +7232,16 @@ sub process { "Prefer $3(sizeof(*$1)...) over $3($4...)\n" . $herecurr); } -# check for k[mz]alloc with multiplies that could be kmalloc_array/kcalloc +# check for (kv|k)[mz]alloc with multiplies that could be kmalloc_array/kvmalloc_array/kvcalloc/kcalloc if ($perl_version_ok && defined $stat && - $stat =~ /^\+\s*($Lval)\s*\=\s*(?:$balanced_parens)?\s*(k[mz]alloc)\s*\(\s*($FuncArg)\s*\*\s*($FuncArg)\s*,/) { + $stat =~ /^\+\s*($Lval)\s*\=\s*(?:$balanced_parens)?\s*((?:kv|k)[mz]alloc)\s*\(\s*($FuncArg)\s*\*\s*($FuncArg)\s*,/) { my $oldfunc = $3; my $a1 = $4; my $a2 = $10; my $newfunc = "kmalloc_array"; + $newfunc = "kvmalloc_array" if ($oldfunc eq "kvmalloc"); + $newfunc = "kvcalloc" if ($oldfunc eq "kvzalloc"); $newfunc = "kcalloc" if ($oldfunc eq "kzalloc"); my $r1 = $a1; my $r2 = $a2; @@ -6908,7 +7258,7 @@ sub process { "Prefer $newfunc over $oldfunc with multiply\n" . $herectx) && $cnt == 1 && $fix) { - $fixed[$fixlinenr] =~ s/\b($Lval)\s*\=\s*(?:$balanced_parens)?\s*(k[mz]alloc)\s*\(\s*($FuncArg)\s*\*\s*($FuncArg)/$1 . ' = ' . "$newfunc(" . trim($r1) . ', ' . trim($r2)/e; + $fixed[$fixlinenr] =~ s/\b($Lval)\s*\=\s*(?:$balanced_parens)?\s*((?:kv|k)[mz]alloc)\s*\(\s*($FuncArg)\s*\*\s*($FuncArg)/$1 . ' = ' . "$newfunc(" . trim($r1) . ', ' . trim($r2)/e; } } } @@ -6922,7 +7272,7 @@ sub process { } # check for alloc argument mismatch - if ($line =~ /\b(kcalloc|kmalloc_array)\s*\(\s*sizeof\b/) { + if ($line =~ /\b((?:devm_)?((?:k|kv)?(calloc|malloc_array)(?:_node)?))\s*\(\s*sizeof\b/) { WARN("ALLOC_ARRAY_ARGS", "$1 uses number as first arg, sizeof is generally wrong\n" . $herecurr); } @@ -7114,6 +7464,27 @@ sub process { "Using $1 should generally have parentheses around the comparison\n" . $herecurr); } +# return sysfs_emit(foo, fmt, ...) fmt without newline + if ($line =~ /\breturn\s+sysfs_emit\s*\(\s*$FuncArg\s*,\s*($String)/ && + substr($rawline, $-[6], $+[6] - $-[6]) !~ /\\n"$/) { + my $offset = $+[6] - 1; + if (WARN("SYSFS_EMIT", + "return sysfs_emit(...) formats should include a terminating newline\n" . $herecurr) && + $fix) { + substr($fixed[$fixlinenr], $offset, 0) = '\\n'; + } + } + +# check for array definition/declarations that should use flexible arrays instead + if ($sline =~ /^[\+ ]\s*\}(?:\s*__packed)?\s*;\s*$/ && + $prevline =~ /^\+\s*(?:\}(?:\s*__packed\s*)?|$Type)\s*$Ident\s*\[\s*(0|1)\s*\]\s*;\s*$/) { + if (ERROR("FLEXIBLE_ARRAY", + "Use C99 flexible arrays - see https://docs.kernel.org/process/deprecated.html#zero-length-and-one-element-arrays\n" . $hereprev) && + $1 == '0' && $fix) { + $fixed[$fixlinenr - 1] =~ s/\[\s*0\s*\]/[]/; + } + } + # nested likely/unlikely calls if ($line =~ /\b(?:(?:un)?likely)\s*\(\s*!?\s*(IS_ERR(?:_OR_NULL|_VALUE)?|WARN)/) { WARN("LIKELY_MISUSE", @@ -7131,6 +7502,30 @@ sub process { } } +# Complain about RCU Tasks Trace used outside of BPF (and of course, RCU). + our $rcu_trace_funcs = qr{(?x: + rcu_read_lock_trace | + rcu_read_lock_trace_held | + rcu_read_unlock_trace | + call_rcu_tasks_trace | + synchronize_rcu_tasks_trace | + rcu_barrier_tasks_trace | + rcu_request_urgent_qs_task + )}; + our $rcu_trace_paths = qr{(?x: + kernel/bpf/ | + include/linux/bpf | + net/bpf/ | + kernel/rcu/ | + include/linux/rcu + )}; + if ($line =~ /\b($rcu_trace_funcs)\s*\(/) { + if ($realfile !~ m{^$rcu_trace_paths}) { + WARN("RCU_TASKS_TRACE", + "use of RCU tasks trace is incorrect outside BPF or core RCU code\n" . $herecurr); + } + } + # check for lockdep_set_novalidate_class if ($line =~ /^.\s*lockdep_set_novalidate_class\s*\(/ || $line =~ /__lockdep_no_validate__\s*\)/ ) { @@ -7272,6 +7667,13 @@ sub process { WARN("MODULE_LICENSE", "unknown module license " . $extracted_string . "\n" . $herecurr); } + if (!$file && $extracted_string eq '"GPL v2"') { + if (WARN("MODULE_LICENSE", + "Prefer \"GPL\" over \"GPL v2\" - see commit bf7fbeeae6db (\"module: Cure the MODULE_LICENSE \"GPL\" vs. \"GPL v2\" bogosity\")\n" . $herecurr) && + $fix) { + $fixed[$fixlinenr] =~ s/\bMODULE_LICENSE\s*\(\s*"GPL v2"\s*\)/MODULE_LICENSE("GPL")/; + } + } } # check for sysctl duplicate constants @@ -7414,4 +7816,4 @@ EOM } } return $clean; -} +}
\ No newline at end of file diff --git a/scripts/ci-checkpatches.sh b/scripts/ci-checkpatches.sh index 383045cd7..4ce077342 100755 --- a/scripts/ci-checkpatches.sh +++ b/scripts/ci-checkpatches.sh @@ -2,22 +2,14 @@ PATCHES=$1 echo "Run checkpatch for ${PATCHES}" -# Generate patches provided with $1. -# In case of force push and range is broken -# validate only the latest commit if it's not merge commit. +# Generate patches provided with $1. If commit range is not available validate +# only the latest commit. if [ "$PATCHES" = "" ]; then git format-patch -1 -M HEAD; - perl ./scripts/checkpatch.pl *.patch; - exit $? +else + git format-patch ${PATCHES} fi -git show --summary HEAD| grep -q '^Merge:'; -if [ $? -ne 0 ]; then - git format-patch -1 -M HEAD; - perl ./scripts/checkpatch.pl *.patch; - exit $? -fi - -git format-patch ${PATCHES} perl ./scripts/checkpatch.pl *.patch; +exit $? diff --git a/scripts/ci/distcheck.sh b/scripts/ci/distcheck.sh index 6cf49f8a7..9d45536f4 100755 --- a/scripts/ci/distcheck.sh +++ b/scripts/ci/distcheck.sh @@ -16,4 +16,4 @@ export CI="true" # Additional configure flags for distcheck export DISTCHECK_CONFIGURE_FLAGS="${CONF}" -make -j $(nproc) distcheck +make distcheck diff --git a/scripts/spelling.txt b/scripts/spelling.txt index 69b5537a7..a22e955c6 100644 --- a/scripts/spelling.txt +++ b/scripts/spelling.txt @@ -23,8 +23,10 @@ absoulte||absolute acccess||access acceess||access accelaration||acceleration +accelearion||acceleration acceleratoin||acceleration accelleration||acceleration +accelrometer||accelerometer accesing||accessing accesnt||accent accessable||accessible @@ -58,11 +60,13 @@ acording||according activete||activate actived||activated actualy||actually +actvie||active acumulating||accumulating acumulative||accumulative acumulator||accumulator acutally||actually adapater||adapter +adderted||asserted addional||additional additionaly||additionally additonal||additional @@ -120,6 +124,7 @@ alue||value ambigious||ambiguous ambigous||ambiguous amoung||among +amount of times||number of times amout||amount amplifer||amplifier amplifyer||amplifier @@ -133,6 +138,7 @@ anniversery||anniversary annoucement||announcement anomolies||anomalies anomoly||anomaly +anonynous||anonymous anway||anyway aplication||application appearence||appearance @@ -151,6 +157,7 @@ aquired||acquired aquisition||acquisition arbitary||arbitrary architechture||architecture +archtecture||architecture arguement||argument arguements||arguments arithmatic||arithmetic @@ -178,7 +185,9 @@ assum||assume assumtpion||assumption asuming||assuming asycronous||asynchronous +asychronous||asynchronous asynchnous||asynchronous +asynchronus||asynchronous asynchromous||asynchronous asymetric||asymmetric asymmeric||asymmetric @@ -230,6 +239,7 @@ baloons||balloons bandwith||bandwidth banlance||balance batery||battery +battey||battery beacuse||because becasue||because becomming||becoming @@ -241,6 +251,7 @@ beter||better betweeen||between bianries||binaries bitmast||bitmask +bitwiedh||bitwidth boardcast||broadcast borad||board boundry||boundary @@ -249,6 +260,7 @@ brigde||bridge broadcase||broadcast broadcat||broadcast bufer||buffer +bufferred||buffered bufufer||buffer cacluated||calculated caculate||calculate @@ -257,6 +269,7 @@ cadidate||candidate cahces||caches calender||calendar calescing||coalescing +calibraiton||calibration calle||called callibration||calibration callled||called @@ -265,7 +278,12 @@ calucate||calculate calulate||calculate cancelation||cancellation cancle||cancel +cant||can't +cant'||can't canot||cannot +cann't||can't +cannnot||cannot +capabiity||capability capabilites||capabilities capabilties||capabilities capabilty||capability @@ -273,9 +291,11 @@ capabitilies||capabilities capablity||capability capatibilities||capabilities capapbilities||capabilities +captuer||capture caputure||capture carefuly||carefully cariage||carriage +casued||caused catagory||category cehck||check challange||challenge @@ -302,6 +322,7 @@ chiled||child chked||checked chnage||change chnages||changes +chnange||change chnnel||channel choosen||chosen chouse||chose @@ -323,11 +344,13 @@ comminucation||communication commited||committed commiting||committing committ||commit +commmand||command commnunication||communication commoditiy||commodity comsume||consume comsumer||consumer comsuming||consuming +comaptible||compatible compability||compatibility compaibility||compatibility comparsion||comparison @@ -348,13 +371,16 @@ compoment||component comppatible||compatible compres||compress compresion||compression +compresser||compressor comression||compression +comsumed||consumed comunicate||communicate comunication||communication conbination||combination conditionaly||conditionally conditon||condition condtion||condition +condtional||conditional conected||connected conector||connector configration||configuration @@ -385,6 +411,7 @@ continious||continuous continous||continuous continously||continuously continueing||continuing +contiuous||continuous contraints||constraints contruct||construct contol||control @@ -407,7 +434,9 @@ cotrol||control cound||could couter||counter coutner||counter +creationg||creating cryptocraphic||cryptographic +cummulative||cumulative cunter||counter curently||currently cylic||cyclic @@ -429,6 +458,7 @@ defferred||deferred definate||definite definately||definitely definiation||definition +definiton||definition defintion||definition defintions||definitions defualt||default @@ -442,6 +472,7 @@ delare||declare delares||declares delaring||declaring delemiter||delimiter +delibrately||deliberately delievered||delivered demodualtor||demodulator demension||dimension @@ -470,7 +501,9 @@ destorys||destroys destroied||destroyed detabase||database deteced||detected +detecion||detection detectt||detect +detroyed||destroyed develope||develop developement||development developped||developed @@ -490,6 +523,7 @@ diferent||different differrence||difference diffrent||different differenciate||differentiate +diffreential||differential diffrentiate||differentiate difinition||definition digial||digital @@ -497,10 +531,12 @@ dimention||dimension dimesions||dimensions diconnected||disconnected disabed||disabled +disasembler||disassembler disble||disable disgest||digest disired||desired dispalying||displaying +dissable||disable diplay||display directon||direction direcly||directly @@ -524,6 +560,7 @@ dissconect||disconnect distiction||distinction divisable||divisible divsiors||divisors +dsiabled||disabled docuentation||documentation documantation||documentation documentaion||documentation @@ -591,10 +628,12 @@ evalute||evaluate evalutes||evaluates evalution||evaluation excecutable||executable +excceed||exceed exceded||exceeded exceds||exceeds exceeed||exceed excellant||excellent +exchnage||exchange execeeded||exceeded execeeds||exceeds exeed||exceed @@ -603,8 +642,11 @@ exeuction||execution existance||existence existant||existent exixt||exist +exsits||exists exlcude||exclude +exlcuding||excluding exlcusive||exclusive +exlusive||exclusive exmaple||example expecially||especially experies||expires @@ -642,17 +684,20 @@ feauture||feature feautures||features fetaure||feature fetaures||features +fetcing||fetching fileystem||filesystem fimrware||firmware fimware||firmware firmare||firmware firmaware||firmware +firtly||firstly firware||firmware firwmare||firmware finanize||finalize findn||find finilizes||finalizes finsih||finish +fliter||filter flusing||flushing folloing||following followign||following @@ -670,6 +715,7 @@ frequence||frequency frequncy||frequency frequancy||frequency frome||from +fronend||frontend fucntion||function fuction||function fuctions||functions @@ -693,6 +739,8 @@ generiously||generously genereate||generate genereted||generated genric||generic +gerenal||general +geting||getting globel||global grabing||grabbing grahical||graphical @@ -711,9 +759,11 @@ hanled||handled happend||happened hardare||hardware harware||hardware +hardward||hardware havind||having heirarchically||hierarchically heirarchy||hierarchy +heirachy||hierarchy helpfull||helpful hearbeat||heartbeat heterogenous||heterogeneous @@ -726,6 +776,7 @@ howver||however hsould||should hypervior||hypervisor hypter||hyper +idel||idle identidier||identifier iligal||illegal illigal||illegal @@ -754,6 +805,7 @@ implmentation||implementation implmenting||implementing incative||inactive incomming||incoming +incompaitiblity||incompatibility incompatabilities||incompatibilities incompatable||incompatible incompatble||incompatible @@ -811,6 +863,7 @@ integrety||integrity integrey||integrity intendet||intended intented||intended +interal||internal interanl||internal interchangable||interchangeable interferring||interfering @@ -862,6 +915,7 @@ iteraions||iterations iternations||iterations itertation||iteration itslef||itself +ivalid||invalid jave||java jeffies||jiffies jumpimng||jumping @@ -916,9 +970,11 @@ matchs||matches mathimatical||mathematical mathimatic||mathematic mathimatics||mathematics +maxmium||maximum maximium||maximum maxium||maximum mechamism||mechanism +mechanim||mechanism meetign||meeting memeory||memory memmber||member @@ -927,6 +983,7 @@ memroy||memory ment||meant mergable||mergeable mesage||message +mesages||messages messags||messages messgaes||messages messsage||message @@ -935,9 +992,12 @@ metdata||metadata micropone||microphone microprocesspr||microprocessor migrateable||migratable +millenium||millennium milliseonds||milliseconds +minimim||minimum minium||minimum minimam||minimum +minimun||minimum miniumum||minimum minumum||minimum misalinged||misaligned @@ -956,6 +1016,7 @@ mmnemonic||mnemonic mnay||many modfiy||modify modifer||modifier +modul||module modulues||modules momery||memory memomry||memory @@ -965,8 +1026,9 @@ monochromo||monochrome monocrome||monochrome mopdule||module mroe||more -multipler||multiplier mulitplied||multiplied +muliple||multiple +multipler||multiplier multidimensionnal||multidimensional multipe||multiple multple||multiple @@ -989,6 +1051,7 @@ negotation||negotiation nerver||never nescessary||necessary nessessary||necessary +none existent||non-existent noticable||noticeable notication||notification notications||notifications @@ -997,7 +1060,9 @@ notifed||notified notity||notify nubmer||number numebr||number +numer||number numner||number +nunber||number obtaion||obtain obusing||abusing occassionally||occasionally @@ -1009,17 +1074,20 @@ occured||occurred occurence||occurrence occure||occurred occuring||occurring +ocurrence||occurrence offser||offset offet||offset offlaod||offload offloded||offloaded offseting||offsetting +oflload||offload omited||omitted omiting||omitting omitt||omit ommiting||omitting ommitted||omitted onself||oneself +onthe||on the ony||only openning||opening operatione||operation @@ -1037,6 +1105,7 @@ oustanding||outstanding overaall||overall overhread||overhead overlaping||overlapping +oveflow||overflow overflw||overflow overlfow||overflow overide||override @@ -1056,6 +1125,7 @@ pakage||package paket||packet pallette||palette paln||plan +palne||plane paramameters||parameters paramaters||parameters paramater||parameter @@ -1085,10 +1155,12 @@ perfomring||performing periperal||peripheral peripherial||peripheral permissons||permissions +permited||permitted peroid||period persistance||persistence persistant||persistent phoneticly||phonetically +plaform||platform plalform||platform platfoem||platform platfrom||platform @@ -1126,15 +1198,19 @@ preprare||prepare pressre||pressure presuambly||presumably previosuly||previously +previsously||previously primative||primitive princliple||principle priorty||priority +priting||printing privilaged||privileged privilage||privilege priviledge||privilege +priviledged||privileged priviledges||privileges privleges||privileges probaly||probably +probabalistic||probabilistic procceed||proceed proccesors||processors procesed||processed @@ -1154,6 +1230,7 @@ programable||programmable programers||programmers programm||program programms||programs +progres||progress progresss||progress prohibitted||prohibited prohibitting||prohibiting @@ -1182,6 +1259,7 @@ purgable||purgeable pwoer||power queing||queuing quering||querying +querrying||querying queus||queues randomally||randomly raoming||roaming @@ -1214,6 +1292,7 @@ refering||referring refernces||references refernnce||reference refrence||reference +regiser||register registed||registered registerd||registered registeration||registration @@ -1254,6 +1333,8 @@ reseting||resetting reseved||reserved reseverd||reserved resizeable||resizable +resonable||reasonable +resotre||restore resouce||resource resouces||resources resoures||resources @@ -1286,11 +1367,14 @@ routins||routines rquest||request runing||running runned||ran +runnnig||running runnning||running runtine||runtime sacrifying||sacrificing safly||safely safty||safety +satify||satisfy +satisifed||satisfied savable||saveable scaleing||scaling scaned||scanned @@ -1328,6 +1412,7 @@ servive||service setts||sets settting||setting shapshot||snapshot +shoft||shift shotdown||shutdown shoud||should shouldnt||shouldn't @@ -1341,14 +1426,18 @@ similiar||similar simlar||similar simliar||similar simpified||simplified +simultaneusly||simultaneously +simultanous||simultaneous singaled||signaled singal||signal singed||signed +slect||select sleeped||slept sliped||slipped softwade||software softwares||software soley||solely +soluation||solution souce||source speach||speech specfic||specific @@ -1380,6 +1469,7 @@ standart||standard standy||standby stardard||standard staticly||statically +statisitcs||statistics statuss||status stoped||stopped stoping||stopping @@ -1398,6 +1488,7 @@ submited||submitted submition||submission succeded||succeeded suceed||succeed +succesfuly||successfully succesfully||successfully succesful||successful successed||succeeded @@ -1413,6 +1504,7 @@ suported||supported suport||support supportet||supported suppored||supported +supporing||supporting supportin||supporting suppoted||supported suppported||supported @@ -1439,6 +1531,8 @@ syfs||sysfs symetric||symmetric synax||syntax synchonized||synchronized +sychronization||synchronization +sychronously||synchronously synchronuously||synchronously syncronize||synchronize syncronized||synchronized @@ -1448,22 +1542,27 @@ syste||system sytem||system sythesis||synthesis taht||that +tained||tainted +tarffic||traffic tansmit||transmit targetted||targeted targetting||targeting taskelt||tasklet teh||the +temeprature||temperature temorary||temporary temproarily||temporarily temperture||temperature -thead||thread +theads||threads therfore||therefore thier||their threds||threads threee||three threshhold||threshold thresold||threshold +throtting||throttling throught||through +tansition||transition trackling||tracking troughput||throughput trys||tries @@ -1476,6 +1575,7 @@ timout||timeout tmis||this toogle||toggle torerable||tolerable +torlence||tolerance traget||target traking||tracking tramsmitted||transmitted @@ -1484,12 +1584,14 @@ tranasction||transaction tranceiver||transceiver tranfer||transfer tranmission||transmission +tranport||transport transcevier||transceiver transciever||transceiver transferd||transferred transfered||transferred transfering||transferring transision||transition +transistioned||transitioned transmittd||transmitted transormed||transformed trasfer||transfer @@ -1503,13 +1605,16 @@ tunning||tuning ture||true tyep||type udpate||update +updtes||updates uesd||used +unknwon||unknown uknown||unknown usccess||success uncommited||uncommitted uncompatible||incompatible unconditionaly||unconditionally undeflow||underflow +undelying||underlying underun||underrun unecessary||unnecessary unexecpted||unexpected @@ -1521,6 +1626,7 @@ unexpexted||unexpected unfortunatelly||unfortunately unifiy||unify uniterrupted||uninterrupted +uninterruptable||uninterruptible unintialized||uninitialized unitialized||uninitialized unkmown||unknown @@ -1537,13 +1643,17 @@ unneccessary||unnecessary unnecesary||unnecessary unneedingly||unnecessarily unnsupported||unsupported +unuspported||unsupported unmached||unmatched unprecise||imprecise +unpriviledged||unprivileged +unpriviliged||unprivileged unregester||unregister unresgister||unregister unrgesiter||unregister unsinged||unsigned unstabel||unstable +unsolicted||unsolicited unsolicitied||unsolicited unsuccessfull||unsuccessful unsuported||unsupported @@ -1553,6 +1663,8 @@ unuseful||useless unvalid||invalid upate||update upsupported||unsupported +upto||up to +useable||usable usefule||useful usefull||useful usege||usage @@ -1574,9 +1686,11 @@ varient||variant vaule||value verbse||verbose veify||verify +verfication||verification veriosn||version verisons||versions verison||version +veritical||vertical verson||version vicefersa||vice-versa virtal||virtual @@ -1586,6 +1700,7 @@ visiters||visitors vitual||virtual vunerable||vulnerable wakeus||wakeups +was't||wasn't wathdog||watchdog wating||waiting wiat||wait @@ -1596,6 +1711,7 @@ whenver||whenever wheter||whether whe||when wierd||weird +wihout||without wiil||will wirte||write withing||within @@ -1607,7 +1723,6 @@ writting||writing wtih||with zombe||zombie zomebie||zombie - # # ODP additions # diff --git a/test/common/odp_cunit_common.c b/test/common/odp_cunit_common.c index a5b9ff1ea..651ae791e 100644 --- a/test/common/odp_cunit_common.c +++ b/test/common/odp_cunit_common.c @@ -708,13 +708,13 @@ int odp_cunit_register(odp_suiteinfo_t testsuites[]) * (hence also helpers options as cunit_common uses the helpers) * Options private to the test calling cunit_common are not parsed here. */ -int odp_cunit_parse_options(int argc, char *argv[]) +int odp_cunit_parse_options(int *argc, char *argv[]) { const char *ctrl_thread_env = getenv("CI_THREAD_TYPE_CONTROL"); const char *env = getenv("CI"); progname = argv[0]; - odph_parse_options(argc, argv); + *argc = odph_parse_options(*argc, argv); /* Check if we need to use control thread */ if (ctrl_thread_env && !strcmp(ctrl_thread_env, "true")) control_thread = true; diff --git a/test/common/odp_cunit_common.h b/test/common/odp_cunit_common.h index 77822ee60..63e95d5fb 100644 --- a/test/common/odp_cunit_common.h +++ b/test/common/odp_cunit_common.h @@ -66,7 +66,7 @@ typedef struct { } test_shared_data_t; /* parse parameters that affect the behaviour of odp_cunit_common */ -int odp_cunit_parse_options(int argc, char *argv[]); +int odp_cunit_parse_options(int *argc, char *argv[]); /* register suites to be run via odp_cunit_run() */ int odp_cunit_register(odp_suiteinfo_t testsuites[]); /* update tests previously registered via odp_cunit_register() */ diff --git a/test/m4/configure.m4 b/test/m4/configure.m4 index 4335476ba..ea05e954f 100644 --- a/test/m4/configure.m4 +++ b/test/m4/configure.m4 @@ -17,9 +17,11 @@ AC_CONFIG_FILES([test/common/Makefile test/miscellaneous/Makefile test/performance/Makefile test/validation/Makefile + test/validation/api/align/Makefile test/validation/api/atomic/Makefile test/validation/api/barrier/Makefile test/validation/api/buffer/Makefile + test/validation/api/byteorder/Makefile test/validation/api/chksum/Makefile test/validation/api/classification/Makefile test/validation/api/comp/Makefile @@ -29,10 +31,12 @@ AC_CONFIG_FILES([test/common/Makefile test/validation/api/errno/Makefile test/validation/api/event/Makefile test/validation/api/hash/Makefile + test/validation/api/hints/Makefile test/validation/api/init/Makefile test/validation/api/ipsec/Makefile test/validation/api/lock/Makefile test/validation/api/Makefile + test/validation/api/ml/Makefile test/validation/api/packet/Makefile test/validation/api/pktio/Makefile test/validation/api/pool/Makefile diff --git a/test/performance/.gitignore b/test/performance/.gitignore index 08a4d5609..46d9e9c2c 100644 --- a/test/performance/.gitignore +++ b/test/performance/.gitignore @@ -20,6 +20,7 @@ odp_mem_perf odp_packet_gen odp_pktio_ordered odp_pktio_perf +odp_pool_latency odp_pool_perf odp_queue_perf odp_random diff --git a/test/performance/Makefile.am b/test/performance/Makefile.am index 7b0adbe09..356e98a2d 100644 --- a/test/performance/Makefile.am +++ b/test/performance/Makefile.am @@ -12,6 +12,7 @@ EXECUTABLES = odp_atomic_perf \ odp_lock_perf \ odp_mem_perf \ odp_pktio_perf \ + odp_pool_latency \ odp_pool_perf \ odp_queue_perf \ odp_stash_perf \ @@ -81,6 +82,7 @@ odp_sched_latency_SOURCES = odp_sched_latency.c odp_sched_pktio_SOURCES = odp_sched_pktio.c odp_scheduling_SOURCES = odp_scheduling.c odp_pktio_perf_SOURCES = odp_pktio_perf.c +odp_pool_latency_SOURCES = odp_pool_latency.c odp_pool_perf_SOURCES = odp_pool_perf.c odp_queue_perf_SOURCES = odp_queue_perf.c odp_random_SOURCES = odp_random.c @@ -121,5 +123,3 @@ clean-local: rm -f $(builddir)/$$f; \ done \ fi - -.NOTPARALLEL: diff --git a/test/performance/bench_common.c b/test/performance/bench_common.c index f838954ab..640889503 100644 --- a/test/performance/bench_common.c +++ b/test/performance/bench_common.c @@ -2,6 +2,8 @@ * Copyright (c) 2023 Nokia */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #include <odp_api.h> #include <odp/helper/odph_api.h> diff --git a/test/performance/bench_common.h b/test/performance/bench_common.h index bd611878d..4b59c941f 100644 --- a/test/performance/bench_common.h +++ b/test/performance/bench_common.h @@ -2,6 +2,8 @@ * Copyright (c) 2023 Nokia */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + #ifndef BENCH_COMMON_H #define BENCH_COMMON_H diff --git a/test/performance/dummy_crc.h b/test/performance/dummy_crc.h index 68928abee..01e6c2433 100644 --- a/test/performance/dummy_crc.h +++ b/test/performance/dummy_crc.h @@ -37,6 +37,8 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +/** @cond _ODP_HIDE_FROM_DOXYGEN_ */ + /** * @file * diff --git a/test/performance/odp_atomic_perf.c b/test/performance/odp_atomic_perf.c index 28217f5d7..e665081a2 100644 --- a/test/performance/odp_atomic_perf.c +++ b/test/performance/odp_atomic_perf.c @@ -5,6 +5,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_atomic_perf.c + * + * Performance test application for atomic operation APIs + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include <stdio.h> #include <string.h> #include <stdint.h> @@ -16,7 +24,7 @@ #include <odp/helper/odph_api.h> /* Default number of test rounds */ -#define NUM_ROUNDS 1000000u +#define NUM_ROUNDS 100000u /* Initial value for atomic variables. Supports up to 2 billion * rounds of 32-bit min and max tests. */ diff --git a/test/performance/odp_bench_buffer.c b/test/performance/odp_bench_buffer.c index 8b041eee0..ce14ec8b3 100644 --- a/test/performance/odp_bench_buffer.c +++ b/test/performance/odp_bench_buffer.c @@ -5,6 +5,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_bench_buffer.c + * + * Microbenchmark application for buffer API functions + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include <odp_api.h> #include <odp/helper/odph_api.h> @@ -26,7 +34,7 @@ #define TEST_REPEAT_COUNT 1000 /** Default number of rounds per test case */ -#define TEST_ROUNDS 1000u +#define TEST_ROUNDS 100u /** Maximum burst size for *_multi operations */ #define TEST_MAX_BURST 64 @@ -81,9 +89,9 @@ typedef struct { /** Array for storing test pool handles */ odp_pool_t pool_tbl[TEST_REPEAT_COUNT]; /** Array for storing test event types */ - odp_event_type_t event_type_tbl[TEST_REPEAT_COUNT]; + odp_event_type_t event_type_tbl[TEST_REPEAT_COUNT * TEST_MAX_BURST]; /** Array for storing test event subtypes */ - odp_event_subtype_t event_subtype_tbl[TEST_REPEAT_COUNT]; + odp_event_subtype_t event_subtype_tbl[TEST_REPEAT_COUNT * TEST_MAX_BURST]; /** CPU mask as string */ char cpumask_str[ODP_CPUMASK_STR_SIZE]; } args_t; @@ -113,14 +121,14 @@ static void allocate_test_buffers(odp_buffer_t buf[], int num) } } -static void alloc_buffers_multi(void) +static void create_buffers(void) { - allocate_test_buffers(gbl_args->buf_tbl, TEST_REPEAT_COUNT * gbl_args->appl.burst_size); + allocate_test_buffers(gbl_args->buf_tbl, TEST_REPEAT_COUNT); } -static void create_buffers(void) +static void create_buffers_multi(void) { - allocate_test_buffers(gbl_args->buf_tbl, TEST_REPEAT_COUNT); + allocate_test_buffers(gbl_args->buf_tbl, TEST_REPEAT_COUNT * gbl_args->appl.burst_size); } static void create_events(void) @@ -176,6 +184,20 @@ static int buffer_from_event(void) return i; } +static int buffer_from_event_multi(void) +{ + odp_buffer_t *buf_tbl = gbl_args->buf_tbl; + odp_event_t *event_tbl = gbl_args->event_tbl; + int burst_size = gbl_args->appl.burst_size; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + odp_buffer_from_event_multi(&buf_tbl[i * burst_size], + &event_tbl[i * burst_size], burst_size); + + return i; +} + static int buffer_to_event(void) { odp_buffer_t *buf_tbl = gbl_args->buf_tbl; @@ -188,6 +210,20 @@ static int buffer_to_event(void) return i; } +static int buffer_to_event_multi(void) +{ + odp_buffer_t *buf_tbl = gbl_args->buf_tbl; + odp_event_t *event_tbl = gbl_args->event_tbl; + int burst_size = gbl_args->appl.burst_size; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + odp_buffer_to_event_multi(&buf_tbl[i * burst_size], + &event_tbl[i * burst_size], burst_size); + + return i; +} + static int buffer_addr(void) { odp_buffer_t *buf_tbl = gbl_args->buf_tbl; @@ -365,6 +401,36 @@ static int event_types(void) return i; } +static int event_types_multi(void) +{ + odp_event_t *event_tbl = gbl_args->event_tbl; + odp_event_type_t *event_type_tbl = gbl_args->event_type_tbl; + odp_event_subtype_t *event_subtype_tbl = gbl_args->event_subtype_tbl; + int burst_size = gbl_args->appl.burst_size; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + odp_event_types_multi(&event_tbl[i * burst_size], + &event_type_tbl[i * burst_size], + &event_subtype_tbl[i * burst_size], burst_size); + + return i; +} + +static int event_types_multi_no_sub(void) +{ + odp_event_t *event_tbl = gbl_args->event_tbl; + odp_event_type_t *event_type_tbl = gbl_args->event_type_tbl; + int burst_size = gbl_args->appl.burst_size; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + odp_event_types_multi(&event_tbl[i * burst_size], + &event_type_tbl[i * burst_size], NULL, burst_size); + + return i; +} + static int event_type_multi(void) { odp_event_t *event_tbl = gbl_args->event_tbl; @@ -379,6 +445,45 @@ static int event_type_multi(void) return ret; } +static int event_pool(void) +{ + odp_event_t *event_tbl = gbl_args->event_tbl; + odp_pool_t *pool_tbl = gbl_args->pool_tbl; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + pool_tbl[i] = odp_event_pool(event_tbl[i]); + + return i; +} + +static int event_user_area(void) +{ + odp_event_t *event_tbl = gbl_args->event_tbl; + void **ptr_tbl = gbl_args->ptr_tbl; + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + ptr_tbl[i] = odp_event_user_area(event_tbl[i]); + + return i; +} + +static int event_user_area_and_flag(void) +{ + odp_event_t *event_tbl = gbl_args->event_tbl; + void **ptr_tbl = gbl_args->ptr_tbl; + int ret = 0; + int flag; + + for (int i = 0; i < TEST_REPEAT_COUNT; i++) { + ptr_tbl[i] = odp_event_user_area_and_flag(event_tbl[i], &flag); + ret += flag; + } + + return ret; +} + static int event_is_valid(void) { odp_event_t *event_tbl = gbl_args->event_tbl; @@ -572,7 +677,9 @@ static void print_info(void) */ bench_info_t test_suite[] = { BENCH_INFO(buffer_from_event, create_events, free_buffers, NULL), + BENCH_INFO(buffer_from_event_multi, create_events_multi, free_buffers_multi, NULL), BENCH_INFO(buffer_to_event, create_buffers, free_buffers, NULL), + BENCH_INFO(buffer_to_event_multi, create_buffers_multi, free_buffers_multi, NULL), BENCH_INFO(buffer_addr, create_buffers, free_buffers, NULL), BENCH_INFO(buffer_size, create_buffers, free_buffers, NULL), BENCH_INFO_COND(buffer_user_area, create_buffers, free_buffers, NULL, check_uarea), @@ -580,14 +687,20 @@ bench_info_t test_suite[] = { BENCH_INFO(buffer_alloc, NULL, free_buffers, NULL), BENCH_INFO(buffer_alloc_multi, NULL, free_buffers_multi, NULL), BENCH_INFO(buffer_free, create_buffers, NULL, NULL), - BENCH_INFO(buffer_free_multi, alloc_buffers_multi, NULL, NULL), + BENCH_INFO(buffer_free_multi, create_buffers_multi, NULL, NULL), BENCH_INFO(buffer_alloc_free, NULL, NULL, NULL), BENCH_INFO(buffer_alloc_free_multi, NULL, NULL, NULL), BENCH_INFO(buffer_is_valid, create_buffers, free_buffers, NULL), BENCH_INFO(event_type, create_events, free_buffers, NULL), - BENCH_INFO(event_subtype, create_buffers, free_buffers, NULL), - BENCH_INFO(event_types, create_buffers, free_buffers, NULL), + BENCH_INFO(event_subtype, create_events, free_buffers, NULL), + BENCH_INFO(event_types, create_events, free_buffers, NULL), + BENCH_INFO(event_types_multi, create_events_multi, free_buffers_multi, NULL), + BENCH_INFO(event_types_multi_no_sub, create_events_multi, free_buffers_multi, + "event_types_multi (no sub)"), BENCH_INFO(event_type_multi, create_events_multi, free_buffers_multi, NULL), + BENCH_INFO(event_pool, create_events, free_buffers, NULL), + BENCH_INFO_COND(event_user_area, create_events, free_buffers, NULL, check_uarea), + BENCH_INFO_COND(event_user_area_and_flag, create_events, free_buffers, NULL, check_uarea), BENCH_INFO(event_is_valid, create_events, free_buffers, NULL), BENCH_INFO(event_free, create_events, NULL, NULL), BENCH_INFO(event_free_multi, create_events_multi, NULL, NULL), diff --git a/test/performance/odp_bench_misc.c b/test/performance/odp_bench_misc.c index 64318938a..61afdc398 100644 --- a/test/performance/odp_bench_misc.c +++ b/test/performance/odp_bench_misc.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_bench_misc.c + * + * Microbenchmark application for miscellaneous API functions + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #ifndef _GNU_SOURCE #define _GNU_SOURCE /* Needed for sigaction */ #endif @@ -20,7 +28,7 @@ #include <unistd.h> /* Number of API function calls per test case */ -#define REPEAT_COUNT 1000 +#define REPEAT_COUNT 1024 /* Default number of rounds per test case */ #define ROUNDS 1000u @@ -733,6 +741,42 @@ static int mb_full(void) return i; } +static int prefetch(void) +{ + uint64_t *a1 = gbl_args->a1; + uint32_t index = 0; + int i; + + for (i = 0; i < REPEAT_COUNT; i++) { + odp_prefetch(&a1[index]); + + /* Prefetch every 64B */ + index += 8; + if (odp_unlikely(index >= REPEAT_COUNT)) + index = 0; + } + + return i; +} + +static int prefetch_store(void) +{ + uint64_t *a1 = gbl_args->a1; + uint32_t index = 0; + int i; + + for (i = 0; i < REPEAT_COUNT; i++) { + odp_prefetch_store(&a1[index]); + + /* Prefetch every 64B */ + index += 8; + if (odp_unlikely(index >= REPEAT_COUNT)) + index = 0; + } + + return i; +} + bench_info_t test_suite[] = { BENCH_INFO(time_local, NULL, 0, NULL), BENCH_INFO(time_local_strict, NULL, 0, NULL), @@ -785,6 +829,8 @@ bench_info_t test_suite[] = { BENCH_INFO(mb_release, NULL, 0, NULL), BENCH_INFO(mb_acquire, NULL, 0, NULL), BENCH_INFO(mb_full, NULL, 0, NULL), + BENCH_INFO(prefetch, NULL, 0, NULL), + BENCH_INFO(prefetch_store, NULL, 0, NULL), }; /* Print usage information */ diff --git a/test/performance/odp_bench_packet.c b/test/performance/odp_bench_packet.c index 67b6b9cfc..cb9e3ca03 100644 --- a/test/performance/odp_bench_packet.c +++ b/test/performance/odp_bench_packet.c @@ -6,9 +6,11 @@ */ /** - * @file + * @example odp_bench_packet.c * - * @example odp_bench_packet.c Microbenchmarks for packet functions + * Microbenchmark application for packet API functions + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include <stdlib.h> @@ -39,7 +41,7 @@ #define TEST_REPEAT_COUNT 1000 /** Number of rounds per test case */ -#define TEST_ROUNDS 10u +#define TEST_ROUNDS 2u /** Maximum burst size for *_multi operations */ #define TEST_MAX_BURST 64 @@ -1094,6 +1096,27 @@ static int packet_user_area_size(void) return ret; } +static int packet_user_flag(void) +{ + int i; + uint32_t ret = 0; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + ret += !odp_packet_user_flag(gbl_args->pkt_tbl[i]); + + return ret; +} + +static int packet_user_flag_set(void) +{ + int i; + + for (i = 0; i < TEST_REPEAT_COUNT; i++) + odp_packet_user_flag_set(gbl_args->pkt_tbl[i], 1); + + return i; +} + static int packet_l2_ptr(void) { int i; @@ -1512,6 +1535,8 @@ bench_info_t test_suite[] = { BENCH_INFO(packet_user_ptr_set, create_packets, free_packets, NULL), BENCH_INFO(packet_user_area, create_packets, free_packets, NULL), BENCH_INFO(packet_user_area_size, create_packets, free_packets, NULL), + BENCH_INFO(packet_user_flag, create_packets, free_packets, NULL), + BENCH_INFO(packet_user_flag_set, create_packets, free_packets, NULL), BENCH_INFO(packet_l2_ptr, create_packets, free_packets, NULL), BENCH_INFO(packet_l2_offset, create_packets, free_packets, NULL), BENCH_INFO(packet_l2_offset_set, create_packets, free_packets, NULL), diff --git a/test/performance/odp_bench_pktio_sp.c b/test/performance/odp_bench_pktio_sp.c index 65d85a062..017e7565f 100644 --- a/test/performance/odp_bench_pktio_sp.c +++ b/test/performance/odp_bench_pktio_sp.c @@ -2,6 +2,14 @@ * Copyright (c) 2023 Nokia */ +/** + * @example odp_bench_pktio_sp.c + * + * Microbenchmark application for packet IO slow path functions + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #ifndef _GNU_SOURCE #define _GNU_SOURCE /* Needed for sigaction */ #endif diff --git a/test/performance/odp_bench_timer.c b/test/performance/odp_bench_timer.c index a53671460..65c7a9168 100644 --- a/test/performance/odp_bench_timer.c +++ b/test/performance/odp_bench_timer.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_bench_timer.c + * + * Microbenchmark application for timer API functions + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #ifndef _GNU_SOURCE #define _GNU_SOURCE /* Needed for sigaction */ #endif @@ -695,8 +703,12 @@ exit: if (gbl_args->pool != ODP_POOL_INVALID) odp_pool_destroy(gbl_args->pool); - if (gbl_args->timer != ODP_TIMER_INVALID) - odp_timer_free(gbl_args->timer); + if (gbl_args->timer != ODP_TIMER_INVALID) { + if (odp_timer_free(gbl_args->timer)) { + ODPH_ERR("Timer free failed\n"); + exit(EXIT_FAILURE); + } + } if (gbl_args->timer_pool != ODP_TIMER_POOL_INVALID) odp_timer_pool_destroy(gbl_args->timer_pool); diff --git a/test/performance/odp_cpu_bench.c b/test/performance/odp_cpu_bench.c index 7ef12dc30..39eff620d 100644 --- a/test/performance/odp_cpu_bench.c +++ b/test/performance/odp_cpu_bench.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_cpu_bench.c + * + * Application for CPU stress testing + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include <odp_api.h> #include <odp/helper/odph_api.h> @@ -327,7 +335,7 @@ static int run_thread(void *arg) odp_event_t ev; ev = odp_schedule(NULL, - odp_schedule_wait_time(ODP_TIME_SEC_IN_NS)); + odp_schedule_wait_time(100 * ODP_TIME_MSEC_IN_NS)); if (ev == ODP_EVENT_INVALID) break; diff --git a/test/performance/odp_crc.c b/test/performance/odp_crc.c index 89e8af837..89e2e971f 100644 --- a/test/performance/odp_crc.c +++ b/test/performance/odp_crc.c @@ -5,6 +5,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_crc.c + * + * Performance test application for CRC hash APIs + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include <stdio.h> #include <string.h> #include <stdint.h> diff --git a/test/performance/odp_crypto.c b/test/performance/odp_crypto.c index 93315ce05..a644da5e1 100644 --- a/test/performance/odp_crypto.c +++ b/test/performance/odp_crypto.c @@ -5,6 +5,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_crypto.c + * + * Performance test application for crypto APIs + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif /* _GNU_SOURCE */ @@ -1233,7 +1241,7 @@ int main(int argc, char *argv[]) odp_pool_capability_t pool_capa; odp_crypto_capability_t crypto_capa; uint32_t max_seg_len; - unsigned i; + uint32_t i; /* Let helper collect its own arguments (e.g. --odph_proc) */ argc = odph_parse_options(argc, argv); @@ -1366,8 +1374,6 @@ int main(int argc, char *argv[]) run_measure_one_config(&test_run_arg); } } else { - unsigned int i; - for (i = 0; i < ODPH_ARRAY_SIZE(algs_config); i++) { test_run_arg.crypto_alg_config = algs_config + i; run_measure_one_config(&test_run_arg); diff --git a/test/performance/odp_dma_perf.c b/test/performance/odp_dma_perf.c index 21c9c0558..2f4ca490d 100644 --- a/test/performance/odp_dma_perf.c +++ b/test/performance/odp_dma_perf.c @@ -1,14 +1,16 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright (c) 2021-2023 Nokia + * Copyright (c) 2021-2024 Nokia */ /** - * DMA performance tester + * @example odp_dma_perf.c * * This tester application can be used to profile the performance of an ODP DMA implementation. * Tester workflow is simple and consists of issuing as many back-to-back DMA transfers as the * implementation allows and then recording key performance statistics (such as function overhead, * latencies etc.). + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #ifndef _GNU_SOURCE @@ -107,7 +109,7 @@ typedef struct { odp_dma_transfer_param_t trs_param; odp_dma_compl_param_t compl_param; odp_ticketlock_t lock; - uint64_t trs_start_tm; + odp_time_t trs_start_tm; uint64_t trs_start_cc; uint64_t trs_poll_cnt; odp_bool_t is_running; @@ -202,7 +204,7 @@ typedef struct prog_config_s { uint32_t src_seg_len; uint32_t dst_seg_len; uint32_t num_inflight; - uint32_t time_sec; + double time_sec; uint32_t num_sessions; uint32_t src_cache_size; uint32_t dst_cache_size; @@ -249,14 +251,14 @@ static void init_config(prog_config_t *config) stats = &config->thread_config[i].stats; memset(sd, 0, sizeof(*sd)); - for (uint32_t i = 0U; i < MAX_SEGS; ++i) { - info = &sd->dma.infos[i]; + for (uint32_t j = 0U; j < MAX_SEGS; ++j) { + info = &sd->dma.infos[j]; info->compl_param.transfer_id = ODP_DMA_TRANSFER_ID_INVALID; info->compl_param.event = ODP_EVENT_INVALID; info->compl_param.queue = ODP_QUEUE_INVALID; odp_ticketlock_init(&info->lock); - sd->seg.src_pkt[i] = ODP_PACKET_INVALID; - sd->seg.dst_pkt[i] = ODP_PACKET_INVALID; + sd->seg.src_pkt[j] = ODP_PACKET_INVALID; + sd->seg.dst_pkt[j] = ODP_PACKET_INVALID; } sd->dma.handle = ODP_DMA_INVALID; @@ -597,7 +599,7 @@ static parse_result_t parse_options(int argc, char **argv, prog_config_t *config config->num_inflight = atoi(optarg); break; case 'T': - config->time_sec = atoi(optarg); + config->time_sec = atof(optarg); break; case 'c': config->num_workers = atoi(optarg); @@ -871,20 +873,21 @@ static void free_memory(const sd_t *sd) static void run_transfer(odp_dma_t handle, trs_info_t *info, stats_t *stats, ver_fn_t ver_fn) { - uint64_t start_tm, end_tm, start_cc, end_cc, trs_tm, trs_cc, start_cc_diff; + odp_time_t start_tm, end_tm; + uint64_t start_cc, end_cc, trs_tm, trs_cc; odp_dma_result_t res; int ret; - start_tm = odp_time_local_strict_ns(); + start_tm = odp_time_local_strict(); start_cc = odp_cpu_cycles(); ret = odp_dma_transfer(handle, &info->trs_param, &res); end_cc = odp_cpu_cycles(); - end_tm = odp_time_local_strict_ns(); + end_tm = odp_time_local_strict(); if (odp_unlikely(ret <= 0)) { ++stats->start_errs; } else { - trs_tm = end_tm - start_tm; + trs_tm = odp_time_diff_ns(end_tm, start_tm); stats->max_trs_tm = ODPH_MAX(trs_tm, stats->max_trs_tm); stats->min_trs_tm = ODPH_MIN(trs_tm, stats->min_trs_tm); stats->trs_tm += trs_tm; @@ -893,10 +896,9 @@ static void run_transfer(odp_dma_t handle, trs_info_t *info, stats_t *stats, ver stats->min_trs_cc = ODPH_MIN(trs_cc, stats->min_trs_cc); stats->trs_cc += trs_cc; ++stats->trs_cnt; - start_cc_diff = odp_cpu_cycles_diff(end_cc, start_cc); - stats->max_start_cc = ODPH_MAX(start_cc_diff, stats->max_start_cc); - stats->min_start_cc = ODPH_MIN(start_cc_diff, stats->min_start_cc); - stats->start_cc += start_cc_diff; + stats->max_start_cc = stats->max_trs_cc; + stats->min_start_cc = stats->min_trs_cc; + stats->start_cc += trs_cc; ++stats->start_cnt; if (odp_unlikely(!res.success)) { @@ -967,7 +969,8 @@ static odp_bool_t configure_poll_compl(sd_t *sd) static void poll_transfer(sd_t *sd, trs_info_t *info, stats_t *stats) { - uint64_t start_cc, end_cc, trs_tm, trs_cc, wait_cc, start_tm, start_cc_diff; + uint64_t start_cc, end_cc, trs_tm, trs_cc, wait_cc, start_cc_diff; + odp_time_t start_tm; odp_dma_t handle = sd->dma.handle; odp_dma_result_t res; int ret; @@ -992,7 +995,7 @@ static void poll_transfer(sd_t *sd, trs_info_t *info, stats_t *stats) if (ret == 0) return; - trs_tm = odp_time_global_strict_ns() - info->trs_start_tm; + trs_tm = odp_time_diff_ns(odp_time_global_strict(), info->trs_start_tm); stats->max_trs_tm = ODPH_MAX(trs_tm, stats->max_trs_tm); stats->min_trs_tm = ODPH_MIN(trs_tm, stats->min_trs_tm); stats->trs_tm += trs_tm; @@ -1017,7 +1020,7 @@ static void poll_transfer(sd_t *sd, trs_info_t *info, stats_t *stats) if (sd->prep_trs_fn != NULL) sd->prep_trs_fn(sd, info); - start_tm = odp_time_global_strict_ns(); + start_tm = odp_time_global_strict(); start_cc = odp_cpu_cycles(); ret = odp_dma_transfer_start(handle, &info->trs_param, &info->compl_param); end_cc = odp_cpu_cycles(); @@ -1146,7 +1149,8 @@ static odp_bool_t configure_event_compl(sd_t *sd) static odp_bool_t start_initial_transfers(sd_t *sd) { - uint64_t start_tm, start_cc; + odp_time_t start_tm; + uint64_t start_cc; trs_info_t *info; int ret; @@ -1156,7 +1160,7 @@ static odp_bool_t start_initial_transfers(sd_t *sd) if (sd->prep_trs_fn != NULL) sd->prep_trs_fn(sd, info); - start_tm = odp_time_global_strict_ns(); + start_tm = odp_time_global_strict(); start_cc = odp_cpu_cycles(); ret = odp_dma_transfer_start(sd->dma.handle, &info->trs_param, &info->compl_param); @@ -1174,7 +1178,8 @@ static odp_bool_t start_initial_transfers(sd_t *sd) static void wait_compl_event(sd_t *sd, stats_t *stats) { - uint64_t start_cc, end_cc, wait_cc, trs_tm, trs_cc, start_tm, start_cc_diff; + uint64_t start_cc, end_cc, wait_cc, trs_tm, trs_cc, start_cc_diff; + odp_time_t start_tm; odp_event_t ev; odp_dma_result_t res; trs_info_t *info; @@ -1191,7 +1196,7 @@ static void wait_compl_event(sd_t *sd, stats_t *stats) odp_dma_compl_result(odp_dma_compl_from_event(ev), &res); info = res.user_ptr; - trs_tm = odp_time_global_strict_ns() - info->trs_start_tm; + trs_tm = odp_time_diff_ns(odp_time_global_strict(), info->trs_start_tm); stats->max_trs_tm = ODPH_MAX(trs_tm, stats->max_trs_tm); stats->min_trs_tm = ODPH_MIN(trs_tm, stats->min_trs_tm); stats->trs_tm += trs_tm; @@ -1218,7 +1223,7 @@ static void wait_compl_event(sd_t *sd, stats_t *stats) if (sd->prep_trs_fn != NULL) sd->prep_trs_fn(sd, info); - start_tm = odp_time_global_strict_ns(); + start_tm = odp_time_global_strict(); start_cc = odp_cpu_cycles(); ret = odp_dma_transfer_start(sd->dma.handle, &info->trs_param, &info->compl_param); end_cc = odp_cpu_cycles(); @@ -1241,7 +1246,7 @@ static void drain_compl_events(ODP_UNUSED sd_t *sd) odp_event_t ev; while (true) { - ev = odp_schedule(NULL, odp_schedule_wait_time(ODP_TIME_SEC_IN_NS)); + ev = odp_schedule(NULL, odp_schedule_wait_time(100 * ODP_TIME_MSEC_IN_NS)); if (ev == ODP_EVENT_INVALID) break; @@ -1250,7 +1255,8 @@ static void drain_compl_events(ODP_UNUSED sd_t *sd) static void run_memcpy(trs_info_t *info, stats_t *stats, ver_fn_t ver_fn) { - uint64_t start_tm, end_tm, start_cc, end_cc, trs_tm, trs_cc, start_cc_diff; + odp_time_t start_tm; + uint64_t start_cc, end_cc, trs_tm, trs_cc; const odp_dma_transfer_param_t *param = &info->trs_param; uint32_t tot_len, src_len, dst_len, min_len, len, i = 0U, j = 0U, src_off = 0U, dst_off = 0U, src_rem, dst_rem; @@ -1265,7 +1271,7 @@ static void run_memcpy(trs_info_t *info, stats_t *stats, ver_fn_t ver_fn) dst_len = param->dst_seg->len; min_len = ODPH_MIN(src_len, dst_len); len = min_len; - start_tm = odp_time_local_strict_ns(); + start_tm = odp_time_local_strict(); start_cc = odp_cpu_cycles(); while (tot_len > 0U) { @@ -1295,8 +1301,7 @@ static void run_memcpy(trs_info_t *info, stats_t *stats, ver_fn_t ver_fn) } end_cc = odp_cpu_cycles(); - end_tm = odp_time_local_strict_ns(); - trs_tm = end_tm - start_tm; + trs_tm = odp_time_diff_ns(odp_time_local_strict(), start_tm); stats->max_trs_tm = ODPH_MAX(trs_tm, stats->max_trs_tm); stats->min_trs_tm = ODPH_MIN(trs_tm, stats->min_trs_tm); stats->trs_tm += trs_tm; @@ -1305,10 +1310,9 @@ static void run_memcpy(trs_info_t *info, stats_t *stats, ver_fn_t ver_fn) stats->min_trs_cc = ODPH_MIN(trs_cc, stats->min_trs_cc); stats->trs_cc += trs_cc; ++stats->trs_cnt; - start_cc_diff = odp_cpu_cycles_diff(end_cc, start_cc); - stats->max_start_cc = ODPH_MAX(start_cc_diff, stats->max_start_cc); - stats->min_start_cc = ODPH_MIN(start_cc_diff, stats->min_start_cc); - stats->start_cc += start_cc_diff; + stats->max_start_cc = stats->max_trs_cc; + stats->min_start_cc = stats->min_trs_cc; + stats->start_cc += trs_cc; ++stats->start_cnt; ++stats->completed; @@ -1548,7 +1552,7 @@ static int transfer(void *args) stats_t *stats = &thr_config->stats; test_api_t *api = &prog_conf->api; odp_thrmask_t mask; - uint64_t start_tm, end_tm; + odp_time_t start_tm; odp_barrier_wait(&prog_config->init_barrier); @@ -1562,13 +1566,12 @@ static int transfer(void *args) } } - start_tm = odp_time_local_strict_ns(); + start_tm = odp_time_local_strict(); while (odp_atomic_load_u32(&prog_config->is_running)) api->wait_fn(sd, stats); - end_tm = odp_time_local_strict_ns(); - thr_config->stats.tot_tm = end_tm - start_tm; + thr_config->stats.tot_tm = odp_time_diff_ns(odp_time_local_strict(), start_tm); if (api->drain_fn != NULL) api->drain_fn(sd); @@ -1690,7 +1693,7 @@ static void print_humanised(uint64_t value, const char *type) else if (value > MEGAS) printf("%.2f M%s\n", (double)value / MEGAS, type); else if (value > KILOS) - printf("%.2f K%s\n", (double)value / KILOS, type); + printf("%.2f k%s\n", (double)value / KILOS, type); else printf("%" PRIu64 " %s\n", value, type); } @@ -1700,8 +1703,8 @@ static void print_stats(const prog_config_t *config) const stats_t *stats; uint64_t data_cnt = config->num_in_segs * config->src_seg_len, tot_completed = 0U, tot_tm = 0U, tot_trs_tm = 0U, tot_trs_cc = 0U, tot_trs_cnt = 0U, tot_min_tm = UINT64_MAX, - tot_max_tm = 0U, tot_min_cc = UINT64_MAX, tot_max_cc = 0U, avg_start_cc, avg_wait_cc, - avg_tot_tm; + tot_max_tm = 0U, tot_min_cc = UINT64_MAX, tot_max_cc = 0U, avg_start_cc, avg_wait_cc; + double avg_tot_tm; printf("\n======================\n\n" "DMA performance test done\n\n" @@ -1769,11 +1772,12 @@ static void print_stats(const prog_config_t *config) stats->trs_cnt > 0U ? stats->trs_cc / stats->trs_cnt : 0U, stats->trs_cnt > 0U ? stats->min_trs_cc : 0U, stats->trs_cnt > 0U ? stats->max_trs_cc : 0U); - print_humanised(stats->completed / (stats->tot_tm / ODP_TIME_SEC_IN_NS), + print_humanised(stats->completed / + ((double)stats->tot_tm / ODP_TIME_SEC_IN_NS), "OPS"); printf(" speed: "); print_humanised(stats->completed * data_cnt / - (stats->tot_tm / ODP_TIME_SEC_IN_NS), "B/s"); + ((double)stats->tot_tm / ODP_TIME_SEC_IN_NS), "B/s"); } avg_start_cc = stats->start_cnt > 0U ? stats->start_cc / stats->start_cnt : 0U; @@ -1816,7 +1820,7 @@ static void print_stats(const prog_config_t *config) printf("\n"); } - avg_tot_tm = tot_tm / config->num_workers / ODP_TIME_SEC_IN_NS; + avg_tot_tm = (double)tot_tm / config->num_workers / ODP_TIME_SEC_IN_NS; printf(" total:\n" " average time per transfer: %" PRIu64 " (min: %" PRIu64 ", max: %" PRIu64 ") ns\n" @@ -1913,8 +1917,12 @@ int main(int argc, char **argv) goto out_test; } - if (prog_conf->time_sec) { - sleep(prog_conf->time_sec); + if (prog_conf->time_sec > 0.001) { + struct timespec ts; + + ts.tv_sec = prog_conf->time_sec; + ts.tv_nsec = (prog_conf->time_sec - ts.tv_sec) * ODP_TIME_SEC_IN_NS; + nanosleep(&ts, NULL); odp_atomic_store_u32(&prog_conf->is_running, 0U); } diff --git a/test/performance/odp_dma_perf_run.sh b/test/performance/odp_dma_perf_run.sh index f5d567740..31948e40a 100755 --- a/test/performance/odp_dma_perf_run.sh +++ b/test/performance/odp_dma_perf_run.sh @@ -10,7 +10,7 @@ BIN_NAME=odp_dma_perf SEGC=0 SEGS=1024 INFL=1 -TIME=1 +TIME=0.1 TESTS_RUN=0 check_result() diff --git a/test/performance/odp_dmafwd.c b/test/performance/odp_dmafwd.c index 188a8e358..694973ce0 100644 --- a/test/performance/odp_dmafwd.c +++ b/test/performance/odp_dmafwd.c @@ -3,12 +3,14 @@ */ /** - * DMA forwarder + * @example odp_dmafwd.c * * This tester application can be used to profile the performance of an ODP DMA implementation. * Tester workflow consists of packet reception, copy and forwarding steps. Packets are first * received from configured interfaces after which packets are copied, either with plain SW memory * copy or with DMA offload copy. Finally, copied packets are echoed back to the sender(s). + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #ifndef _GNU_SOURCE @@ -20,6 +22,7 @@ #include <signal.h> #include <stdio.h> #include <unistd.h> +#include <time.h> #include <odp_api.h> #include <odp/helper/odph_api.h> @@ -152,7 +155,7 @@ typedef struct prog_config_s { uint32_t trs_cache_size; uint32_t compl_cache_size; uint32_t stash_cache_size; - uint32_t time_sec; + double time_sec; odp_stash_type_t stash_type; int num_thrs; uint8_t num_ifs; @@ -505,7 +508,7 @@ static parse_result_t parse_options(int argc, char **argv, prog_config_t *config config->cache_size = atoi(optarg); break; case 'T': - config->time_sec = atoi(optarg); + config->time_sec = atof(optarg); break; case 'h': print_usage(&config->dyn_defs); @@ -758,7 +761,7 @@ static void drain_events(thread_config_t *config ODP_UNUSED) transfer_t *trs; while (true) { - ev = odp_schedule(NULL, odp_schedule_wait_time(ODP_TIME_SEC_IN_NS * 2U)); + ev = odp_schedule(NULL, odp_schedule_wait_time(100 * ODP_TIME_MSEC_IN_NS)); if (ev == ODP_EVENT_INVALID) break; @@ -1378,7 +1381,7 @@ int main(int argc, char **argv) odp_init_param_init(&init_param); init_param.mem_model = odph_opts.mem_model; - if (odp_init_global(&odp_instance, NULL, NULL)) { + if (odp_init_global(&odp_instance, &init_param, NULL)) { ODPH_ERR("ODP global init failed, exiting\n"); exit(EXIT_FAILURE); } @@ -1436,8 +1439,12 @@ int main(int argc, char **argv) goto out_test; } - if (prog_conf->time_sec) { - sleep(prog_conf->time_sec); + if (prog_conf->time_sec > 0.001) { + struct timespec ts; + + ts.tv_sec = prog_conf->time_sec; + ts.tv_nsec = (prog_conf->time_sec - ts.tv_sec) * ODP_TIME_SEC_IN_NS; + nanosleep(&ts, NULL); odp_atomic_store_u32(&prog_conf->is_running, 0U); } else { while (odp_atomic_load_u32(&prog_conf->is_running)) diff --git a/test/performance/odp_dmafwd_run.sh b/test/performance/odp_dmafwd_run.sh index fa629bd0c..ebb9b153a 100755 --- a/test/performance/odp_dmafwd_run.sh +++ b/test/performance/odp_dmafwd_run.sh @@ -10,7 +10,7 @@ PERF_TEST_DIR=${TEST_SRC_DIR}/../../${PERF_TEST_DIR} BIN_NAME=odp_dmafwd BATCH=10 -TIME=2 +TIME=0.1 TESTS_RUN=0 check_env() diff --git a/test/performance/odp_ipsec.c b/test/performance/odp_ipsec.c index 8a0bc2989..3ea93ec96 100644 --- a/test/performance/odp_ipsec.c +++ b/test/performance/odp_ipsec.c @@ -6,6 +6,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_ipsec.c + * + * Performance test application for IPsec APIs + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif /* _GNU_SOURCE */ @@ -1375,8 +1383,6 @@ int main(int argc, char *argv[]) run_measure_one_config(&cargs, cargs.alg_config); } } else { - unsigned int i; - for (i = 0; i < ODPH_ARRAY_SIZE(algs_config); i++) { if (cargs.ah && algs_config[i].crypto.cipher_alg != diff --git a/test/performance/odp_ipsecfwd.c b/test/performance/odp_ipsecfwd.c index 5c35d67f7..0220cf6ae 100644 --- a/test/performance/odp_ipsecfwd.c +++ b/test/performance/odp_ipsecfwd.c @@ -2,6 +2,15 @@ * Copyright (c) 2022-2023 Nokia */ +/** + * @example odp_ipsecfwd.c + * + * Simple IPsec performance tester application which forwards and processes + * plain and IPsec packets. + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif diff --git a/test/performance/odp_l2fwd.c b/test/performance/odp_l2fwd.c index 1e9b79db0..b993de4cb 100644 --- a/test/performance/odp_l2fwd.c +++ b/test/performance/odp_l2fwd.c @@ -1,11 +1,19 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2019-2023, Nokia + * Copyright (c) 2019-2024, Nokia * Copyright (c) 2020-2021, Marvell * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_l2fwd.c + * + * L2 forwarding example application + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + /* enable strtok */ #ifndef _GNU_SOURCE #define _GNU_SOURCE @@ -81,7 +89,32 @@ static inline int sched_mode(pktin_mode_t in_mode) */ typedef struct { /* Some extra features (e.g. error checks) have been enabled */ - int extra_feat; + uint8_t extra_feat; + + /* Prefetch packet data */ + uint8_t prefetch; + + /* Change destination eth addresses */ + uint8_t dst_change; + + /* Change source eth addresses */ + uint8_t src_change; + + /* Read packet data in uint64_t words */ + uint16_t data_rd; + + /* Check packet errors */ + uint8_t error_check; + + /* Packet copy */ + uint8_t packet_copy; + + /* Checksum offload */ + uint8_t chksum; + + /* Print debug info on every packet */ + uint8_t verbose_pkt; + unsigned int cpu_count; int if_count; /* Number of interfaces to be used */ int addr_count; /* Number of dst addresses to be used */ @@ -93,11 +126,6 @@ typedef struct { int time; /* Time in seconds to run. */ int accuracy; /* Number of seconds to get and print stats */ char *if_str; /* Storage for interface names */ - int dst_change; /* Change destination eth addresses */ - int src_change; /* Change source eth addresses */ - int error_check; /* Check packet errors */ - int packet_copy; /* Packet copy */ - int chksum; /* Checksum offload */ int sched_mode; /* Scheduler mode */ int num_groups; /* Number of scheduling groups */ int group_mode; /* How threads join groups */ @@ -109,11 +137,12 @@ typedef struct { uint32_t num_vec; /* Number of vectors per pool */ uint64_t vec_tmo_ns; /* Vector formation timeout in ns */ uint32_t vec_size; /* Vector size */ - int verbose; /* Verbose output */ - uint32_t packet_len; /* Maximum packet length supported */ - uint32_t seg_len; /* Pool segment length */ + int verbose; /* Verbose output */ + uint32_t packet_len; /* Maximum packet length supported */ + uint32_t seg_len; /* Pool segment length */ int promisc_mode; /* Promiscuous mode enabled */ int flow_aware; /* Flow aware scheduling enabled */ + uint8_t input_ts; /* Packet input timestamping enabled */ int mtu; /* Interface MTU */ int num_prio; odp_schedule_prio_t prio[MAX_PKTIOS]; /* Priority of input queues of an interface */ @@ -131,6 +160,8 @@ typedef union ODP_ALIGNED_CACHE { uint64_t tx_drops; /* Number of failed packet copies */ uint64_t copy_fails; + /* Dummy sum of packet data */ + uint64_t dummy_sum; } s; uint8_t padding[ODP_CACHE_LINE_SIZE]; @@ -251,6 +282,15 @@ static inline int drop_err_pkts(odp_packet_t pkt_tbl[], unsigned num) return dropped; } +static inline void prefetch_data(uint8_t prefetch, odp_packet_t pkt_tbl[], uint32_t num) +{ + if (prefetch == 0) + return; + + for (uint32_t i = 0; i < num; i++) + odp_packet_prefetch(pkt_tbl[i], 0, prefetch * 64); +} + /* * Fill packets' eth addresses according to the destination port * @@ -270,9 +310,6 @@ static inline void fill_eth_addrs(odp_packet_t pkt_tbl[], for (i = 0; i < num; ++i) { pkt = pkt_tbl[i]; - - odp_packet_prefetch(pkt, 0, ODPH_ETHHDR_LEN); - eth = odp_packet_data(pkt); if (gbl_args->appl.src_change) @@ -318,6 +355,57 @@ static inline void chksum_insert(odp_packet_t *pkt_tbl, int pkts) } } +static void print_packets(odp_packet_t *pkt_tbl, int num) +{ + odp_packet_t pkt; + uintptr_t data_ptr; + uint32_t bit, align; + + for (int i = 0; i < num; i++) { + pkt = pkt_tbl[i]; + data_ptr = (uintptr_t)odp_packet_data(pkt); + + for (bit = 0, align = 1; bit < 32; bit++, align *= 2) + if (data_ptr & (0x1 << bit)) + break; + + printf(" Packet data: 0x%" PRIxPTR "\n" + " Packet len: %u\n" + " Packet seg len: %u\n" + " Data align: %u\n" + " Num segments: %i\n" + " Headroom size: %u\n" + " User area size: %u\n\n", + data_ptr, odp_packet_len(pkt), odp_packet_seg_len(pkt), align, + odp_packet_num_segs(pkt), odp_packet_headroom(pkt), + odp_packet_user_area_size(pkt)); + } +} + +static inline void data_rd(odp_packet_t *pkt_tbl, int num, uint16_t rd_words, stats_t *stats) +{ + odp_packet_t pkt; + uint64_t *data; + int i; + uint32_t len, words, j; + uint64_t sum = 0; + + for (i = 0; i < num; i++) { + pkt = pkt_tbl[i]; + data = odp_packet_data(pkt); + len = odp_packet_seg_len(pkt); + + words = rd_words; + if (rd_words * 8 > len) + words = len / 8; + + for (j = 0; j < words; j++) + sum += data[j]; + } + + stats->s.dummy_sum += sum; +} + static inline int copy_packets(odp_packet_t *pkt_tbl, int pkts) { odp_packet_t old_pkt, new_pkt; @@ -343,21 +431,29 @@ static inline int copy_packets(odp_packet_t *pkt_tbl, int pkts) /* * Return number of packets remaining in the pkt_tbl */ -static inline int process_extra_features(odp_packet_t *pkt_tbl, int pkts, - stats_t *stats) +static inline int process_extra_features(const appl_args_t *appl_args, odp_packet_t *pkt_tbl, + int pkts, stats_t *stats) { - if (odp_unlikely(gbl_args->appl.extra_feat)) { - if (gbl_args->appl.packet_copy) { + if (odp_unlikely(appl_args->extra_feat)) { + uint16_t rd_words = appl_args->data_rd; + + if (appl_args->verbose_pkt) + print_packets(pkt_tbl, pkts); + + if (rd_words) + data_rd(pkt_tbl, pkts, rd_words, stats); + + if (appl_args->packet_copy) { int fails; fails = copy_packets(pkt_tbl, pkts); stats->s.copy_fails += fails; } - if (gbl_args->appl.chksum) + if (appl_args->chksum) chksum_insert(pkt_tbl, pkts); - if (gbl_args->appl.error_check) { + if (appl_args->error_check) { int rx_drops; /* Drop packets with errors */ @@ -421,6 +517,7 @@ static int run_worker_sched_mode_vector(void *arg) odp_queue_t tx_queue[MAX_PKTIOS]; thread_args_t *thr_args = arg; stats_t *stats = &thr_args->stats; + const appl_args_t *appl_args = &gbl_args->appl; int use_event_queue = gbl_args->appl.out_mode; pktin_mode_t in_mode = gbl_args->appl.in_mode; @@ -484,7 +581,10 @@ static int run_worker_sched_mode_vector(void *arg) pkts = odp_packet_vector_tbl(pkt_vec, &pkt_tbl); } - pkts = process_extra_features(pkt_tbl, pkts, stats); + prefetch_data(appl_args->prefetch, pkt_tbl, pkts); + + pkts = process_extra_features(appl_args, pkt_tbl, pkts, stats); + if (odp_unlikely(pkts) == 0) { if (pkt_vec != ODP_PACKET_VECTOR_INVALID) odp_packet_vector_free(pkt_vec); @@ -567,6 +667,7 @@ static int run_worker_sched_mode(void *arg) char extra_str[EXTRA_STR_LEN]; thread_args_t *thr_args = arg; stats_t *stats = &thr_args->stats; + const appl_args_t *appl_args = &gbl_args->appl; int use_event_queue = gbl_args->appl.out_mode; pktin_mode_t in_mode = gbl_args->appl.in_mode; @@ -630,7 +731,10 @@ static int run_worker_sched_mode(void *arg) odp_packet_from_event_multi(pkt_tbl, ev_tbl, pkts); - pkts = process_extra_features(pkt_tbl, pkts, stats); + prefetch_data(appl_args->prefetch, pkt_tbl, pkts); + + pkts = process_extra_features(appl_args, pkt_tbl, pkts, stats); + if (odp_unlikely(pkts) == 0) continue; @@ -704,6 +808,7 @@ static int run_worker_plain_queue_mode(void *arg) int pktio = 0; thread_args_t *thr_args = arg; stats_t *stats = &thr_args->stats; + const appl_args_t *appl_args = &gbl_args->appl; int use_event_queue = gbl_args->appl.out_mode; int i; @@ -743,7 +848,10 @@ static int run_worker_plain_queue_mode(void *arg) odp_packet_from_event_multi(pkt_tbl, event, pkts); - pkts = process_extra_features(pkt_tbl, pkts, stats); + prefetch_data(appl_args->prefetch, pkt_tbl, pkts); + + pkts = process_extra_features(appl_args, pkt_tbl, pkts, stats); + if (odp_unlikely(pkts) == 0) continue; @@ -801,6 +909,7 @@ static int run_worker_direct_mode(void *arg) int pktio = 0; thread_args_t *thr_args = arg; stats_t *stats = &thr_args->stats; + const appl_args_t *appl_args = &gbl_args->appl; int use_event_queue = gbl_args->appl.out_mode; thr = odp_thread_id(); @@ -835,7 +944,10 @@ static int run_worker_direct_mode(void *arg) if (odp_unlikely(pkts <= 0)) continue; - pkts = process_extra_features(pkt_tbl, pkts, stats); + prefetch_data(appl_args->prefetch, pkt_tbl, pkts); + + pkts = process_extra_features(appl_args, pkt_tbl, pkts, stats); + if (odp_unlikely(pkts) == 0) continue; @@ -961,6 +1073,14 @@ static int create_pktio(const char *dev, int idx, int num_rx, int num_tx, odp_po odp_pktio_config_init(&config); + if (gbl_args->appl.input_ts) { + if (!pktio_capa.config.pktin.bit.ts_all) { + ODPH_ERR("Packet input timestamping not supported: %s\n", dev); + return -1; + } + config.pktin.bit.ts_all = 1; + } + config.parser.layer = ODP_PROTO_LAYER_NONE; if (gbl_args->appl.error_check || gbl_args->appl.chksum) config.parser.layer = ODP_PROTO_LAYER_ALL; @@ -1495,6 +1615,10 @@ static void usage(char *progname) " -p, --packet_copy 0: Don't copy packet (default)\n" " 1: Create and send copy of the received packet.\n" " Free the original packet.\n" + " -R, --data_rd <num> Number of packet data words (uint64_t) to read from\n" + " every received packet. Number of words is rounded down\n" + " to fit into the first segment of a packet. Default\n" + " is 0.\n" " -y, --pool_per_if Create a packet (and packet vector) pool per interface.\n" " 0: Share a single pool between all interfaces (default)\n" " 1: Create a pool per interface\n" @@ -1511,8 +1635,11 @@ static void usage(char *progname) " -l, --packet_len <len> Maximum length of packets supported (default %d).\n" " -L, --seg_len <len> Packet pool segment length\n" " (default equal to packet length).\n" + " -F, --prefetch <num> Prefetch packet data in 64 byte multiples (default 1).\n" " -f, --flow_aware Enable flow aware scheduling.\n" + " -T, --input_ts Enable packet input timestamping.\n" " -v, --verbose Verbose output.\n" + " -V, --verbose_pkt Print debug information on every received packet.\n" " -h, --help Display help and exit.\n\n" "\n", DEFAULT_VEC_SIZE, DEFAULT_VEC_TMO, POOL_PKT_LEN); } @@ -1550,6 +1677,7 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) {"burst_rx", required_argument, NULL, 'b'}, {"rx_queues", required_argument, NULL, 'q'}, {"packet_copy", required_argument, NULL, 'p'}, + {"data_rd", required_argument, NULL, 'R'}, {"pool_per_if", required_argument, NULL, 'y'}, {"num_pkt", required_argument, NULL, 'n'}, {"num_vec", required_argument, NULL, 'w'}, @@ -1560,13 +1688,17 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) {"promisc_mode", no_argument, NULL, 'P'}, {"packet_len", required_argument, NULL, 'l'}, {"seg_len", required_argument, NULL, 'L'}, + {"prefetch", required_argument, NULL, 'F'}, {"flow_aware", no_argument, NULL, 'f'}, + {"input_ts", no_argument, NULL, 'T'}, {"verbose", no_argument, NULL, 'v'}, + {"verbose_pkt", no_argument, NULL, 'V'}, {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} }; - static const char *shortopts = "+c:t:a:i:m:o:r:d:s:e:k:g:G:I:b:q:p:y:n:l:L:w:x:z:M:uPfvh"; + static const char *shortopts = "+c:t:a:i:m:o:r:d:s:e:k:g:G:I:" + "b:q:p:R:y:n:l:L:w:x:z:M:F:uPfTvVh"; appl_args->time = 0; /* loop forever if time to run is 0 */ appl_args->accuracy = 1; /* get and print pps stats second */ @@ -1580,6 +1712,7 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) appl_args->burst_rx = 0; appl_args->rx_queues = 0; appl_args->verbose = 0; + appl_args->verbose_pkt = 0; appl_args->chksum = 0; /* don't use checksum offload by default */ appl_args->pool_per_if = 0; appl_args->num_pkt = 0; @@ -1592,7 +1725,10 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) appl_args->vec_size = 0; appl_args->vec_tmo_ns = 0; appl_args->flow_aware = 0; + appl_args->input_ts = 0; appl_args->num_prio = 0; + appl_args->prefetch = 1; + appl_args->data_rd = 0; while (1) { opt = getopt_long(argc, argv, shortopts, longopts, &long_index); @@ -1764,6 +1900,9 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) case 'p': appl_args->packet_copy = atoi(optarg); break; + case 'R': + appl_args->data_rd = atoi(optarg); + break; case 'y': appl_args->pool_per_if = atoi(optarg); break; @@ -1794,12 +1933,21 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) case 'z': appl_args->vec_tmo_ns = atoi(optarg); break; + case 'F': + appl_args->prefetch = atoi(optarg); + break; case 'f': appl_args->flow_aware = 1; break; + case 'T': + appl_args->input_ts = 1; + break; case 'v': appl_args->verbose = 1; break; + case 'V': + appl_args->verbose_pkt = 1; + break; case 'h': usage(argv[0]); exit(EXIT_SUCCESS); @@ -1835,16 +1983,13 @@ static void parse_args(int argc, char *argv[], appl_args_t *appl_args) appl_args->extra_feat = 0; if (appl_args->error_check || appl_args->chksum || - appl_args->packet_copy) + appl_args->packet_copy || appl_args->data_rd || appl_args->verbose_pkt) appl_args->extra_feat = 1; optind = 1; /* reset 'extern optind' from the getopt lib */ } -/* - * Print system and application info - */ -static void print_info(void) +static void print_options(void) { int i; appl_args_t *appl_args = &gbl_args->appl; @@ -1884,16 +2029,19 @@ static void print_info(void) "enabled" : "disabled"); printf("Flow aware: %s\n", appl_args->flow_aware ? "yes" : "no"); + printf("Input TS: %s\n", appl_args->input_ts ? "yes" : "no"); printf("Burst size: %i\n", appl_args->burst_rx); printf("RX queues per IF: %i\n", appl_args->rx_queues); printf("Number of pools: %i\n", appl_args->pool_per_if ? appl_args->if_count : 1); if (appl_args->extra_feat) { - printf("Extra features: %s%s%s\n", + printf("Extra features: %s%s%s%s%s\n", appl_args->error_check ? "error_check " : "", appl_args->chksum ? "chksum " : "", - appl_args->packet_copy ? "packet_copy" : ""); + appl_args->packet_copy ? "packet_copy " : "", + appl_args->data_rd ? "data_rd" : "", + appl_args->verbose_pkt ? "verbose_pkt" : ""); } printf("Num worker threads: %i\n", appl_args->num_workers); @@ -1906,11 +2054,14 @@ static void print_info(void) else printf("group: ODP_SCHED_GROUP_WORKER\n"); - printf("Packets per pool: %u\n", gbl_args->num_pkt); - printf("Packet length: %u\n", gbl_args->pkt_len); - printf("Segment length: %u\n", gbl_args->seg_len); - printf("Vectors per pool: %u\n", gbl_args->vector_num); - printf("Vector size: %u\n", gbl_args->vector_max_size); + printf("Packets per pool: %u\n", appl_args->num_pkt); + printf("Packet length: %u\n", appl_args->packet_len); + printf("Segment length: %u\n", appl_args->seg_len == UINT32_MAX ? 0 : + appl_args->seg_len); + printf("Read data: %u bytes\n", appl_args->data_rd * 8); + printf("Prefetch data %u bytes\n", appl_args->prefetch * 64); + printf("Vectors per pool: %u\n", appl_args->num_vec); + printf("Vector size: %u\n", appl_args->vec_size); printf("Priority per IF: "); for (i = 0; i < appl_args->if_count; i++) @@ -2099,8 +2250,7 @@ int main(int argc, char *argv[]) gbl_args->appl.num_workers = num_workers; - /* Print application information */ - print_info(); + print_options(); for (i = 0; i < num_workers; i++) gbl_args->thread_args[i].thr_idx = i; @@ -2148,6 +2298,12 @@ int main(int argc, char *argv[]) printf("\nWarning: Segment length requested %d configured %d\n", gbl_args->appl.seg_len, seg_len); + if (seg_len < gbl_args->appl.data_rd * 8) { + ODPH_ERR("Requested data read length %u exceeds maximum segment length %u\n", + gbl_args->appl.data_rd * 8, seg_len); + return -1; + } + /* zero means default number of packets */ if (gbl_args->appl.num_pkt == 0) num_pkt = DEFAULT_NUM_PKT; @@ -2170,6 +2326,11 @@ int main(int argc, char *argv[]) gbl_args->pkt_len = pkt_len; gbl_args->seg_len = seg_len; + printf("Resulting pool parameter values:\n"); + printf("Packets per pool: %u\n", num_pkt); + printf("Packet length: %u\n", pkt_len); + printf("Segment length: %u\n", seg_len); + /* Create packet pool */ odp_pool_param_init(¶ms); params.pkt.seg_len = seg_len; @@ -2210,6 +2371,10 @@ int main(int argc, char *argv[]) gbl_args->vector_num = params.vector.num; gbl_args->vector_max_size = params.vector.max_size; + /* Print resulting values */ + printf("Vectors per pool: %u\n", gbl_args->vector_num); + printf("Vector size: %u\n", gbl_args->vector_max_size); + for (i = 0; i < num_vec_pools; i++) { vec_pool_tbl[i] = odp_pool_create("vector pool", ¶ms); @@ -2223,6 +2388,8 @@ int main(int argc, char *argv[]) } } + printf("\n"); + bind_workers(); odp_schedule_config_init(&sched_config); diff --git a/test/performance/odp_lock_perf.c b/test/performance/odp_lock_perf.c index c12f8c950..0f78db3b8 100644 --- a/test/performance/odp_lock_perf.c +++ b/test/performance/odp_lock_perf.c @@ -5,6 +5,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_lock_perf.c + * + * Performance test application for lock APIs + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include <stdio.h> #include <string.h> #include <stdint.h> diff --git a/test/performance/odp_mem_perf.c b/test/performance/odp_mem_perf.c index 56a3cdf9a..241128b1f 100644 --- a/test/performance/odp_mem_perf.c +++ b/test/performance/odp_mem_perf.c @@ -5,6 +5,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_mem_perf.c + * + * Test application for measuring memory system bandwidth + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include <stdio.h> #include <string.h> #include <stdint.h> diff --git a/test/performance/odp_packet_gen.c b/test/performance/odp_packet_gen.c index 247ea2eb6..c88535791 100644 --- a/test/performance/odp_packet_gen.c +++ b/test/performance/odp_packet_gen.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_packet_gen.c + * + * Performance optimized packet generator application + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + /* enable usleep */ #ifndef _GNU_SOURCE #define _GNU_SOURCE diff --git a/test/performance/odp_pktio_ordered.c b/test/performance/odp_pktio_ordered.c index bd43ad53d..6177a8160 100644 --- a/test/performance/odp_pktio_ordered.c +++ b/test/performance/odp_pktio_ordered.c @@ -5,9 +5,11 @@ */ /** - * @file + * @example odp_pktio_ordered.c * - * @example odp_pktio_ordered.c ODP ordered pktio test application + * Test application for ordered packet IO + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ /** enable strtok */ diff --git a/test/performance/odp_pktio_perf.c b/test/performance/odp_pktio_perf.c index 06620fd27..4cfeb50cf 100644 --- a/test/performance/odp_pktio_perf.c +++ b/test/performance/odp_pktio_perf.c @@ -2,14 +2,16 @@ * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @example odp_pktio_perf.c * - * ODP Packet IO basic performance test application. - * - * Runs a number of transmit and receive workers on separate cores, the - * transmitters generate packets at a defined rate and the receivers consume - * them. Generated packets are UDP and each packet is marked with a magic - * number in the UDP payload allowing receiver to distinguish them from other - * traffic. + * Packet IO basic performance test application. Runs a number of transmit and + * receive workers on separate cores, the transmitters generate packets at a + * defined rate and the receivers consume them. Generated packets are UDP and + * each packet is marked with a magic number in the UDP payload allowing + * receiver to distinguish them from other traffic. * * Each test iteration runs for a fixed period, at the end of the iteration * it is verified that the number of packets transmitted was as expected and @@ -19,6 +21,7 @@ * determine the maximum rate at which no packet loss occurs. Alternatively * a single packet rate can be specified on the command line. * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include <odp_api.h> diff --git a/test/performance/odp_pool_latency.c b/test/performance/odp_pool_latency.c new file mode 100644 index 000000000..6b964e773 --- /dev/null +++ b/test/performance/odp_pool_latency.c @@ -0,0 +1,1382 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2024 Nokia + */ + +/** + * @example odp_pool_latency.c + * + * Pool latency tester. Allocate from different kind of pools with a varying set of configurations + * and record latencies. + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include <inttypes.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> + +#include <odp_api.h> +#include <odp/helper/odph_api.h> + +#define PROG_NAME "odp_pool_latency" +#define DELIMITER "," +#define ALLOC '+' +#define FREE '-' +#define TOP 't' +#define BOTTOM 'b' +#define DELAY 'd' + +enum { + BUFFER = 0U, + PACKET, + TMO, + VECTOR +}; + +enum { + SINGLE = 0U, + MANY +}; + +#define DEF_ALLOC 1U +#define DEF_FREE 1U +#define DEF_DIR TOP +#define DEF_TYPE BUFFER +#define DEF_CNT 32768U +#define DEF_SIZE 1024U +#define DEF_POLICY MANY +#define DEF_ROUNDS 100000U +#define DEF_IGNORE 0U +#define DEF_WORKERS 1U +#define DEF_UA_SIZE 0U + +#define MAX_PATTERN_LEN 32U +#define MAX_WORKERS ((uint32_t)(ODP_THREAD_COUNT_MAX - 1)) +#define MAX_RETRIES 10U + +#define COND_MIN(a, b) ((a) > 0U ? ODPH_MIN((a), (b)) : (b)) +#define UA_DATA 0xAA + +ODP_STATIC_ASSERT(MAX_PATTERN_LEN < UINT8_MAX, "Too long pattern length"); + +typedef struct { + uint32_t num_evs_buf; + uint32_t num_evs_pkt; + uint32_t num_evs_tmo; + uint32_t num_evs_vec; + uint32_t data_size_buf; + uint32_t data_size_pkt; + uint32_t data_size_vec; + uint32_t cache_size_buf; + uint32_t cache_size_pkt; + uint32_t cache_size_tmo; + uint32_t cache_size_vec; +} dynamic_defs_t; + +typedef enum { + PRS_OK, + PRS_NOK, + PRS_TERM +} parse_result_t; + +typedef struct { + uint64_t tot_tm; + uint64_t alloc_tm; + uint64_t max_alloc_tm; + uint64_t min_alloc_tm; + uint64_t max_alloc_rnd; + uint64_t min_alloc_rnd; + uint64_t alloc_cnt; + uint64_t alloc_b_cnt; + uint64_t uarea_tm; + uint64_t max_uarea_tm; + uint64_t min_uarea_tm; + uint64_t max_uarea_rnd; + uint64_t min_uarea_rnd; + uint64_t free_tm; + uint64_t max_free_tm; + uint64_t min_free_tm; + uint64_t max_free_rnd; + uint64_t min_free_rnd; + uint64_t free_b_cnt; + uint64_t reallocs; + uint64_t alloc_errs; + uint64_t pattern_errs; + uint8_t max_alloc_pt; + uint8_t min_alloc_pt; + uint8_t max_uarea_pt; + uint8_t min_uarea_pt; + uint8_t max_free_pt; + uint8_t min_free_pt; +} stats_t; + +typedef struct { + uint32_t val; + uint8_t op; + uint8_t opt; +} alloc_elem_t; + +typedef struct prog_config_s prog_config_t; + +typedef struct ODP_ALIGNED_CACHE { + stats_t stats; + odp_pool_t pool; + void *data; + prog_config_t *prog_config; + odp_shm_t shm; + uint32_t data_size; + uint32_t uarea_size; +} worker_config_t; + +typedef uint32_t (*alloc_fn_t)(worker_config_t *config, void *data, uint32_t idx, uint32_t num, + uint64_t round, uint8_t pattern, odp_bool_t is_saved); +typedef void (*free_fn_t)(void *data, uint32_t idx, uint32_t num, stats_t *stats, + uint64_t round, uint8_t pattern, odp_bool_t is_saved); + +typedef struct prog_config_s { + odph_thread_t thread_tbl[MAX_WORKERS]; + worker_config_t worker_config[MAX_WORKERS]; + alloc_elem_t alloc_elems[MAX_PATTERN_LEN]; + dynamic_defs_t dyn_defs; + odp_instance_t odp_instance; + odp_cpumask_t worker_mask; + odp_barrier_t init_barrier; + odp_barrier_t term_barrier; + alloc_fn_t alloc_fn; + free_fn_t free_fn; + int64_t cache_size; + uint32_t num_data_elems; + uint32_t seg_len; + uint32_t handle_size; + uint32_t num_evs; + uint32_t data_size; + uint32_t num_rounds; + uint32_t num_ignore; + uint32_t num_workers; + uint32_t uarea_size; + uint8_t num_elems; + uint8_t type; + uint8_t policy; +} prog_config_t; + +static prog_config_t *prog_conf; + +static void init_config(prog_config_t *config) +{ + alloc_elem_t *alloc_elem; + odp_pool_capability_t capa; + odp_pool_param_t param; + worker_config_t *worker; + + memset(config, 0, sizeof(*config)); + alloc_elem = &config->alloc_elems[0]; + alloc_elem->val = DEF_ALLOC; + alloc_elem->op = ALLOC; + alloc_elem = &config->alloc_elems[1]; + alloc_elem->val = DEF_FREE; + alloc_elem->op = FREE; + alloc_elem->opt = DEF_DIR; + config->num_elems = 2U; + + if (odp_pool_capability(&capa) == 0) { + config->dyn_defs.num_evs_buf = COND_MIN(capa.buf.max_num, DEF_CNT); + config->dyn_defs.num_evs_pkt = COND_MIN(capa.pkt.max_num, DEF_CNT); + config->dyn_defs.num_evs_tmo = COND_MIN(capa.tmo.max_num, DEF_CNT); + config->dyn_defs.num_evs_vec = COND_MIN(capa.vector.max_num, DEF_CNT); + config->dyn_defs.data_size_buf = COND_MIN(capa.buf.max_size, DEF_SIZE); + config->dyn_defs.data_size_pkt = COND_MIN(capa.pkt.max_len, DEF_SIZE); + config->dyn_defs.data_size_vec = COND_MIN(capa.vector.max_size, DEF_SIZE); + odp_pool_param_init(¶m); + config->dyn_defs.cache_size_buf = param.buf.cache_size; + config->dyn_defs.cache_size_pkt = param.pkt.cache_size; + config->dyn_defs.cache_size_tmo = param.tmo.cache_size; + config->dyn_defs.cache_size_vec = param.vector.cache_size; + } + + config->cache_size = -1; + config->num_rounds = DEF_ROUNDS; + config->num_ignore = DEF_IGNORE; + config->num_workers = DEF_WORKERS; + config->uarea_size = DEF_UA_SIZE; + config->type = DEF_TYPE; + config->policy = DEF_POLICY; + + for (uint32_t i = 0U; i < MAX_WORKERS; ++i) { + worker = &config->worker_config[i]; + worker->stats.min_alloc_tm = UINT64_MAX; + worker->stats.min_uarea_tm = UINT64_MAX; + worker->stats.min_free_tm = UINT64_MAX; + worker->pool = ODP_POOL_INVALID; + worker->shm = ODP_SHM_INVALID; + } +} + +static void parse_burst_pattern(prog_config_t *config, const char *optarg) +{ + char *tmp_str = strdup(optarg), *tmp, op, opt; + uint8_t num_elems = 0U; + alloc_elem_t *elem; + uint32_t val; + int ret; + + if (tmp_str == NULL) + return; + + tmp = strtok(tmp_str, DELIMITER); + + while (tmp && num_elems < MAX_PATTERN_LEN) { + elem = &config->alloc_elems[num_elems]; + ret = sscanf(tmp, "%c%u%c", &op, &val, &opt); + + if (ret == 2 || ret == 3) { + if (op == ALLOC || (op == FREE && (opt == TOP || opt == BOTTOM)) || + op == DELAY) { + if (op == FREE) + elem->opt = opt; + + elem->val = val; + elem->op = op; + ++num_elems; + } + } + + tmp = strtok(NULL, DELIMITER); + } + + free(tmp_str); + config->num_elems = num_elems; +} + +static void print_usage(const dynamic_defs_t *dyn_defs) +{ + printf("\n" + "Pool latency tester. Allocate from different kind of pools with a varying set of\n" + "configurations and record latencies.\n" + "\n" + "Usage: " PROG_NAME " [OPTIONS]\n"); + printf("\n" + " E.g. " PROG_NAME "\n" + " " PROG_NAME " -b %c7" DELIMITER "%c1%c" DELIMITER "%c3" DELIMITER "%c9%c\n", + ALLOC, FREE, TOP, ALLOC, FREE, BOTTOM); + printf(" " PROG_NAME " -b %c10" DELIMITER "%c1000" DELIMITER "%c10%c -t 1 -d 2048 " + "-p 0 -w 64\n", ALLOC, DELAY, FREE, TOP); + printf("\n" + "Optional OPTIONS:\n" + "\n" + " -b, --burst_pattern Burst pattern for allocations, frees and delays per round,\n" + " delimited by '%s', no spaces. Allocations are indicated\n" + " with a '%c' prefix, frees with a '%c' prefix. The location\n" + " of frees are indicated from the top of a previously\n" + " allocated array of events with a '%c' suffix and from the\n" + " bottom with a '%c' suffix. Delays are indicated with a\n" + " '%c' prefix, followed by a delay in nanoseconds.\n" + " Allocations and frees should be equal in the aggregate and\n" + " frees should never outnumber allocations at any instant.\n" + " '%c%u%s%c%u%c' by default. Maximum pattern length is %u.\n" + " -t, --type Pool type. %u by default.\n" + " 0: buffer\n" + " 1: packet\n" + " 2: timeout\n" + " 3: vector\n" + " -e, --event_count Number of events. Defaults:\n" + " buffer: %u\n" + " packet: %u\n" + " timeout: %u\n" + " vector: %u\n" + " -d, --data_size Data size in bytes, ignored in case of timeout pools, with\n" + " vector pools, defines the vector size.\n" + " Defaults:\n" + " buffer: %u\n" + " packet: %u\n" + " vector: %u\n" + " -p, --policy Pool allocation policy. %u by default.\n" + " Policies:\n" + " 0: One pool shared by workers\n" + " 1: One pool per worker\n" + " -r, --round_count Number of rounds to run. %u by default.\n" + " -i, --ignore_rounds Ignore an amount of initial rounds. %u by default.\n" + " -c, --worker_count Number of workers. %u by default.\n" + " -C, --cache_size Maximum cache size for pools. Defaults:\n" + " buffer: %u\n" + " packet: %u\n" + " timeout: %u\n" + " vector: %u\n" + " -w, --write_uarea Write data to allocated event user areas. 0 bytes disables\n" + " user area write. %u by default.\n" + " -h, --help This help.\n" + "\n", DELIMITER, ALLOC, FREE, TOP, BOTTOM, DELAY, ALLOC, DEF_ALLOC, DELIMITER, FREE, + DEF_FREE, DEF_DIR, MAX_PATTERN_LEN, DEF_TYPE, dyn_defs->num_evs_buf, + dyn_defs->num_evs_pkt, dyn_defs->num_evs_tmo, dyn_defs->num_evs_vec, + dyn_defs->data_size_buf, dyn_defs->data_size_pkt, dyn_defs->data_size_vec, + DEF_POLICY, DEF_ROUNDS, DEF_IGNORE, DEF_WORKERS, dyn_defs->cache_size_buf, + dyn_defs->cache_size_pkt, dyn_defs->cache_size_tmo, dyn_defs->cache_size_vec, + DEF_UA_SIZE); +} + +static parse_result_t check_options(prog_config_t *config) +{ + odp_pool_capability_t pool_capa; + uint32_t max_workers, num_pools; + alloc_elem_t *elem; + int64_t num_tot = 0; + odp_shm_capability_t shm_capa; + uint64_t shm_size; + + if (config->type != BUFFER && config->type != PACKET && config->type != TMO && + config->type != VECTOR) { + ODPH_ERR("Invalid pool type: %u\n", config->type); + return PRS_NOK; + } + + if (odp_pool_capability(&pool_capa) < 0) { + ODPH_ERR("Error querying pool capabilities\n"); + return PRS_NOK; + } + + max_workers = ODPH_MIN(MAX_WORKERS, (uint32_t)odp_cpumask_default_worker(NULL, 0)); + + if (config->num_workers == 0U || config->num_workers > max_workers) { + ODPH_ERR("Invalid worker count: %u (min: 1, max: %u)\n", config->num_workers, + max_workers); + return PRS_NOK; + } + + (void)odp_cpumask_default_worker(&config->worker_mask, config->num_workers); + num_pools = config->policy == SINGLE ? 1U : config->num_workers; + + if (config->type == BUFFER) { + if (config->num_evs == 0U) + config->num_evs = config->dyn_defs.num_evs_buf; + + if (config->data_size == 0U) + config->data_size = config->dyn_defs.data_size_buf; + + if (config->cache_size == -1) + config->cache_size = config->dyn_defs.cache_size_buf; + + if (config->num_evs > pool_capa.buf.max_num) { + ODPH_ERR("Invalid event count: %u (max: %u)\n", config->num_evs, + pool_capa.buf.max_num); + return PRS_NOK; + } + + if (config->data_size > pool_capa.buf.max_size) { + ODPH_ERR("Invalid data size: %u (max: %u)\n", config->data_size, + pool_capa.buf.max_size); + return PRS_NOK; + } + + if (config->cache_size < pool_capa.buf.min_cache_size || + config->cache_size > pool_capa.buf.max_cache_size) { + ODPH_ERR("Invalid cache size: %" PRIi64 " (min: %u, max: %u)\n", + config->cache_size, pool_capa.buf.min_cache_size, + pool_capa.buf.max_cache_size); + return PRS_NOK; + } + + if (num_pools > pool_capa.buf.max_pools) { + ODPH_ERR("Invalid pool count: %u (max: %u)\n", num_pools, + pool_capa.buf.max_pools); + return PRS_NOK; + } + + config->handle_size = sizeof(odp_buffer_t); + config->uarea_size = ODPH_MIN(config->uarea_size, pool_capa.buf.max_uarea_size); + } else if (config->type == PACKET) { + if (config->num_evs == 0U) + config->num_evs = config->dyn_defs.num_evs_pkt; + + if (config->data_size == 0U) + config->data_size = config->dyn_defs.data_size_pkt; + + if (config->cache_size == -1) + config->cache_size = config->dyn_defs.cache_size_pkt; + + if (config->num_evs > pool_capa.pkt.max_num) { + ODPH_ERR("Invalid event count: %u (max: %u)\n", config->num_evs, + pool_capa.pkt.max_num); + return PRS_NOK; + } + + if (config->data_size > pool_capa.pkt.max_len) { + ODPH_ERR("Invalid data size: %u (max: %u)\n", config->data_size, + pool_capa.pkt.max_len); + return PRS_NOK; + } + + if (config->cache_size < pool_capa.pkt.min_cache_size || + config->cache_size > pool_capa.pkt.max_cache_size) { + ODPH_ERR("Invalid cache size: %" PRIi64 " (min: %u, max: %u)\n", + config->cache_size, pool_capa.pkt.min_cache_size, + pool_capa.pkt.max_cache_size); + return PRS_NOK; + } + + if (num_pools > pool_capa.pkt.max_pools) { + ODPH_ERR("Invalid pool count: %u (max: %u)\n", num_pools, + pool_capa.pkt.max_pools); + return PRS_NOK; + } + + config->seg_len = pool_capa.pkt.max_seg_len > config->data_size ? + config->data_size : pool_capa.pkt.max_seg_len; + config->handle_size = sizeof(odp_packet_t); + config->uarea_size = ODPH_MIN(config->uarea_size, pool_capa.pkt.max_uarea_size); + } else if (config->type == TMO) { + if (config->num_evs == 0U) + config->num_evs = config->dyn_defs.num_evs_tmo; + + if (config->cache_size == -1) + config->cache_size = config->dyn_defs.cache_size_tmo; + + if (config->num_evs > pool_capa.tmo.max_num) { + ODPH_ERR("Invalid event count: %u (max: %u)\n", config->num_evs, + pool_capa.tmo.max_num); + return PRS_NOK; + } + + if (config->cache_size < pool_capa.tmo.min_cache_size || + config->cache_size > pool_capa.tmo.max_cache_size) { + ODPH_ERR("Invalid cache size: %" PRIi64 " (min: %u, max: %u)\n", + config->cache_size, pool_capa.tmo.min_cache_size, + pool_capa.tmo.max_cache_size); + return PRS_NOK; + } + + if (num_pools > pool_capa.tmo.max_pools) { + ODPH_ERR("Invalid pool count: %u (max: %u)\n", num_pools, + pool_capa.tmo.max_pools); + return PRS_NOK; + } + + config->handle_size = sizeof(odp_timeout_t); + config->uarea_size = ODPH_MIN(config->uarea_size, pool_capa.tmo.max_uarea_size); + } else { + if (config->num_evs == 0U) + config->num_evs = config->dyn_defs.num_evs_vec; + + if (config->data_size == 0U) + config->data_size = config->dyn_defs.data_size_vec; + + if (config->cache_size == -1) + config->cache_size = config->dyn_defs.cache_size_vec; + + if (config->num_evs > pool_capa.vector.max_num) { + ODPH_ERR("Invalid event count: %u (max: %u)\n", config->num_evs, + pool_capa.vector.max_num); + return PRS_NOK; + } + + if (config->data_size > pool_capa.vector.max_size) { + ODPH_ERR("Invalid vector size: %u (max: %u)\n", config->data_size, + pool_capa.vector.max_size); + return PRS_NOK; + } + + if (config->cache_size < pool_capa.vector.min_cache_size || + config->cache_size > pool_capa.vector.max_cache_size) { + ODPH_ERR("Invalid cache size: %" PRIi64 " (min: %u, max: %u)\n", + config->cache_size, pool_capa.vector.min_cache_size, + pool_capa.vector.max_cache_size); + return PRS_NOK; + } + + if (num_pools > pool_capa.vector.max_pools) { + ODPH_ERR("Invalid pool count: %u (max: %u)\n", num_pools, + pool_capa.vector.max_pools); + return PRS_NOK; + } + + config->handle_size = sizeof(odp_packet_vector_t); + config->uarea_size = ODPH_MIN(config->uarea_size, pool_capa.vector.max_uarea_size); + } + + if (config->num_elems == 0U) { + ODPH_ERR("Invalid burst pattern, no elements\n"); + return PRS_NOK; + } + + for (uint8_t i = 0U; i < config->num_elems; ++i) { + elem = &config->alloc_elems[i]; + + if (elem->op == ALLOC) + num_tot += elem->val; + else if (elem->op == FREE) + num_tot -= elem->val; + + if (num_tot < 0) { + ODPH_ERR("Invalid burst pattern, frees exceed allocations " + "instantaneously\n"); + return PRS_NOK; + } + + config->num_data_elems += (elem->op == ALLOC ? elem->val : 0U); + } + + if (num_tot != 0) { + ODPH_ERR("Invalid burst pattern, cumulative sum not zero: %" PRId64 "\n", num_tot); + return PRS_NOK; + } + + if (odp_shm_capability(&shm_capa) < 0) { + ODPH_ERR("Error querying SHM capabilities\n"); + return PRS_NOK; + } + + if (shm_capa.max_blocks < config->num_workers + 1U) { + ODPH_ERR("Invalid amount of SHM blocks: %u (max: %u)\n", config->num_workers + 1U, + shm_capa.max_blocks); + return PRS_NOK; + } + + shm_size = (uint64_t)config->num_data_elems * config->handle_size; + + if (shm_capa.max_size != 0U && shm_size > shm_capa.max_size) { + ODPH_ERR("Invalid total SHM block size: %" PRIu64 " (max: %" PRIu64 ")\n", + shm_size, shm_capa.max_size); + return PRS_NOK; + } + + if (config->policy != SINGLE && config->policy != MANY) { + ODPH_ERR("Invalid pool policy: %u\n", config->policy); + return PRS_NOK; + } + + if (config->num_rounds == 0U) { + ODPH_ERR("Invalid round count: %u (min: 1)\n", config->num_rounds); + return PRS_NOK; + } + + if (config->num_ignore >= config->num_rounds) { + ODPH_ERR("Invalid round ignorance count: %u (max: %u)\n", config->num_ignore, + config->num_rounds - 1U); + return PRS_NOK; + } + + return PRS_OK; +} + +static parse_result_t parse_options(int argc, char **argv, prog_config_t *config) +{ + int opt, long_index; + + static const struct option longopts[] = { + { "burst_pattern", required_argument, NULL, 'b' }, + { "type", required_argument, NULL, 't' }, + { "event_count", required_argument, NULL, 'e' }, + { "data_size", required_argument, NULL, 'd' }, + { "policy", required_argument, NULL, 'p' }, + { "round_count", required_argument, NULL, 'r' }, + { "ignore_rounds", required_argument, NULL, 'i' }, + { "worker_count", required_argument, NULL, 'c' }, + { "cache_size", required_argument, NULL, 'C' }, + { "write_uarea", required_argument, NULL, 'w' }, + { "help", no_argument, NULL, 'h' }, + { NULL, 0, NULL, 0 } + }; + + static const char *shortopts = "b:t:e:d:p:r:i:c:C:w:h"; + + init_config(config); + + while (1) { + opt = getopt_long(argc, argv, shortopts, longopts, &long_index); + + if (opt == -1) + break; + + switch (opt) { + case 'b': + parse_burst_pattern(config, optarg); + break; + case 't': + config->type = atoi(optarg); + break; + case 'e': + config->num_evs = atoi(optarg); + break; + case 'd': + config->data_size = atoi(optarg); + break; + case 'p': + config->policy = atoi(optarg); + break; + case 'r': + config->num_rounds = atoi(optarg); + break; + case 'i': + config->num_ignore = atoi(optarg); + break; + case 'c': + config->num_workers = atoi(optarg); + break; + case 'C': + config->cache_size = atoi(optarg); + break; + case 'w': + config->uarea_size = atoi(optarg); + break; + case 'h': + print_usage(&config->dyn_defs); + return PRS_TERM; + case '?': + default: + print_usage(&config->dyn_defs); + return PRS_NOK; + } + } + + return check_options(config); +} + +static inline void save_alloc_stats(odp_time_t t1, odp_time_t t2, uint32_t num_alloc, + uint64_t round, uint8_t pattern, stats_t *stats) +{ + const uint64_t tm_diff = odp_time_diff_ns(t2, t1); + + stats->alloc_tm += tm_diff; + stats->alloc_cnt += num_alloc; + ++stats->alloc_b_cnt; + + if (tm_diff > stats->max_alloc_tm) { + stats->max_alloc_tm = tm_diff; + stats->max_alloc_rnd = round; + stats->max_alloc_pt = pattern; + } + + if (tm_diff < stats->min_alloc_tm) { + stats->min_alloc_tm = tm_diff; + stats->min_alloc_rnd = round; + stats->min_alloc_pt = pattern; + } +} + +static inline void write_to_uarea(uint8_t *data, uint32_t size) +{ + memset(data, UA_DATA, size); +} + +static inline void save_uarea_stats(odp_time_t t1, odp_time_t t2, uint64_t round, uint8_t pattern, + stats_t *stats) +{ + const uint64_t tm_diff = odp_time_diff_ns(t2, t1); + + stats->uarea_tm += tm_diff; + + if (tm_diff > stats->max_uarea_tm) { + stats->max_uarea_tm = tm_diff; + stats->max_uarea_rnd = round; + stats->max_uarea_pt = pattern; + } + + if (tm_diff < stats->min_uarea_tm) { + stats->min_uarea_tm = tm_diff; + stats->min_uarea_rnd = round; + stats->min_uarea_pt = pattern; + } +} + +static inline void save_free_stats(odp_time_t t1, odp_time_t t2, uint64_t round, uint8_t pattern, + stats_t *stats) +{ + const uint64_t tm_diff = odp_time_diff_ns(t2, t1); + + stats->free_tm += tm_diff; + ++stats->free_b_cnt; + + if (tm_diff > stats->max_free_tm) { + stats->max_free_tm = tm_diff; + stats->max_free_rnd = round; + stats->max_free_pt = pattern; + } + + if (tm_diff < stats->min_free_tm) { + stats->min_free_tm = tm_diff; + stats->min_free_rnd = round; + stats->min_free_pt = pattern; + } + + stats->max_free_tm = ODPH_MAX(tm_diff, stats->max_free_tm); + stats->min_free_tm = ODPH_MIN(tm_diff, stats->min_free_tm); +} + +static uint32_t allocate_buffers(worker_config_t *config, void *data, uint32_t idx, uint32_t num, + uint64_t round, uint8_t pattern, odp_bool_t is_saved) +{ + odp_time_t t1, t2; + odp_pool_t pool = config->pool; + uint32_t retries = MAX_RETRIES; + odp_buffer_t *bufs = &((odp_buffer_t *)data)[idx]; + uint32_t num_alloc, num_tot = 0U; + int ret; + stats_t *stats = &config->stats; + + while (retries-- > 0U && num_tot < num) { + num_alloc = num - num_tot; + t1 = odp_time_local_strict(); + ret = odp_buffer_alloc_multi(pool, &bufs[num_tot], num_alloc); + t2 = odp_time_local_strict(); + + if (odp_unlikely(ret < 0)) { + ++stats->alloc_errs; + break; + } + + if (odp_unlikely((uint32_t)ret < num_alloc)) + ++stats->reallocs; + + num_tot += ret; + + if (odp_likely(is_saved)) + save_alloc_stats(t1, t2, ret, round, pattern, stats); + } + + if (config->uarea_size > 0U) { + t1 = odp_time_local_strict(); + + for (uint32_t i = 0U; i < num_tot; ++i) + write_to_uarea(odp_buffer_user_area(bufs[i]), config->uarea_size); + + t2 = odp_time_local_strict(); + + if (odp_likely(is_saved)) + save_uarea_stats(t1, t2, round, pattern, stats); + } + + return num_tot; +} + +static void free_buffers(void *data, uint32_t idx, uint32_t num, stats_t *stats, uint64_t round, + uint8_t pattern, odp_bool_t is_saved) +{ + odp_time_t t1, t2; + odp_buffer_t *bufs = &((odp_buffer_t *)data)[idx]; + + t1 = odp_time_local_strict(); + odp_buffer_free_multi(bufs, num); + t2 = odp_time_local_strict(); + + if (odp_likely(is_saved)) + save_free_stats(t1, t2, round, pattern, stats); +} + +static uint32_t allocate_packets(worker_config_t *config, void *data, uint32_t idx, uint32_t num, + uint64_t round, uint8_t pattern, odp_bool_t is_saved) +{ + odp_time_t t1, t2; + odp_pool_t pool = config->pool; + uint32_t retries = MAX_RETRIES, data_size = config->data_size; + odp_packet_t *pkts = &((odp_packet_t *)data)[idx]; + uint32_t num_alloc, num_tot = 0U; + int ret; + stats_t *stats = &config->stats; + + while (retries-- > 0U && num_tot < num) { + num_alloc = num - num_tot; + t1 = odp_time_local_strict(); + ret = odp_packet_alloc_multi(pool, data_size, &pkts[num_tot], num_alloc); + t2 = odp_time_local_strict(); + + if (odp_unlikely(ret < 0)) { + ++stats->alloc_errs; + break; + } + + if (odp_unlikely((uint32_t)ret < num_alloc)) + ++stats->reallocs; + + num_tot += ret; + + if (odp_likely(is_saved)) + save_alloc_stats(t1, t2, ret, round, pattern, stats); + } + + if (config->uarea_size > 0U) { + t1 = odp_time_local_strict(); + + for (uint32_t i = 0U; i < num_tot; ++i) + write_to_uarea(odp_packet_user_area(pkts[i]), config->uarea_size); + + t2 = odp_time_local_strict(); + + if (odp_likely(is_saved)) + save_uarea_stats(t1, t2, round, pattern, stats); + } + + return num_tot; +} + +static void free_packets(void *data, uint32_t idx, uint32_t num, stats_t *stats, uint64_t round, + uint8_t pattern, odp_bool_t is_saved) +{ + odp_time_t t1, t2; + odp_packet_t *pkts = &((odp_packet_t *)data)[idx]; + + t1 = odp_time_local_strict(); + odp_packet_free_multi(pkts, num); + t2 = odp_time_local_strict(); + + if (odp_likely(is_saved)) + save_free_stats(t1, t2, round, pattern, stats); +} + +static uint32_t allocate_timeouts(worker_config_t *config, void *data, uint32_t idx, uint32_t num, + uint64_t round, uint8_t pattern, odp_bool_t is_saved) +{ + odp_time_t t1, t2; + odp_pool_t pool = config->pool; + uint32_t retries = MAX_RETRIES; + odp_timeout_t *tmos = &((odp_timeout_t *)data)[idx]; + uint32_t num_alloc, num_tot = 0U; + int ret; + stats_t *stats = &config->stats; + + while (retries-- > 0U && num_tot < num) { + num_alloc = num - num_tot; + t1 = odp_time_local_strict(); + ret = odp_timeout_alloc_multi(pool, &tmos[num_tot], num_alloc); + t2 = odp_time_local_strict(); + + if (odp_unlikely(ret < 0)) { + ++stats->alloc_errs; + break; + } + + if (odp_unlikely((uint32_t)ret < num_alloc)) + ++stats->reallocs; + + num_tot += ret; + + if (odp_likely(is_saved)) + save_alloc_stats(t1, t2, ret, round, pattern, stats); + } + + if (config->uarea_size > 0U) { + t1 = odp_time_local_strict(); + + for (uint32_t i = 0U; i < num_tot; ++i) + write_to_uarea(odp_timeout_user_area(tmos[i]), config->uarea_size); + + t2 = odp_time_local_strict(); + + if (odp_likely(is_saved)) + save_uarea_stats(t1, t2, round, pattern, stats); + } + + return num_tot; +} + +static void free_timeouts(void *data, uint32_t idx, uint32_t num, stats_t *stats, uint64_t round, + uint8_t pattern, odp_bool_t is_saved) +{ + odp_time_t t1, t2; + odp_timeout_t *tmos = &((odp_timeout_t *)data)[idx]; + + t1 = odp_time_local_strict(); + odp_timeout_free_multi(tmos, num); + t2 = odp_time_local_strict(); + + if (odp_likely(is_saved)) + save_free_stats(t1, t2, round, pattern, stats); +} + +static uint32_t allocate_vectors(worker_config_t *config, void *data, uint32_t idx, uint32_t num, + uint64_t round, uint8_t pattern, odp_bool_t is_saved) +{ + odp_time_t t1, t2; + odp_pool_t pool = config->pool; + uint32_t num_tot = 0U; + odp_packet_vector_t *vecs = &((odp_packet_vector_t *)data)[idx], vec; + stats_t *stats = &config->stats; + + t1 = odp_time_local_strict(); + + for (uint32_t i = 0U; i < num; ++i) { + vec = odp_packet_vector_alloc(pool); + + if (odp_unlikely(vec == ODP_PACKET_VECTOR_INVALID)) + break; + + vecs[num_tot++] = vec; + } + + t2 = odp_time_local_strict(); + + if (odp_unlikely(num_tot == 0)) + ++stats->alloc_errs; + else if (odp_likely(is_saved)) + save_alloc_stats(t1, t2, num_tot, round, pattern, stats); + + if (config->uarea_size > 0U) { + t1 = odp_time_local_strict(); + + for (uint32_t i = 0U; i < num_tot; ++i) + write_to_uarea(odp_packet_vector_user_area(vecs[i]), config->uarea_size); + + t2 = odp_time_local_strict(); + + if (odp_likely(is_saved)) + save_uarea_stats(t1, t2, round, pattern, stats); + } + + return num_tot; +} + +static void free_vectors(void *data, uint32_t idx, uint32_t num, stats_t *stats, uint64_t round, + uint8_t pattern, odp_bool_t is_saved) +{ + odp_time_t t1, t2; + odp_packet_vector_t *vecs = &((odp_packet_vector_t *)data)[idx]; + + t1 = odp_time_local_strict(); + + for (uint32_t i = 0U; i < num; ++i) + odp_packet_vector_free(vecs[i]); + + t2 = odp_time_local_strict(); + + if (odp_likely(is_saved)) + save_free_stats(t1, t2, round, pattern, stats); +} + +static odp_pool_t create_pool(const char *name, const odp_pool_param_t *params, uint8_t policy) +{ + static odp_pool_t pool = ODP_POOL_INVALID; + + if (policy == SINGLE && pool != ODP_POOL_INVALID) + return pool; + + pool = odp_pool_create(name, params); + + return pool; +} + +static odp_bool_t setup_worker_config(prog_config_t *config) +{ + odp_pool_param_t param; + odp_pool_t pool; + worker_config_t *worker; + odp_shm_t shm; + void *data; + + odp_pool_param_init(¶m); + + if (config->type == BUFFER) { + param.type = ODP_POOL_BUFFER; + param.buf.num = config->num_evs; + param.buf.size = config->data_size; + param.buf.uarea_size = config->uarea_size; + param.buf.cache_size = config->cache_size; + config->alloc_fn = allocate_buffers; + config->free_fn = free_buffers; + } else if (config->type == PACKET) { + param.type = ODP_POOL_PACKET; + param.pkt.num = config->num_evs; + param.pkt.len = config->data_size; + param.pkt.seg_len = config->seg_len; + param.pkt.uarea_size = config->uarea_size; + param.pkt.cache_size = config->cache_size; + config->alloc_fn = allocate_packets; + config->free_fn = free_packets; + } else if (config->type == TMO) { + param.type = ODP_POOL_TIMEOUT; + param.tmo.num = config->num_evs; + param.tmo.uarea_size = config->uarea_size; + param.tmo.cache_size = config->cache_size; + config->alloc_fn = allocate_timeouts; + config->free_fn = free_timeouts; + } else { + param.type = ODP_POOL_VECTOR; + param.vector.num = config->num_evs; + param.vector.max_size = config->data_size; + param.vector.uarea_size = config->uarea_size; + param.vector.cache_size = config->cache_size; + config->alloc_fn = allocate_vectors; + config->free_fn = free_vectors; + } + + for (uint32_t i = 0U; i < config->num_workers; ++i) { + pool = create_pool(PROG_NAME "_pool", ¶m, config->policy); + + if (pool == ODP_POOL_INVALID) { + ODPH_ERR("Error creating worker pool\n"); + return false; + } + + shm = odp_shm_reserve(PROG_NAME "_shm", + config->handle_size * config->num_data_elems, + ODP_CACHE_LINE_SIZE, 0U); + + if (shm == ODP_SHM_INVALID) { + ODPH_ERR("Error creating worker SHM\n"); + return false; + } + + data = odp_shm_addr(shm); + + if (data == NULL) { + ODPH_ERR("Error resolving worker SHM\n"); + return false; + } + + worker = &config->worker_config[i]; + worker->pool = pool; + worker->data = data; + worker->prog_config = config; + worker->shm = shm; + worker->data_size = config->data_size; + worker->uarea_size = config->uarea_size; + } + + return true; +} + +static int run_test(void *args) +{ + worker_config_t *config = args; + odp_time_t t1, t2; + uint32_t head_idx, cur_idx, num_ignore = config->prog_config->num_ignore, val, num_alloc, + idx; + odp_bool_t is_saved; + const uint8_t num_elems = config->prog_config->num_elems; + const alloc_elem_t *elems = config->prog_config->alloc_elems, *elem; + uint8_t op; + void *data = config->data; + const alloc_fn_t alloc_fn = config->prog_config->alloc_fn; + stats_t *stats = &config->stats; + const free_fn_t free_fn = config->prog_config->free_fn; + + odp_barrier_wait(&config->prog_config->init_barrier); + t1 = odp_time_local_strict(); + + for (uint32_t i = 0U; i < config->prog_config->num_rounds; ++i) { + head_idx = 0U; + cur_idx = head_idx; + is_saved = (num_ignore > 0U ? num_ignore-- : num_ignore) == 0U; + + for (uint8_t j = 0U; j < num_elems; ++j) { + elem = &elems[j]; + val = elem->val; + op = elem->op; + + if (op == ALLOC) { + num_alloc = alloc_fn(config, data, cur_idx, val, i, j, is_saved); + + if (odp_unlikely(num_alloc < val)) + ++stats->pattern_errs; + + cur_idx += num_alloc; + } else if (op == FREE) { + /* Due to potential pattern errors, there might not be expected + * amount of freeable events. */ + val = ODPH_MIN(val, cur_idx - head_idx); + + if (elem->opt == TOP) { + idx = head_idx; + head_idx += val; + } else { + cur_idx -= val; + idx = cur_idx; + } + + free_fn(data, idx, val, stats, i, j, is_saved); + } else { + odp_time_wait_ns(val); + } + } + } + + t2 = odp_time_local_strict(); + stats->tot_tm = odp_time_diff_ns(t2, t1); + odp_barrier_wait(&config->prog_config->term_barrier); + + return 0; +} + +static odp_bool_t setup_workers(prog_config_t *config) +{ + odph_thread_common_param_t thr_common; + odph_thread_param_t thr_params[config->num_workers], *thr_param; + + odp_barrier_init(&config->init_barrier, config->num_workers + 1); + odp_barrier_init(&config->term_barrier, config->num_workers + 1); + odph_thread_common_param_init(&thr_common); + thr_common.instance = config->odp_instance; + thr_common.cpumask = &config->worker_mask; + + for (uint32_t i = 0; i < config->num_workers; ++i) { + thr_param = &thr_params[i]; + odph_thread_param_init(thr_param); + thr_param->start = run_test; + thr_param->thr_type = ODP_THREAD_WORKER; + thr_param->arg = &config->worker_config[i]; + } + + if ((uint32_t)odph_thread_create(config->thread_tbl, &thr_common, thr_params, + config->num_workers) != config->num_workers) { + ODPH_ERR("Error configuring worker threads\n"); + return false; + } + + odp_barrier_wait(&config->init_barrier); + + return true; +} + +static odp_bool_t setup_test(prog_config_t *config) +{ + return setup_worker_config(config) && setup_workers(config); +} + +static void stop_test(prog_config_t *config) +{ + odp_barrier_wait(&config->term_barrier); + (void)odph_thread_join(config->thread_tbl, config->num_workers); +} + +static void print_stats(const prog_config_t *config) +{ + const alloc_elem_t *elem; + const stats_t *stats; + uint64_t ev_rate, ave_b_alloc_tm, b_alloc_min, b_alloc_max, ave_b_free_tm, b_free_min, + b_free_max, ave_alloc_tm, ave_free_tm, ave_ua_b_tm, b_ua_min, b_ua_max, ave_ua_tm, + tot_b_alloc_tm = 0U, tot_b_free_tm = 0U, tot_alloc_tm = 0U, tot_free_tm = 0U, + tot_alloc_min = 0U, tot_alloc_max = 0U, tot_free_min = 0U, tot_free_max = 0U, + tot_b_ua_tm = 0U, tot_ua_tm = 0U, tot_ua_min = 0U, tot_ua_max = 0U; + + printf("\n==================\n\n" + "Pool latency test done\n\n" + " type: %s\n" + " event count: %u\n", config->type == BUFFER ? "buffer" : + config->type == PACKET ? "packet" : config->type == TMO ? "timeout" : "vector", + config->num_evs); + + if (config->type != TMO) + printf(" %s %u\n", config->type != VECTOR ? "data size: " : "vector size:", + config->data_size); + + printf(" pool policy: %s\n" + " round count: %u\n" + " ignore count: %u\n" + " cache size: %" PRIi64 "\n" + " user area: %u (B)\n" + " burst pattern:\n", config->policy == SINGLE ? "shared" : "per-worker", + config->num_rounds, config->num_ignore, config->cache_size, config->uarea_size); + + for (uint8_t i = 0U; i < config->num_elems; ++i) { + elem = &config->alloc_elems[i]; + printf(" %s %u%s\n", elem->op == ALLOC ? "allocate:" : + elem->op == FREE && elem->opt == TOP ? "free (t):" : + elem->op == FREE && elem->opt == BOTTOM ? "free (b):" : + "delay: ", elem->val, elem->op == DELAY ? " (ns)" : ""); + } + + printf("\n"); + + for (uint32_t i = 0U; i < config->num_workers; ++i) { + stats = &config->worker_config[i].stats; + ev_rate = stats->tot_tm > 0U ? + (double)stats->alloc_cnt / stats->tot_tm * ODP_TIME_SEC_IN_NS : 0U; + ave_b_alloc_tm = stats->alloc_b_cnt > 0U ? + stats->alloc_tm / stats->alloc_b_cnt : 0U; + b_alloc_min = ave_b_alloc_tm > 0U ? stats->min_alloc_tm : 0U; + b_alloc_max = ave_b_alloc_tm > 0U ? stats->max_alloc_tm : 0U; + ave_b_free_tm = stats->free_b_cnt > 0U ? + stats->free_tm / stats->free_b_cnt : 0U; + b_free_min = ave_b_free_tm > 0U ? stats->min_free_tm : 0U; + b_free_max = ave_b_free_tm > 0U ? stats->max_free_tm : 0U; + ave_alloc_tm = stats->alloc_cnt > 0U ? stats->alloc_tm / stats->alloc_cnt : 0U; + ave_free_tm = stats->alloc_cnt > 0U ? stats->free_tm / stats->alloc_cnt : 0U; + + printf(" worker %d:\n" + " significant events allocated/freed: %" PRIu64 "\n" + " allocation retries: %" PRIu64 "\n" + " allocation errors: %" PRIu64 "\n" + " pattern errors: %" PRIu64 "\n" + " run time: %" PRIu64 " (ns)\n" + " event rate %" PRIu64 " (evs/s)\n" + " average latency breakdown (ns):\n" + " per allocation burst: %" PRIu64 " (min: %" PRIu64 " (round: %" + PRIu64 ", pattern: %u), max: %" PRIu64 " (round: %" PRIu64 ", pattern: %u))" + "\n" + " per allocation: %" PRIu64 "\n" + " per free burst: %" PRIu64 " (min: %" PRIu64 " (round: %" + PRIu64 ", pattern: %u), max: %" PRIu64 " (round: %" PRIu64 ", pattern: %u))" + "\n" + " per free: %" PRIu64 "\n", i, stats->alloc_cnt, + stats->reallocs, stats->alloc_errs, stats->pattern_errs, stats->tot_tm, + ev_rate, ave_b_alloc_tm, b_alloc_min, stats->min_alloc_rnd, + stats->min_alloc_pt, b_alloc_max, stats->max_alloc_rnd, stats->max_alloc_pt, + ave_alloc_tm, ave_b_free_tm, b_free_min, stats->min_free_rnd, + stats->min_free_pt, b_free_max, stats->max_free_rnd, stats->max_free_pt, + ave_free_tm); + tot_b_alloc_tm += ave_b_alloc_tm; + tot_b_free_tm += ave_b_free_tm; + tot_alloc_tm += ave_alloc_tm; + tot_free_tm += ave_free_tm; + tot_alloc_min += b_alloc_min; + tot_alloc_max += b_alloc_max; + tot_free_min += b_free_min; + tot_free_max += b_free_max; + + if (config->uarea_size > 0U) { + ave_ua_b_tm = stats->alloc_b_cnt > 0U ? + stats->uarea_tm / stats->alloc_b_cnt : 0U; + ave_ua_tm = stats->alloc_cnt > 0U ? + stats->uarea_tm / stats->alloc_cnt : 0U; + b_ua_min = ave_ua_b_tm > 0U ? stats->min_uarea_tm : 0U; + b_ua_max = ave_ua_b_tm > 0U ? stats->max_uarea_tm : 0U; + printf(" per ua write burst: %" PRIu64 " (min: %" PRIu64 " (" + "round: %" PRIu64 ", pattern: %u), max: %" PRIu64 " (round: %" + PRIu64 ", pattern: %u))\n" + " per ua write: %" PRIu64 "\n", ave_ua_b_tm, + b_ua_min, stats->min_uarea_rnd, stats->min_uarea_pt, b_ua_max, + stats->max_uarea_rnd, stats->max_uarea_pt, ave_ua_tm); + tot_b_ua_tm += ave_ua_b_tm; + tot_ua_tm += ave_ua_tm; + tot_ua_min += b_ua_min; + tot_ua_max += b_ua_max; + } + + printf("\n"); + } + + printf(" total (ns):\n" + " per allocation burst: %" PRIu64 " (min: %" PRIu64 ", max: %" PRIu64 ")\n" + " per allocation: %" PRIu64 "\n" + " per free burst: %" PRIu64 " (min: %" PRIu64 ", max: %" PRIu64 ")\n" + " per free: %" PRIu64 "\n", + tot_b_alloc_tm / config->num_workers, tot_alloc_min / config->num_workers, + tot_alloc_max / config->num_workers, tot_alloc_tm / config->num_workers, + tot_b_free_tm / config->num_workers, tot_free_min / config->num_workers, + tot_free_max / config->num_workers, tot_free_tm / config->num_workers); + + if (config->uarea_size > 0U) { + printf(" per ua write burst: %" PRIu64 " (min: %" PRIu64 ", max: %" + PRIu64 ")\n" + " per ua write: %" PRIu64 "\n", + tot_b_ua_tm / config->num_workers, tot_ua_min / config->num_workers, + tot_ua_max / config->num_workers, tot_ua_tm / config->num_workers); + } + + printf("\n==================\n"); +} + +static void destroy_pool(odp_pool_t pool, uint8_t policy) +{ + static odp_bool_t is_destroyed; + + if (policy == SINGLE && is_destroyed) + return; + + (void)odp_pool_destroy(pool); + is_destroyed = true; +} + +static void teardown(const prog_config_t *config) +{ + const worker_config_t *worker; + + for (uint32_t i = 0U; i < config->num_workers; ++i) { + worker = &config->worker_config[i]; + + if (worker->pool != ODP_POOL_INVALID) + destroy_pool(worker->pool, config->policy); + + if (worker->shm != ODP_SHM_INVALID) + (void)odp_shm_free(worker->shm); + } +} + +int main(int argc, char **argv) +{ + odph_helper_options_t odph_opts; + odp_init_t init_param; + odp_instance_t odp_instance; + odp_shm_t shm_cfg = ODP_SHM_INVALID; + int ret = EXIT_SUCCESS; + parse_result_t parse_res; + + argc = odph_parse_options(argc, argv); + + if (odph_options(&odph_opts) == -1) { + ODPH_ERR("Error while reading ODP helper options, exiting\n"); + exit(EXIT_FAILURE); + } + + odp_init_param_init(&init_param); + init_param.mem_model = odph_opts.mem_model; + + if (odp_init_global(&odp_instance, &init_param, NULL)) { + ODPH_ERR("ODP global init failed, exiting\n"); + exit(EXIT_FAILURE); + } + + if (odp_init_local(odp_instance, ODP_THREAD_CONTROL)) { + ODPH_ERR("ODP local init failed, exiting\n"); + exit(EXIT_FAILURE); + } + + shm_cfg = odp_shm_reserve(PROG_NAME "_cfg", sizeof(prog_config_t), ODP_CACHE_LINE_SIZE, + 0U); + + if (shm_cfg == ODP_SHM_INVALID) { + ODPH_ERR("Error reserving shared memory\n"); + ret = EXIT_FAILURE; + goto out; + } + + prog_conf = odp_shm_addr(shm_cfg); + + if (prog_conf == NULL) { + ODPH_ERR("Error resolving shared memory address\n"); + ret = EXIT_FAILURE; + goto out; + } + + parse_res = parse_options(argc, argv, prog_conf); + + if (parse_res == PRS_NOK) { + ret = EXIT_FAILURE; + goto out; + } + + if (parse_res == PRS_TERM) { + ret = EXIT_SUCCESS; + goto out; + } + + prog_conf->odp_instance = odp_instance; + + if (!setup_test(prog_conf)) { + ret = EXIT_FAILURE; + goto out_test; + } + + stop_test(prog_conf); + print_stats(prog_conf); + +out_test: + teardown(prog_conf); + +out: + if (shm_cfg != ODP_SHM_INVALID) + (void)odp_shm_free(shm_cfg); + + if (odp_term_local()) { + ODPH_ERR("ODP local terminate failed, exiting\n"); + exit(EXIT_FAILURE); + } + + if (odp_term_global(odp_instance)) { + ODPH_ERR("ODP global terminate failed, exiting\n"); + exit(EXIT_FAILURE); + } + + return ret; +} diff --git a/test/performance/odp_pool_perf.c b/test/performance/odp_pool_perf.c index 4ae2cf7d3..43a39a21e 100644 --- a/test/performance/odp_pool_perf.c +++ b/test/performance/odp_pool_perf.c @@ -6,6 +6,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_pool_perf.c + * + * Performance test application for pool APIs + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include <stdio.h> #include <string.h> #include <stdint.h> diff --git a/test/performance/odp_queue_perf.c b/test/performance/odp_queue_perf.c index 4e4446610..7d4612cb8 100644 --- a/test/performance/odp_queue_perf.c +++ b/test/performance/odp_queue_perf.c @@ -5,6 +5,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_queue_perf.c + * + * Performance test application for queue APIs + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include <stdio.h> #include <string.h> #include <stdint.h> diff --git a/test/performance/odp_random.c b/test/performance/odp_random.c index 46134ac0c..99714d7b3 100644 --- a/test/performance/odp_random.c +++ b/test/performance/odp_random.c @@ -5,6 +5,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_random.c + * + * Performance test application for random data APIs + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include <stdio.h> #include <string.h> #include <stdint.h> diff --git a/test/performance/odp_sched_latency.c b/test/performance/odp_sched_latency.c index c8dc74656..0fec49fb9 100644 --- a/test/performance/odp_sched_latency.c +++ b/test/performance/odp_sched_latency.c @@ -6,9 +6,11 @@ */ /** - * @file + * @example odp_sched_latency.c * - * @example odp_sched_latency.c ODP scheduling latency benchmark application + * Scheduling latency benchmark application + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include <string.h> diff --git a/test/performance/odp_sched_latency_run.sh b/test/performance/odp_sched_latency_run.sh index 372fdb166..b051c1a4e 100755 --- a/test/performance/odp_sched_latency_run.sh +++ b/test/performance/odp_sched_latency_run.sh @@ -19,7 +19,7 @@ run() if [ $(nproc) -lt $1 ]; then echo "Not enough CPU cores. Skipping test." else - $TEST_DIR/odp_sched_latency${EXEEXT} -c $1 || exit $? + $TEST_DIR/odp_sched_latency${EXEEXT} -c $1 -d 1 || exit $? fi } diff --git a/test/performance/odp_sched_perf.c b/test/performance/odp_sched_perf.c index f89705576..47f703338 100644 --- a/test/performance/odp_sched_perf.c +++ b/test/performance/odp_sched_perf.c @@ -1,10 +1,23 @@ /* Copyright (c) 2018, Linaro Limited - * Copyright (c) 2020-2022, Nokia + * Copyright (c) 2020-2024, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_sched_perf.c + * + * Performance test application for scheduling + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE /* Needed for sigaction */ +#endif + +#include <signal.h> #include <stdio.h> #include <string.h> #include <stdint.h> @@ -52,6 +65,9 @@ typedef struct test_options_t { uint32_t ctx_size; uint32_t ctx_rd_words; uint32_t ctx_rw_words; + uint32_t uarea_rd; + uint32_t uarea_rw; + uint32_t uarea_size; uint64_t wait_ns; int verbose; @@ -88,6 +104,7 @@ typedef struct test_global_t { test_stat_t stat[ODP_THREAD_COUNT_MAX]; thread_arg_t thread_arg[ODP_THREAD_COUNT_MAX]; odp_atomic_u32_t num_worker; + odp_atomic_u32_t exit_threads; } test_global_t; @@ -96,6 +113,23 @@ typedef struct { odp_atomic_u64_t count; } queue_context_t; +static test_global_t *test_globals; + +static void sig_handler(int signum ODP_UNUSED) +{ + odp_atomic_store_u32(&test_globals->exit_threads, 1); +} + +static int setup_sig_handler(void) +{ + struct sigaction action = { .sa_handler = sig_handler }; + + if (sigemptyset(&action.sa_mask) || sigaction(SIGINT, &action, NULL)) + return -1; + + return 0; +} + static void print_usage(void) { printf("\n" @@ -111,7 +145,8 @@ static void print_usage(void) " the queues are default (or lowest) priority. Default: 0.\n" " -d, --num_dummy Number of empty queues. Default: 0.\n" " -e, --num_event Number of events per queue. Default: 100.\n" - " -s, --num_sched Number of events to schedule per thread. Default: 100 000.\n" + " -s, --num_sched Number of events to schedule per thread. If zero, the application runs\n" + " until SIGINT is received. Default: 100 000.\n" " -g, --num_group Number of schedule groups. Round robins threads and queues into groups.\n" " -1: SCHED_GROUP_WORKER\n" " 0: SCHED_GROUP_ALL (default)\n" @@ -127,6 +162,8 @@ static void print_usage(void) " -l, --ctx_rw_words Number of queue context words (uint64_t) to modify on every event. Default: 0.\n" " -n, --rd_words Number of event data words (uint64_t) to read before enqueueing it. Default: 0.\n" " -m, --rw_words Number of event data words (uint64_t) to modify before enqueueing it. Default: 0.\n" + " -u, --uarea_rd Number of user area words (uint64_t) to read on every event. Default: 0.\n" + " -U, --uarea_rw Number of user area words (uint64_t) to modify on every event. Default: 0.\n" " -p, --pool_type Pool type. 0: buffer, 1: packet. Default: 0.\n" " -v, --verbose Verbose output.\n" " -h, --help This help\n" @@ -159,13 +196,15 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) {"ctx_rw_words", required_argument, NULL, 'l'}, {"rd_words", required_argument, NULL, 'n'}, {"rw_words", required_argument, NULL, 'm'}, + {"uarea_rd", required_argument, NULL, 'u'}, + {"uarea_rw", required_argument, NULL, 'U'}, {"pool_type", required_argument, NULL, 'p'}, {"verbose", no_argument, NULL, 'v'}, {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} }; - static const char *shortopts = "+c:q:L:H:d:e:s:g:j:b:t:f:a:w:k:l:n:m:p:vh"; + static const char *shortopts = "+c:q:L:H:d:e:s:g:j:b:t:f:a:w:k:l:n:m:p:u:U:vh"; test_options->num_cpu = 1; test_options->num_queue = 1; @@ -184,6 +223,8 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) test_options->ctx_rw_words = 0; test_options->rd_words = 0; test_options->rw_words = 0; + test_options->uarea_rd = 0; + test_options->uarea_rw = 0; test_options->wait_ns = 0; test_options->verbose = 0; @@ -245,6 +286,12 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) case 'm': test_options->rw_words = atoi(optarg); break; + case 'u': + test_options->uarea_rd = atoi(optarg); + break; + case 'U': + test_options->uarea_rw = atoi(optarg); + break; case 'p': pool_type = atoi(optarg); break; @@ -337,6 +384,7 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) ctx_size = ROUNDUP(ctx_size, ODP_CACHE_LINE_SIZE); test_options->ctx_size = ctx_size; + test_options->uarea_size = 8 * (test_options->uarea_rd + test_options->uarea_rw); return ret; } @@ -377,7 +425,7 @@ static int create_pool(test_global_t *global) odp_pool_capability_t pool_capa; odp_pool_param_t pool_param; odp_pool_t pool; - uint32_t max_num, max_size; + uint32_t max_num, max_size, max_uarea; test_options_t *test_options = &global->test_options; uint32_t num_cpu = test_options->num_cpu; uint32_t num_queue = test_options->num_queue; @@ -395,6 +443,7 @@ static int create_pool(test_global_t *global) uint32_t event_size = 16; int touch_data = test_options->touch_data; uint32_t ctx_size = test_options->ctx_size; + uint32_t uarea_size = test_options->uarea_size; if (touch_data) { event_size = test_options->rd_words + test_options->rw_words; @@ -426,22 +475,22 @@ static int create_pool(test_global_t *global) printf(" max burst size %u\n", max_burst); printf(" total events %u\n", tot_event); printf(" event size %u bytes", event_size); - if (touch_data) { - printf(" (rd: %u, rw: %u)\n", - 8 * test_options->rd_words, - 8 * test_options->rw_words); - } else { - printf("\n"); - } + if (touch_data) + printf(" (rd: %u, rw: %u)", 8 * test_options->rd_words, 8 * test_options->rw_words); + printf("\n"); printf(" context size %u bytes", ctx_size); if (test_options->ctx_rd_words || test_options->ctx_rw_words) { - printf(" (rd: %u, rw: %u)\n", + printf(" (rd: %u, rw: %u)", 8 * test_options->ctx_rd_words, 8 * test_options->ctx_rw_words); - } else { - printf("\n"); } + printf("\n"); + + printf(" user area size %u bytes", uarea_size); + if (uarea_size) + printf(" (rd: %u, rw: %u)", 8 * test_options->uarea_rd, 8 * test_options->uarea_rw); + printf("\n"); if (odp_pool_capability(&pool_capa)) { ODPH_ERR("Error: pool capa failed\n"); @@ -452,11 +501,12 @@ static int create_pool(test_global_t *global) printf(" pool type buffer\n"); max_num = pool_capa.buf.max_num; max_size = pool_capa.buf.max_size; - + max_uarea = pool_capa.buf.max_uarea_size; } else { printf(" pool type packet\n"); max_num = pool_capa.pkt.max_num; max_size = pool_capa.pkt.max_seg_len; + max_uarea = pool_capa.pkt.max_uarea_size; } if (max_num && tot_event > max_num) { @@ -469,18 +519,25 @@ static int create_pool(test_global_t *global) return -1; } + if (uarea_size > max_uarea) { + ODPH_ERR("Error: max supported user area size %u\n", max_uarea); + return -1; + } + odp_pool_param_init(&pool_param); if (test_options->pool_type == ODP_POOL_BUFFER) { pool_param.type = ODP_POOL_BUFFER; pool_param.buf.num = tot_event; pool_param.buf.size = event_size; pool_param.buf.align = 8; + pool_param.buf.uarea_size = uarea_size; } else { pool_param.type = ODP_POOL_PACKET; pool_param.pkt.num = tot_event; pool_param.pkt.len = event_size; pool_param.pkt.seg_len = event_size; pool_param.pkt.align = 8; + pool_param.pkt.uarea_size = uarea_size; } pool = odp_pool_create("sched perf", &pool_param); @@ -853,6 +910,28 @@ static int destroy_groups(test_global_t *global) return 0; } +static uint64_t rw_uarea(odp_event_t ev[], int num, uint32_t rd_words, uint32_t rw_words) +{ + uint64_t *data; + int i; + uint32_t j; + uint64_t sum = 0; + + for (i = 0; i < num; i++) { + data = odp_event_user_area(ev[i]); + + for (j = 0; j < rd_words; j++) + sum += data[j]; + + for (; j < rd_words + rw_words; j++) { + sum += data[j]; + data[j] += 1; + } + } + + return sum; +} + static inline uint64_t rw_ctx_data(void *ctx, uint32_t offset, uint32_t rd_words, uint32_t rw_words) { @@ -921,12 +1000,17 @@ static int test_sched(void *arg) uint32_t ctx_size = test_options->ctx_size; uint32_t ctx_rd_words = test_options->ctx_rd_words; uint32_t ctx_rw_words = test_options->ctx_rw_words; + const uint32_t uarea_size = test_options->uarea_size; + const uint32_t uarea_rd = test_options->uarea_rd; + const uint32_t uarea_rw = test_options->uarea_rw; odp_pool_type_t pool_type = test_options->pool_type; int touch_ctx = ctx_rd_words || ctx_rw_words; + odp_atomic_u32_t *exit_threads = &global->exit_threads; uint32_t ctx_offset = 0; uint32_t sched_retries = 0; uint64_t data_sum = 0; uint64_t ctx_sum = 0; + uint64_t uarea_sum = 0; uint64_t wait_ns = test_options->wait_ns; odp_event_t ev[max_burst]; @@ -981,7 +1065,10 @@ static int test_sched(void *arg) c1 = odp_cpu_cycles(); last_retry_ts = t1; - for (rounds = 0; events < num_sched; rounds++) { + for (rounds = 0; odp_likely(!odp_atomic_load_u32(exit_threads)); rounds++) { + if (odp_unlikely(num_sched && events >= num_sched)) + break; + num = odp_schedule_multi(&queue, ODP_SCHED_NO_WAIT, ev, max_burst); @@ -990,6 +1077,9 @@ static int test_sched(void *arg) events += num; i = 0; + if (odp_unlikely(uarea_size)) + uarea_sum += rw_uarea(ev, num, uarea_rd, uarea_rw); + if (odp_unlikely(ctx_size)) { queue_context_t *ctx = odp_queue_context(queue); @@ -1077,7 +1167,7 @@ static int test_sched(void *arg) global->stat[thr].nsec = nsec; global->stat[thr].cycles = cycles; global->stat[thr].waits = waits; - global->stat[thr].dummy_sum = data_sum + ctx_sum; + global->stat[thr].dummy_sum = data_sum + ctx_sum + uarea_sum; global->stat[thr].failed = ret; if (odp_atomic_fetch_dec_u32(&global->num_worker) == 1) { @@ -1337,10 +1427,17 @@ int main(int argc, char **argv) ODPH_ERR("Error: SHM alloc failed\n"); exit(EXIT_FAILURE); } + test_globals = global; memset(global, 0, sizeof(test_global_t)); global->pool = ODP_POOL_INVALID; global->ctx_shm = ODP_SHM_INVALID; + odp_atomic_init_u32(&global->exit_threads, 0); + + if (setup_sig_handler()) { + ODPH_ERR("Error: signal handler setup failed\n"); + exit(EXIT_FAILURE); + } if (parse_options(argc, argv, &global->test_options)) return -1; diff --git a/test/performance/odp_sched_pktio.c b/test/performance/odp_sched_pktio.c index 3a85a91a5..d8ab1b279 100644 --- a/test/performance/odp_sched_pktio.c +++ b/test/performance/odp_sched_pktio.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_sched_pktio.c + * + * Test application for scheduled packet IO + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include <stdio.h> #include <string.h> #include <signal.h> diff --git a/test/performance/odp_scheduling.c b/test/performance/odp_scheduling.c index f9c083c92..c9f3eb89f 100644 --- a/test/performance/odp_scheduling.c +++ b/test/performance/odp_scheduling.c @@ -1,13 +1,14 @@ -/* Copyright (c) 2013-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2013-2018 Linaro Limited + * Copyright (c) 2019-2023 Nokia */ /** - * @file + * @example odp_scheduling.c + * + * Performance test application for miscellaneous scheduling operations * - * @example odp_example.c ODP example application + * @cond _ODP_HIDE_FROM_DOXYGEN_ */ #include <string.h> @@ -47,6 +48,7 @@ typedef struct { /** Test arguments */ typedef struct { + double test_sec; /**< CPU frequency test duration in seconds */ unsigned int cpu_count; /**< CPU count */ int fairness; /**< Check fairness */ } test_args_t; @@ -685,17 +687,17 @@ static int run_thread(void *arg ODP_UNUSED) /** * @internal Test cycle counter frequency */ -static void test_cpu_freq(void) +static void test_cpu_freq(double test_sec) { odp_time_t cur_time, test_time, start_time, end_time; uint64_t c1, c2, cycles; uint64_t nsec; double diff_max_hz, max_cycles; - printf("\nCPU cycle count frequency test (runs about %i sec)\n", - TEST_SEC); + printf("\nCPU cycle count frequency test (runs about %f sec)\n", + test_sec); - test_time = odp_time_local_from_ns(TEST_SEC * ODP_TIME_SEC_IN_NS); + test_time = odp_time_local_from_ns(test_sec * ODP_TIME_SEC_IN_NS); start_time = odp_time_local(); end_time = odp_time_sum(start_time, test_time); @@ -732,6 +734,7 @@ static void print_usage(void) { printf("\n\nUsage: ./odp_example [options]\n"); printf("Options:\n"); + printf(" -t, --time <number> test duration, default=%.1f\n", (double)TEST_SEC); printf(" -c, --count <number> CPU count, 0=all available, default=1\n"); printf(" -h, --help this help\n"); printf(" -f, --fair collect fairness statistics\n"); @@ -751,15 +754,17 @@ static void parse_args(int argc, char *argv[], test_args_t *args) int long_index; static const struct option longopts[] = { + {"time", required_argument, NULL, 't'}, {"count", required_argument, NULL, 'c'}, {"fair", no_argument, NULL, 'f'}, {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0} }; - static const char *shortopts = "+c:fh"; + static const char *shortopts = "+t:c:fh"; args->cpu_count = 1; /* use one worker by default */ + args->test_sec = TEST_SEC; while (1) { opt = getopt_long(argc, argv, shortopts, longopts, &long_index); @@ -772,6 +777,10 @@ static void parse_args(int argc, char *argv[], test_args_t *args) args->fairness = 1; break; + case 't': + args->test_sec = atof(optarg); + break; + case 'c': args->cpu_count = atoi(optarg); break; @@ -855,14 +864,14 @@ int main(int argc, char *argv[]) printf("first CPU: %i\n", odp_cpumask_first(&cpumask)); printf("cpu mask: %s\n", cpumaskstr); - thread_tbl = calloc(sizeof(odph_thread_t), num_workers); + thread_tbl = calloc(num_workers, sizeof(odph_thread_t)); if (!thread_tbl) { ODPH_ERR("no memory for thread_tbl\n"); return -1; } /* Test cycle count frequency */ - test_cpu_freq(); + test_cpu_freq(args.test_sec); shm = odp_shm_reserve("test_globals", sizeof(test_globals_t), ODP_CACHE_LINE_SIZE, 0); diff --git a/test/performance/odp_scheduling_run.sh b/test/performance/odp_scheduling_run.sh index 2b4281ee9..4e004264e 100755 --- a/test/performance/odp_scheduling_run.sh +++ b/test/performance/odp_scheduling_run.sh @@ -19,7 +19,7 @@ run() if [ $(nproc) -lt $1 ]; then echo "Not enough CPU cores. Skipping test." else - $TEST_DIR/odp_scheduling${EXEEXT} -c $1 + $TEST_DIR/odp_scheduling${EXEEXT} -c $1 -t 0.1 RET_VAL=$? if [ $RET_VAL -ne 0 ]; then echo odp_scheduling FAILED diff --git a/test/performance/odp_stash_perf.c b/test/performance/odp_stash_perf.c index ffbc92b4b..cb223999e 100644 --- a/test/performance/odp_stash_perf.c +++ b/test/performance/odp_stash_perf.c @@ -4,6 +4,14 @@ * Copyright (c) 2023 Arm */ +/** + * @example odp_stash_perf.c + * + * Performance test application for stash APIs + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include <stdio.h> #include <string.h> #include <stdint.h> diff --git a/test/performance/odp_stress.c b/test/performance/odp_stress.c index 84bc4fe6c..3ec01df33 100644 --- a/test/performance/odp_stress.c +++ b/test/performance/odp_stress.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_stress.c + * + * Test application that can be used to stress CPU, memory, and HW accelerators. + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include <stdio.h> #include <string.h> #include <stdint.h> @@ -73,8 +81,8 @@ static void print_usage(void) "Stress test options:\n" "\n" " -c, --num_cpu Number of CPUs (worker threads). 0: all available CPUs. Default: 1\n" - " -p, --period_ns Timeout period in nsec. Default: 1 sec\n" - " -r, --rounds Number of timeout rounds. Default: 10\n" + " -p, --period_ns Timeout period in nsec. Default: 100 ms\n" + " -r, --rounds Number of timeout rounds. Default: 2\n" " -m, --mode Select test mode. Default: 1\n" " 0: No stress, just wait for timeouts\n" " 1: Memcpy\n" @@ -106,8 +114,8 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options) static const char *shortopts = "+c:p:r:m:s:g:h"; test_options->num_cpu = 1; - test_options->period_ns = 1000 * ODP_TIME_MSEC_IN_NS; - test_options->rounds = 10; + test_options->period_ns = 100 * ODP_TIME_MSEC_IN_NS; + test_options->rounds = 2; test_options->mode = 1; test_options->mem_size = 2048; test_options->group_mode = 1; diff --git a/test/performance/odp_timer_perf.c b/test/performance/odp_timer_perf.c index 8632fcb73..918267a1b 100644 --- a/test/performance/odp_timer_perf.c +++ b/test/performance/odp_timer_perf.c @@ -4,6 +4,14 @@ * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @example odp_timer_perf.c + * + * Performance test application for timer APIs + * + * @cond _ODP_HIDE_FROM_DOXYGEN_ + */ + #include <stdio.h> #include <string.h> #include <stdint.h> @@ -659,10 +667,7 @@ static int cancel_timers(test_global_t *global, uint32_t worker_idx) int ret = 0; for (i = 0; i < num_tp; i++) { - for (j = 0; j < num_timer; j++) { - if ((j % num_worker) != worker_idx) - continue; - + for (j = worker_idx; j < num_timer; j += num_worker) { timer = global->timer[i][j]; if (timer == ODP_TIMER_INVALID) continue; @@ -686,7 +691,7 @@ static int cancel_timers(test_global_t *global, uint32_t worker_idx) static int set_cancel_mode_worker(void *arg) { uint64_t tick, start_tick, period_tick, nsec; - uint64_t c1, c2, diff; + uint64_t c1, c2; int thr, status; uint32_t i, j, worker_idx; odp_event_t ev; @@ -707,11 +712,12 @@ static int set_cancel_mode_worker(void *arg) uint64_t num_tmo = 0; uint64_t num_cancel = 0; uint64_t num_set = 0; + uint64_t cancel_cycles = 0, start_cycles = 0; + odp_event_t ev_tbl[MAX_TIMERS]; thr = odp_thread_id(); worker_idx = thread_arg->worker_idx; t1 = ODP_TIME_NULL; - c1 = 0; /* Start all workers at the same time */ odp_barrier_wait(&global->barrier); @@ -761,7 +767,6 @@ static int set_cancel_mode_worker(void *arg) /* Start measurements */ started = 1; t1 = odp_time_local(); - c1 = odp_cpu_cycles(); } /* Cancel and set timers again */ @@ -774,16 +779,16 @@ static int set_cancel_mode_worker(void *arg) period_tick = global->timer_pool[i].period_tick; tick = odp_timer_current_tick(tp) + start_tick; + c1 = odp_cpu_cycles(); - for (j = 0; j < num_timer; j++) { - if ((j % num_worker) != worker_idx) - continue; + for (j = worker_idx; j < num_timer; j += num_worker) { + ev_tbl[j] = ODP_EVENT_INVALID; timer = global->timer[i][j]; if (timer == ODP_TIMER_INVALID) continue; - status = odp_timer_cancel(timer, &ev); + status = odp_timer_cancel(timer, &ev_tbl[j]); num_cancel++; if (odp_unlikely(status == ODP_TIMER_TOO_NEAR)) { @@ -794,10 +799,23 @@ static int set_cancel_mode_worker(void *arg) ret = -1; break; } + } + + c2 = odp_cpu_cycles(); + cancel_cycles += odp_cpu_cycles_diff(c2, c1); + c1 = c2; + + for (j = worker_idx; j < num_timer; j += num_worker) { + if (ev_tbl[j] == ODP_EVENT_INVALID) + continue; + + timer = global->timer[i][j]; + if (timer == ODP_TIMER_INVALID) + continue; start_param.tick_type = ODP_TIMER_TICK_ABS; start_param.tick = tick + j * period_tick; - start_param.tmo_ev = ev; + start_param.tmo_ev = ev_tbl[j]; status = odp_timer_start(timer, &start_param); num_set++; @@ -809,6 +827,9 @@ static int set_cancel_mode_worker(void *arg) break; } } + + c2 = odp_cpu_cycles(); + start_cycles += odp_cpu_cycles_diff(c2, c1); } if (test_rounds) { @@ -819,9 +840,7 @@ static int set_cancel_mode_worker(void *arg) } t2 = odp_time_local(); - c2 = odp_cpu_cycles(); nsec = odp_time_diff_ns(t2, t1); - diff = odp_cpu_cycles_diff(c2, c1); /* Cancel all timers that belong to this thread */ if (cancel_timers(global, worker_idx)) @@ -831,7 +850,8 @@ static int set_cancel_mode_worker(void *arg) global->stat[thr].events = num_tmo; global->stat[thr].rounds = test_options->test_rounds - test_rounds; global->stat[thr].nsec = nsec; - global->stat[thr].cycles_0 = diff; + global->stat[thr].cycles_0 = cancel_cycles; + global->stat[thr].cycles_1 = start_cycles; global->stat[thr].cancels = num_cancel; global->stat[thr].sets = num_set; @@ -1104,16 +1124,38 @@ static void print_stat_set_cancel_mode(test_global_t *global) int num = 0; printf("\n"); - printf("RESULTS - timer cancel + set cycles per thread:\n"); - printf("-----------------------------------------------\n"); + printf("RESULTS\n"); + printf("odp_timer_cancel() cycles per thread:\n"); + printf("-------------------------------------------------\n"); printf(" 1 2 3 4 5 6 7 8 9 10"); for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) { - if (global->stat[i].sets) { + const test_stat_t *si = &global->stat[i]; + + if (si->cancels) { + if ((num % 10) == 0) + printf("\n "); + + printf("%6.1f ", (double)si->cycles_0 / si->cancels); + num++; + } + } + + printf("\n\n"); + + num = 0; + printf("odp_timer_start() cycles per thread:\n"); + printf("-------------------------------------------------\n"); + printf(" 1 2 3 4 5 6 7 8 9 10"); + + for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) { + const test_stat_t *si = &global->stat[i]; + + if (si->sets) { if ((num % 10) == 0) printf("\n "); - printf("%6.1f ", (double)global->stat[i].cycles_0 / global->stat[i].sets); + printf("%6.1f ", (double)si->cycles_1 / si->sets); num++; } } diff --git a/test/validation/api/Makefile.am b/test/validation/api/Makefile.am index ade387152..5a3c0216b 100644 --- a/test/validation/api/Makefile.am +++ b/test/validation/api/Makefile.am @@ -1,6 +1,8 @@ -ODP_MODULES = atomic \ +ODP_MODULES = align \ + atomic \ barrier \ buffer \ + byteorder \ chksum \ classification \ comp \ @@ -10,9 +12,11 @@ ODP_MODULES = atomic \ errno \ event \ hash \ + hints \ init \ ipsec \ lock \ + ml \ queue \ packet \ pktio \ @@ -31,12 +35,14 @@ ODP_MODULES = atomic \ SUBDIRS = $(ODP_MODULES) include $(top_srcdir)/test/Makefile.inc -TESTS_ENVIRONMENT += TEST_DIR=${top_builddir}/test/validation +TESTS_ENVIRONMENT += TEST_DIR=${top_builddir}/test/validation/api TESTS = \ + align/align_main$(EXEEXT) \ atomic/atomic_main$(EXEEXT) \ barrier/barrier_main$(EXEEXT) \ buffer/buffer_main$(EXEEXT) \ + byteorder/byteorder_main$(EXEEXT) \ chksum/chksum_main$(EXEEXT) \ classification/classification_main$(EXEEXT) \ comp/comp_main$(EXEEXT) \ @@ -46,20 +52,22 @@ TESTS = \ errno/errno_main$(EXEEXT) \ event/event_main$(EXEEXT) \ hash/hash_main$(EXEEXT) \ - init/init_defaults$(EXEEXT) \ - init/init_abort$(EXEEXT) \ - init/init_log$(EXEEXT) \ - init/init_log_thread$(EXEEXT) \ - init/init_num_thr$(EXEEXT) \ - init/init_feature_enabled$(EXEEXT) \ - init/init_feature_disabled$(EXEEXT) \ - init/init_test_param_init$(EXEEXT) \ - init/init_test_term_abnormal$(EXEEXT) \ - ipsec/ipsec_sync$(EXEEXT) \ - ipsec/ipsec_async$(EXEEXT) \ - ipsec/ipsec_inline_in$(EXEEXT) \ - ipsec/ipsec_inline_out$(EXEEXT) \ + hints/hints_main$(EXEEXT) \ + init/init_defaults.sh \ + init/init_abort.sh \ + init/init_log.sh \ + init/init_log_thread.sh \ + init/init_num_thr.sh \ + init/init_feature_enabled.sh \ + init/init_feature_disabled.sh \ + init/init_test_param_init.sh \ + init/init_test_term_abnormal.sh \ + ipsec/ipsec_sync.sh \ + ipsec/ipsec_async.sh \ + ipsec/ipsec_inline_in.sh \ + ipsec/ipsec_inline_out.sh \ lock/lock_main$(EXEEXT) \ + ml/ml_main$(EXEEXT) \ packet/packet_main$(EXEEXT) \ pktio/pktio_main$(EXEEXT) \ pool/pool_main$(EXEEXT) \ diff --git a/test/validation/api/align/.gitignore b/test/validation/api/align/.gitignore new file mode 100644 index 000000000..3031151a1 --- /dev/null +++ b/test/validation/api/align/.gitignore @@ -0,0 +1 @@ +align_main diff --git a/test/validation/api/align/Makefile.am b/test/validation/api/align/Makefile.am new file mode 100644 index 000000000..dd6e36bcd --- /dev/null +++ b/test/validation/api/align/Makefile.am @@ -0,0 +1,4 @@ +include ../Makefile.inc + +test_PROGRAMS = align_main +align_main_SOURCES = align.c diff --git a/test/validation/api/align/align.c b/test/validation/api/align/align.c new file mode 100644 index 000000000..ddbd1a2fe --- /dev/null +++ b/test/validation/api/align/align.c @@ -0,0 +1,156 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2024 Nokia + */ + +#include <odp_api.h> +#include <odp_cunit_common.h> + +#include <stddef.h> +#include <stdint.h> + +/* Test struct without gaps */ +typedef struct ODP_PACKED { + uint8_t a; + uint8_t b; + uint16_t c; + uint32_t d; +} test_type_t; + +/* Test struct with gaps */ +typedef struct ODP_PACKED { + uint8_t a; + uint16_t b; + uint8_t c; + uint32_t d; +} test_type_2_t; + +static void test_aligned(void) +{ + uint8_t align_2 ODP_ALIGNED(2); + uint16_t align_4 ODP_ALIGNED(4); + uint32_t align_8 ODP_ALIGNED(8); + uint64_t align_16 ODP_ALIGNED(16); + + CU_ASSERT((uintptr_t)&align_2 % 2 == 0); + CU_ASSERT((uintptr_t)&align_4 % 4 == 0); + CU_ASSERT((uintptr_t)&align_8 % 8 == 0); + CU_ASSERT((uintptr_t)&align_16 % 16 == 0); +} + +static void test_packed(void) +{ + uint32_t offset; + + offset = 0; + CU_ASSERT(offsetof(test_type_t, a) == offset); + + offset += sizeof(uint8_t); + CU_ASSERT(offsetof(test_type_t, b) == offset); + + offset += sizeof(uint8_t); + CU_ASSERT(offsetof(test_type_t, c) == offset); + + offset += sizeof(uint16_t); + CU_ASSERT(offsetof(test_type_t, d) == offset); + + offset = 0; + CU_ASSERT(offsetof(test_type_2_t, a) == offset); + + offset += sizeof(uint8_t); + CU_ASSERT(offsetof(test_type_2_t, b) == offset); + + offset += sizeof(uint16_t); + CU_ASSERT(offsetof(test_type_2_t, c) == offset); + + offset += sizeof(uint8_t); + CU_ASSERT(offsetof(test_type_2_t, d) == offset); +} + +static void test_offsetof(void) +{ + CU_ASSERT(ODP_OFFSETOF(test_type_t, a) == offsetof(test_type_t, a)); + CU_ASSERT(ODP_OFFSETOF(test_type_t, b) == offsetof(test_type_t, b)); + CU_ASSERT(ODP_OFFSETOF(test_type_t, c) == offsetof(test_type_t, c)); + CU_ASSERT(ODP_OFFSETOF(test_type_t, d) == offsetof(test_type_t, d)); + CU_ASSERT(ODP_OFFSETOF(test_type_2_t, a) == offsetof(test_type_2_t, a)); + CU_ASSERT(ODP_OFFSETOF(test_type_2_t, b) == offsetof(test_type_2_t, b)); + CU_ASSERT(ODP_OFFSETOF(test_type_2_t, c) == offsetof(test_type_2_t, c)); + CU_ASSERT(ODP_OFFSETOF(test_type_2_t, d) == offsetof(test_type_2_t, d)); +} + +static void test_field_sizeof(void) +{ + test_type_t tt; + + CU_ASSERT(ODP_FIELD_SIZEOF(test_type_t, a) == sizeof(tt.a)); + CU_ASSERT(ODP_FIELD_SIZEOF(test_type_t, b) == sizeof(tt.b)); + CU_ASSERT(ODP_FIELD_SIZEOF(test_type_t, c) == sizeof(tt.c)); + CU_ASSERT(ODP_FIELD_SIZEOF(test_type_t, d) == sizeof(tt.d)); +} + +static void test_cache_line_size(void) +{ + CU_ASSERT(ODP_CACHE_LINE_SIZE > 0); + CU_ASSERT(ODP_CACHE_LINE_SIZE % 2 == 0); +} + +static void test_page_size(void) +{ + CU_ASSERT(ODP_PAGE_SIZE > 0); + CU_ASSERT(ODP_PAGE_SIZE % 2 == 0); +} + +static void test_aligned_cache(void) +{ + uint8_t arr[123] ODP_ALIGNED_CACHE; + + CU_ASSERT((uintptr_t)arr % ODP_CACHE_LINE_SIZE == 0); +} + +static void test_aligned_page(void) +{ + uint8_t arr[123] ODP_ALIGNED_PAGE; + + CU_ASSERT((uintptr_t)arr % ODP_PAGE_SIZE == 0); +} + +static void test_cache_line_roundup(void) +{ + CU_ASSERT(ODP_CACHE_LINE_ROUNDUP(123) % ODP_CACHE_LINE_SIZE == 0); + CU_ASSERT(ODP_CACHE_LINE_ROUNDUP(ODP_CACHE_LINE_SIZE) == ODP_CACHE_LINE_SIZE); + CU_ASSERT(ODP_CACHE_LINE_ROUNDUP(0) == 0); +} + +odp_testinfo_t align_suite[] = { + ODP_TEST_INFO(test_aligned), + ODP_TEST_INFO(test_packed), + ODP_TEST_INFO(test_offsetof), + ODP_TEST_INFO(test_field_sizeof), + ODP_TEST_INFO(test_cache_line_size), + ODP_TEST_INFO(test_page_size), + ODP_TEST_INFO(test_aligned_cache), + ODP_TEST_INFO(test_aligned_page), + ODP_TEST_INFO(test_cache_line_roundup), + ODP_TEST_INFO_NULL, +}; + +odp_suiteinfo_t align_suites[] = { + {"align", NULL, NULL, align_suite}, + ODP_SUITE_INFO_NULL +}; + +int main(int argc, char *argv[]) +{ + int ret; + + /* Parse common options */ + if (odp_cunit_parse_options(&argc, argv)) + return -1; + + ret = odp_cunit_register(align_suites); + + if (ret == 0) + ret = odp_cunit_run(); + + return ret; +} diff --git a/test/validation/api/atomic/atomic.c b/test/validation/api/atomic/atomic.c index 76e3c0d49..fab982462 100644 --- a/test/validation/api/atomic/atomic.c +++ b/test/validation/api/atomic/atomic.c @@ -1702,7 +1702,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(atomic_init); diff --git a/test/validation/api/barrier/barrier.c b/test/validation/api/barrier/barrier.c index e4fba770f..7dc9a44c6 100644 --- a/test/validation/api/barrier/barrier.c +++ b/test/validation/api/barrier/barrier.c @@ -444,7 +444,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(barrier_init); diff --git a/test/validation/api/buffer/buffer.c b/test/validation/api/buffer/buffer.c index 91cfbfb5f..2a79ed27e 100644 --- a/test/validation/api/buffer/buffer.c +++ b/test/validation/api/buffer/buffer.c @@ -598,7 +598,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(buffer_suites); diff --git a/test/validation/api/byteorder/.gitignore b/test/validation/api/byteorder/.gitignore new file mode 100644 index 000000000..0b0d91053 --- /dev/null +++ b/test/validation/api/byteorder/.gitignore @@ -0,0 +1 @@ +byteorder_main diff --git a/test/validation/api/byteorder/Makefile.am b/test/validation/api/byteorder/Makefile.am new file mode 100644 index 000000000..186f4ecbc --- /dev/null +++ b/test/validation/api/byteorder/Makefile.am @@ -0,0 +1,4 @@ +include ../Makefile.inc + +test_PROGRAMS = byteorder_main +byteorder_main_SOURCES = byteorder.c diff --git a/test/validation/api/byteorder/byteorder.c b/test/validation/api/byteorder/byteorder.c new file mode 100644 index 000000000..087dfce95 --- /dev/null +++ b/test/validation/api/byteorder/byteorder.c @@ -0,0 +1,107 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2024 Nokia + */ + +#include <odp_api.h> +#include <odp_cunit_common.h> + +#include <stdint.h> + +static void test_defines(void) +{ + /* Endianness */ + CU_ASSERT(ODP_BIG_ENDIAN || ODP_LITTLE_ENDIAN); + + if (ODP_BIG_ENDIAN) { + CU_ASSERT(ODP_BYTE_ORDER == ODP_BIG_ENDIAN); + CU_ASSERT(!ODP_LITTLE_ENDIAN); + } + + if (ODP_LITTLE_ENDIAN) { + CU_ASSERT(ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN); + CU_ASSERT(!ODP_BIG_ENDIAN); + } + + /* Bitfield endianness */ + CU_ASSERT(ODP_BIG_ENDIAN_BITFIELD || ODP_LITTLE_ENDIAN_BITFIELD); + + if (ODP_BIG_ENDIAN_BITFIELD) { + CU_ASSERT(ODP_BITFIELD_ORDER == ODP_BIG_ENDIAN_BITFIELD); + CU_ASSERT(!ODP_LITTLE_ENDIAN_BITFIELD); + } + + if (ODP_LITTLE_ENDIAN_BITFIELD) { + CU_ASSERT(ODP_BITFIELD_ORDER == ODP_LITTLE_ENDIAN_BITFIELD); + CU_ASSERT(!ODP_BIG_ENDIAN_BITFIELD); + } +} + +static void test_types(void) +{ + const uint16_t u16_val = 0x1234; + const uint32_t u32_val = 0x12345678; + const uint64_t u64_val = 0x1234567890123456; + const uint16_t u16_val_conv = 0x3412; + const uint32_t u32_val_conv = 0x78563412; + const uint64_t u64_val_conv = 0x5634129078563412; + odp_u16be_t be16 = odp_cpu_to_be_16(u16_val); + odp_u32be_t be32 = odp_cpu_to_be_32(u32_val); + odp_u64be_t be64 = odp_cpu_to_be_64(u64_val); + odp_u16le_t le16 = odp_cpu_to_le_16(u16_val); + odp_u32le_t le32 = odp_cpu_to_le_32(u32_val); + odp_u64le_t le64 = odp_cpu_to_le_64(u64_val); + odp_u16sum_t sum16 = u16_val; + odp_u32sum_t sum32 = u16_val; + + CU_ASSERT(sum16 == sum32); + + if (ODP_BIG_ENDIAN) { + CU_ASSERT(be16 == u16_val); + CU_ASSERT(be32 == u32_val); + CU_ASSERT(be64 == u64_val); + CU_ASSERT(le16 == u16_val_conv); + CU_ASSERT(le32 == u32_val_conv); + CU_ASSERT(le64 == u64_val_conv); + } else { + CU_ASSERT(le16 == u16_val); + CU_ASSERT(le32 == u32_val); + CU_ASSERT(le64 == u64_val); + CU_ASSERT(be16 == u16_val_conv); + CU_ASSERT(be32 == u32_val_conv); + CU_ASSERT(be64 == u64_val_conv); + } + + CU_ASSERT(odp_be_to_cpu_16(be16) == u16_val); + CU_ASSERT(odp_be_to_cpu_32(be32) == u32_val); + CU_ASSERT(odp_be_to_cpu_64(be64) == u64_val); + CU_ASSERT(odp_le_to_cpu_16(le16) == u16_val); + CU_ASSERT(odp_le_to_cpu_32(le32) == u32_val); + CU_ASSERT(odp_le_to_cpu_64(le64) == u64_val); +} + +odp_testinfo_t byteorder_suite[] = { + ODP_TEST_INFO(test_defines), + ODP_TEST_INFO(test_types), + ODP_TEST_INFO_NULL, +}; + +odp_suiteinfo_t byteorder_suites[] = { + {"byteorder", NULL, NULL, byteorder_suite}, + ODP_SUITE_INFO_NULL +}; + +int main(int argc, char *argv[]) +{ + int ret; + + /* Parse common options */ + if (odp_cunit_parse_options(&argc, argv)) + return -1; + + ret = odp_cunit_register(byteorder_suites); + + if (ret == 0) + ret = odp_cunit_run(); + + return ret; +} diff --git a/test/validation/api/chksum/chksum.c b/test/validation/api/chksum/chksum.c index 86306ab0b..0be418f3a 100644 --- a/test/validation/api/chksum/chksum.c +++ b/test/validation/api/chksum/chksum.c @@ -442,7 +442,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(chksum_suites); diff --git a/test/validation/api/classification/classification.c b/test/validation/api/classification/classification.c index ef35377dc..ef9a647cb 100644 --- a/test/validation/api/classification/classification.c +++ b/test/validation/api/classification/classification.c @@ -36,7 +36,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(classification_suites); diff --git a/test/validation/api/comp/comp.c b/test/validation/api/comp/comp.c index cb771c989..b7dfcd359 100644 --- a/test/validation/api/comp/comp.c +++ b/test/validation/api/comp/comp.c @@ -558,7 +558,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(comp_init); diff --git a/test/validation/api/cpumask/cpumask.c b/test/validation/api/cpumask/cpumask.c index b358a5be6..db500df3a 100644 --- a/test/validation/api/cpumask/cpumask.c +++ b/test/validation/api/cpumask/cpumask.c @@ -188,7 +188,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(cpumask_suites); diff --git a/test/validation/api/crypto/crypto_op_test.c b/test/validation/api/crypto/crypto_op_test.c index 1c6944fdf..ae1465581 100644 --- a/test/validation/api/crypto/crypto_op_test.c +++ b/test/validation/api/crypto/crypto_op_test.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2021-2023, Nokia + * Copyright (c) 2021-2024, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -50,7 +50,7 @@ int crypto_op(odp_packet_t pkt_in, odp_crypto_op_type_t op_type) { int rc; - odp_event_t event; + odp_event_t event, event2; odp_crypto_packet_result_t result; odp_event_subtype_t subtype; odp_packet_t orig_pkt_out; @@ -95,15 +95,19 @@ int crypto_op(odp_packet_t pkt_in, if (op_type != ODP_CRYPTO_OP_TYPE_BASIC) CU_ASSERT(*pkt_out == orig_pkt_out); - CU_ASSERT(ODP_EVENT_PACKET == - odp_event_type(odp_packet_to_event(*pkt_out))); - CU_ASSERT(ODP_EVENT_PACKET_CRYPTO == - odp_event_subtype(odp_packet_to_event(*pkt_out))); - CU_ASSERT(ODP_EVENT_PACKET == - odp_event_types(odp_packet_to_event(*pkt_out), &subtype)); + + event = odp_packet_to_event(*pkt_out); + CU_ASSERT(ODP_EVENT_PACKET == odp_event_type(event)); + CU_ASSERT(ODP_EVENT_PACKET_CRYPTO == odp_event_subtype(event)); + CU_ASSERT(ODP_EVENT_PACKET == odp_event_types(event, &subtype)); CU_ASSERT(ODP_EVENT_PACKET_CRYPTO == subtype); CU_ASSERT(odp_packet_subtype(*pkt_out) == ODP_EVENT_PACKET_CRYPTO); + event2 = odp_crypto_packet_to_event(*pkt_out); + CU_ASSERT(ODP_EVENT_PACKET == odp_event_type(event2)); + CU_ASSERT(ODP_EVENT_PACKET_CRYPTO == odp_event_subtype(event2)); + CU_ASSERT(odp_event_to_u64(event) == odp_event_to_u64(event2)); + rc = odp_crypto_result(&result, *pkt_out); if (rc < -1) CU_FAIL("Failed odp_crypto_result()"); diff --git a/test/validation/api/crypto/odp_crypto_test_inp.c b/test/validation/api/crypto/odp_crypto_test_inp.c index d5f2e3e7a..532aaf525 100644 --- a/test/validation/api/crypto/odp_crypto_test_inp.c +++ b/test/validation/api/crypto/odp_crypto_test_inp.c @@ -2399,7 +2399,7 @@ int main(int argc, char *argv[]) printf("Test mode: %s\n", full_test ? "full" : "partial"); /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(crypto_init); diff --git a/test/validation/api/dma/dma.c b/test/validation/api/dma/dma.c index 4f454168d..efc7fa039 100644 --- a/test/validation/api/dma/dma.c +++ b/test/validation/api/dma/dma.c @@ -1693,7 +1693,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(dma_suites); diff --git a/test/validation/api/errno/errno.c b/test/validation/api/errno/errno.c index ed2ab391d..70708ce01 100644 --- a/test/validation/api/errno/errno.c +++ b/test/validation/api/errno/errno.c @@ -33,7 +33,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(errno_suites); diff --git a/test/validation/api/event/event.c b/test/validation/api/event/event.c index f6ad86365..fbcc08d6f 100644 --- a/test/validation/api/event/event.c +++ b/test/validation/api/event/event.c @@ -461,7 +461,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(event_suites); diff --git a/test/validation/api/hash/hash.c b/test/validation/api/hash/hash.c index 5e4209800..a935ef7ac 100644 --- a/test/validation/api/hash/hash.c +++ b/test/validation/api/hash/hash.c @@ -753,7 +753,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(hash_suites); diff --git a/test/validation/api/hints/.gitignore b/test/validation/api/hints/.gitignore new file mode 100644 index 000000000..586f429bc --- /dev/null +++ b/test/validation/api/hints/.gitignore @@ -0,0 +1 @@ +hints_main diff --git a/test/validation/api/hints/Makefile.am b/test/validation/api/hints/Makefile.am new file mode 100644 index 000000000..bcc77f606 --- /dev/null +++ b/test/validation/api/hints/Makefile.am @@ -0,0 +1,4 @@ +include ../Makefile.inc + +test_PROGRAMS = hints_main +hints_main_SOURCES = hints.c diff --git a/test/validation/api/hints/hints.c b/test/validation/api/hints/hints.c new file mode 100644 index 000000000..4c049f33b --- /dev/null +++ b/test/validation/api/hints/hints.c @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2024 Nokia + */ + +#include <odp_api.h> +#include <odp_cunit_common.h> + +#include <stdint.h> +#include <stdlib.h> + +ODP_NORETURN static void test_noreturn(void) +{ + abort(); +} + +int test_weak(void); + +ODP_WEAK_SYMBOL int test_weak(void) +{ + return 0; +} + +ODP_COLD_CODE static int test_cold(void) +{ + return -1; +} + +ODP_HOT_CODE static int test_hot(void) +{ + return 1; +} + +ODP_PRINTF_FORMAT(2, 3) +static int test_printf_format(int level ODP_UNUSED, const char *fmt ODP_UNUSED, ...) +{ + return 0; +} + +static void test_hints(void) +{ + volatile int val = 1; + + if (odp_unlikely(!val)) + test_noreturn(); + + test_weak(); + test_cold(); + + if (odp_likely(val)) + test_hot(); + + test_printf_format(0, "test"); +} + +static void test_prefetch(void) +{ + const int rounds = 10; + uint64_t data[rounds]; + + for (int i = 0; i < rounds; i++) + odp_prefetch(&data[i]); + + for (int i = 0; i < rounds; i++) + odp_prefetch_store(&data[i]); +} + +odp_testinfo_t hints_suite[] = { + ODP_TEST_INFO(test_hints), + ODP_TEST_INFO(test_prefetch), + ODP_TEST_INFO_NULL, +}; + +odp_suiteinfo_t align_suites[] = { + {"hints", NULL, NULL, hints_suite}, + ODP_SUITE_INFO_NULL +}; + +int main(int argc, char *argv[]) +{ + int ret; + + /* Parse common options */ + if (odp_cunit_parse_options(&argc, argv)) + return -1; + + ret = odp_cunit_register(align_suites); + + if (ret == 0) + ret = odp_cunit_run(); + + return ret; +} diff --git a/test/validation/api/init/.gitignore b/test/validation/api/init/.gitignore index 4e14eb040..f0ce98bdd 100644 --- a/test/validation/api/init/.gitignore +++ b/test/validation/api/init/.gitignore @@ -1,9 +1 @@ -init_defaults -init_abort -init_log -init_log_thread -init_num_thr -init_feature_enabled -init_feature_disabled -init_test_param_init -init_test_term_abnormal +init_main diff --git a/test/validation/api/init/Makefile.am b/test/validation/api/init/Makefile.am index 8226b87ce..7465f683f 100644 --- a/test/validation/api/init/Makefile.am +++ b/test/validation/api/init/Makefile.am @@ -1,27 +1,15 @@ include ../Makefile.inc -# Keep init test cases in separate binaries. Some implementations may not allow -# the same application process to call odp_init_global() multiple times. -test_PROGRAMS = init_defaults init_abort init_log init_num_thr \ - init_feature_enabled init_feature_disabled init_log_thread \ - init_test_param_init init_test_term_abnormal +test_PROGRAMS = init_main +init_main_SOURCES = init_main.c -init_defaults_CPPFLAGS = -DINIT_TEST=0 $(AM_CPPFLAGS) -init_abort_CPPFLAGS = -DINIT_TEST=1 $(AM_CPPFLAGS) -init_log_CPPFLAGS = -DINIT_TEST=2 $(AM_CPPFLAGS) -init_num_thr_CPPFLAGS = -DINIT_TEST=3 $(AM_CPPFLAGS) -init_feature_enabled_CPPFLAGS = -DINIT_TEST=4 $(AM_CPPFLAGS) -init_feature_disabled_CPPFLAGS = -DINIT_TEST=5 $(AM_CPPFLAGS) -init_log_thread_CPPFLAGS = -DINIT_TEST=6 $(AM_CPPFLAGS) -init_test_param_init_CPPFLAGS = -DINIT_TEST=7 $(AM_CPPFLAGS) -init_test_term_abnormal_CPPFLAGS = -DINIT_TEST=8 $(AM_CPPFLAGS) - -init_defaults_SOURCES = init_main.c -init_abort_SOURCES = init_main.c -init_log_SOURCES = init_main.c -init_num_thr_SOURCES = init_main.c -init_feature_enabled_SOURCES = init_main.c -init_feature_disabled_SOURCES = init_main.c -init_log_thread_SOURCES = init_main.c -init_test_param_init_SOURCES = init_main.c -init_test_term_abnormal_SOURCES = init_main.c +EXTRA_DIST = \ + init_defaults.sh \ + init_abort.sh \ + init_log.sh \ + init_num_thr.sh \ + init_feature_enabled.sh \ + init_feature_disabled.sh \ + init_log_thread.sh \ + init_test_param_init.sh \ + init_test_term_abnormal.sh diff --git a/test/validation/api/init/init_abort.sh b/test/validation/api/init/init_abort.sh new file mode 100755 index 000000000..27796fcf9 --- /dev/null +++ b/test/validation/api/init/init_abort.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/init" +$TEST_DIR/init_main$EXEEXT 1 diff --git a/test/validation/api/init/init_defaults.sh b/test/validation/api/init/init_defaults.sh new file mode 100755 index 000000000..2215a65a9 --- /dev/null +++ b/test/validation/api/init/init_defaults.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/init" +$TEST_DIR/init_main$EXEEXT 0 diff --git a/test/validation/api/init/init_feature_disabled.sh b/test/validation/api/init/init_feature_disabled.sh new file mode 100755 index 000000000..e538429b6 --- /dev/null +++ b/test/validation/api/init/init_feature_disabled.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/init" +$TEST_DIR/init_main$EXEEXT 5 diff --git a/test/validation/api/init/init_feature_enabled.sh b/test/validation/api/init/init_feature_enabled.sh new file mode 100755 index 000000000..18237cbf9 --- /dev/null +++ b/test/validation/api/init/init_feature_enabled.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/init" +$TEST_DIR/init_main$EXEEXT 4 diff --git a/test/validation/api/init/init_log.sh b/test/validation/api/init/init_log.sh new file mode 100755 index 000000000..cf4177ed5 --- /dev/null +++ b/test/validation/api/init/init_log.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/init" +$TEST_DIR/init_main$EXEEXT 2 diff --git a/test/validation/api/init/init_log_thread.sh b/test/validation/api/init/init_log_thread.sh new file mode 100755 index 000000000..b0bb02220 --- /dev/null +++ b/test/validation/api/init/init_log_thread.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/init" +$TEST_DIR/init_main$EXEEXT 6 diff --git a/test/validation/api/init/init_main.c b/test/validation/api/init/init_main.c index 64cefa30b..ab1db421b 100644 --- a/test/validation/api/init/init_main.c +++ b/test/validation/api/init/init_main.c @@ -1,8 +1,6 @@ -/* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2019-2023, Nokia - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2018 Linaro Limited + * Copyright (c) 2019-2024 Nokia */ #include <odp_api.h> @@ -297,12 +295,19 @@ static int fill_testinfo(odp_testinfo_t *info, unsigned int test_case) int main(int argc, char *argv[]) { int ret; + int test_id; - if (fill_testinfo(&init_suite[0], INIT_TEST)) + /* Parse common options */ + if (odp_cunit_parse_options(&argc, argv)) return -1; - /* Parse common options */ - if (odp_cunit_parse_options(argc, argv)) + if (argc < 2) { + ODPH_ERR("Usage: init_main <test case number>\n"); + return -1; + } + test_id = atoi(argv[1]); + + if (fill_testinfo(&init_suite[0], test_id)) return -1; /* Prevent default ODP init */ diff --git a/test/validation/api/init/init_num_thr.sh b/test/validation/api/init/init_num_thr.sh new file mode 100755 index 000000000..3889d2a14 --- /dev/null +++ b/test/validation/api/init/init_num_thr.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/init" +$TEST_DIR/init_main$EXEEXT 3 diff --git a/test/validation/api/init/init_test_param_init.sh b/test/validation/api/init/init_test_param_init.sh new file mode 100755 index 000000000..afb4d17a2 --- /dev/null +++ b/test/validation/api/init/init_test_param_init.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/init" +$TEST_DIR/init_main$EXEEXT 7 diff --git a/test/validation/api/init/init_test_term_abnormal.sh b/test/validation/api/init/init_test_term_abnormal.sh new file mode 100755 index 000000000..b3edd8391 --- /dev/null +++ b/test/validation/api/init/init_test_term_abnormal.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/init" +$TEST_DIR/init_main$EXEEXT 8 diff --git a/test/validation/api/ipsec/.gitignore b/test/validation/api/ipsec/.gitignore index 967356064..2def047f3 100644 --- a/test/validation/api/ipsec/.gitignore +++ b/test/validation/api/ipsec/.gitignore @@ -1,4 +1 @@ -ipsec_sync -ipsec_async -ipsec_inline_in -ipsec_inline_out +ipsec_main diff --git a/test/validation/api/ipsec/Makefile.am b/test/validation/api/ipsec/Makefile.am index 1a29d687d..51b50dd02 100644 --- a/test/validation/api/ipsec/Makefile.am +++ b/test/validation/api/ipsec/Makefile.am @@ -11,14 +11,15 @@ libtestipsec_la_SOURCES = \ reass_test_vectors.c test_PROGRAMS = \ - ipsec_sync \ - ipsec_async \ - ipsec_inline_in \ - ipsec_inline_out + ipsec_main -ipsec_sync_SOURCES = ipsec_sync.c -ipsec_async_SOURCES = ipsec_async.c -ipsec_inline_in_SOURCES = ipsec_inline_in.c -ipsec_inline_out_SOURCES = ipsec_inline_out.c +ipsec_main_SOURCES = \ + ipsec_main.c PRELDADD += libtestipsec.la + +EXTRA_DIST = \ + ipsec_sync.sh \ + ipsec_async.sh \ + ipsec_inline_in.sh \ + ipsec_inline_out.sh diff --git a/test/validation/api/ipsec/ipsec.c b/test/validation/api/ipsec/ipsec.c index 9849a44b5..5ad7bd48d 100644 --- a/test/validation/api/ipsec/ipsec.c +++ b/test/validation/api/ipsec/ipsec.c @@ -29,7 +29,6 @@ static odp_ipsec_capability_t capa; static int sched_ev_buffer_tail; odp_bool_t sa_expiry_notified; - #define PKT_POOL_NUM 64 #define EVENT_WAIT_TIME ODP_TIME_SEC_IN_NS #define STATUS_EVENT_WAIT_TIME ODP_TIME_MSEC_IN_NS @@ -38,6 +37,8 @@ odp_bool_t sa_expiry_notified; #define PACKET_USER_PTR ((void *)0x1212fefe) #define IPSEC_SA_CTX ((void *)0xfefefafa) +static int ipsec_config(void); + static odp_pktio_t pktio_create(odp_pool_t pool) { odp_pktio_t pktio; @@ -1296,16 +1297,6 @@ int ipsec_suite_term(void) return 0; } -int ipsec_in_term(void) -{ - return ipsec_suite_term(); -} - -int ipsec_out_term(void) -{ - return ipsec_suite_term(); -} - static odp_queue_t sched_queue_create(const char *name) { odp_queue_param_t qparam; @@ -1366,12 +1357,10 @@ int ipsec_suite_sched_init(void) return ipsec_suite_init(); } -int ipsec_init(odp_instance_t *inst, odp_ipsec_op_mode_t mode) +int ipsec_init(odp_instance_t *inst) { odp_pool_param_t params; - odp_pool_t pool; odp_pool_capability_t pool_capa; - odp_pktio_t pktio; odp_init_t init_param; odph_helper_options_t helper_options; @@ -1427,25 +1416,26 @@ int ipsec_init(odp_instance_t *inst, odp_ipsec_op_mode_t mode) return -1; } - pool = odp_pool_create("packet_pool", ¶ms); + suite_context.pool = odp_pool_create("packet_pool", ¶ms); - if (ODP_POOL_INVALID == pool) { + if (suite_context.pool == ODP_POOL_INVALID) { ODPH_ERR("Packet pool creation failed\n"); return -1; } - if (mode == ODP_IPSEC_OP_MODE_INLINE) { - pktio = pktio_create(pool); - if (ODP_PKTIO_INVALID == pktio) { + if (suite_context.inbound_op_mode == ODP_IPSEC_OP_MODE_INLINE || + suite_context.outbound_op_mode == ODP_IPSEC_OP_MODE_INLINE) { + suite_context.pktio = pktio_create(suite_context.pool); + if (suite_context.pktio == ODP_PKTIO_INVALID) { ODPH_ERR("IPsec pktio creation failed\n"); return -1; } } - return 0; + return ipsec_config(); } -int ipsec_config(odp_instance_t ODP_UNUSED inst) +static int ipsec_config(void) { odp_ipsec_config_t ipsec_config; diff --git a/test/validation/api/ipsec/ipsec.h b/test/validation/api/ipsec/ipsec.h index 8eed65c50..47612e3b3 100644 --- a/test/validation/api/ipsec/ipsec.h +++ b/test/validation/api/ipsec/ipsec.h @@ -20,9 +20,8 @@ extern odp_testinfo_t ipsec_in_suite[]; extern odp_testinfo_t ipsec_out_suite[]; -int ipsec_init(odp_instance_t *inst, odp_ipsec_op_mode_t mode); +int ipsec_init(odp_instance_t *inst); int ipsec_term(odp_instance_t inst); -int ipsec_config(odp_instance_t inst); int ipsec_in_inline_init(void); int ipsec_out_inline_init(void); @@ -31,8 +30,6 @@ int ipsec_suite_sync_init(void); int ipsec_suite_plain_init(void); int ipsec_suite_sched_init(void); int ipsec_suite_term(void); -int ipsec_in_term(void); -int ipsec_out_term(void); struct suite_context_s { odp_bool_t reass_ipv4; diff --git a/test/validation/api/ipsec/ipsec_async.c b/test/validation/api/ipsec/ipsec_async.c deleted file mode 100644 index 44c968a4d..000000000 --- a/test/validation/api/ipsec/ipsec_async.c +++ /dev/null @@ -1,55 +0,0 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include "ipsec.h" - -static int ipsec_async_init(odp_instance_t *inst) -{ - int rc; - - suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; - suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; - - rc = ipsec_init(inst, ODP_IPSEC_OP_MODE_ASYNC); - if (rc != 0) - return rc; - - suite_context.pool = odp_pool_lookup("packet_pool"); - if (suite_context.pool == ODP_POOL_INVALID) - return -1; - - return ipsec_config(*inst); -} - -odp_suiteinfo_t ipsec_suites[] = { - {"IPsec-plain-in", ipsec_suite_plain_init, ipsec_suite_term, - ipsec_in_suite}, - {"IPsec-sched-in", ipsec_suite_sched_init, ipsec_suite_term, - ipsec_in_suite}, - {"IPsec-plain-out", ipsec_suite_plain_init, ipsec_suite_term, - ipsec_out_suite}, - {"IPsec-sched-out", ipsec_suite_sched_init, ipsec_suite_term, - ipsec_out_suite}, - ODP_SUITE_INFO_NULL, -}; - -int main(int argc, char *argv[]) -{ - int ret; - - /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) - return -1; - - odp_cunit_register_global_init(ipsec_async_init); - odp_cunit_register_global_term(ipsec_term); - - ret = odp_cunit_register(ipsec_suites); - if (ret == 0) - ret = odp_cunit_run(); - - return ret; -} diff --git a/test/validation/api/ipsec/ipsec_async.sh b/test/validation/api/ipsec/ipsec_async.sh new file mode 100755 index 000000000..c12f3e70a --- /dev/null +++ b/test/validation/api/ipsec/ipsec_async.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/ipsec" +$TEST_DIR/ipsec_main$EXEEXT async diff --git a/test/validation/api/ipsec/ipsec_inline_in.c b/test/validation/api/ipsec/ipsec_inline_in.c deleted file mode 100644 index cbb3a178a..000000000 --- a/test/validation/api/ipsec/ipsec_inline_in.c +++ /dev/null @@ -1,54 +0,0 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include "ipsec.h" - -static int ipsec_sync_init(odp_instance_t *inst) -{ - int rc; - - suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_INLINE; - suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; - - rc = ipsec_init(inst, ODP_IPSEC_OP_MODE_INLINE); - if (rc != 0) - return rc; - - suite_context.pool = odp_pool_lookup("packet_pool"); - if (suite_context.pool == ODP_POOL_INVALID) - return -1; - suite_context.pktio = odp_pktio_lookup("loop"); - if (suite_context.pktio == ODP_PKTIO_INVALID) - return -1; - - return ipsec_config(*inst); -} - -odp_suiteinfo_t ipsec_suites[] = { - {"IPsec-plain-in", ipsec_suite_plain_init, ipsec_suite_term, - ipsec_in_suite}, - {"IPsec-sched-in", ipsec_suite_sched_init, ipsec_suite_term, - ipsec_in_suite}, - ODP_SUITE_INFO_NULL, -}; - -int main(int argc, char *argv[]) -{ - int ret; - - /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) - return -1; - - odp_cunit_register_global_init(ipsec_sync_init); - odp_cunit_register_global_term(ipsec_term); - - ret = odp_cunit_register(ipsec_suites); - if (ret == 0) - ret = odp_cunit_run(); - - return ret; -} diff --git a/test/validation/api/ipsec/ipsec_inline_in.sh b/test/validation/api/ipsec/ipsec_inline_in.sh new file mode 100755 index 000000000..c1d1fed36 --- /dev/null +++ b/test/validation/api/ipsec/ipsec_inline_in.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/ipsec" +$TEST_DIR/ipsec_main$EXEEXT inline-in diff --git a/test/validation/api/ipsec/ipsec_inline_out.c b/test/validation/api/ipsec/ipsec_inline_out.c deleted file mode 100644 index 3da19892b..000000000 --- a/test/validation/api/ipsec/ipsec_inline_out.c +++ /dev/null @@ -1,54 +0,0 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include "ipsec.h" - -static int ipsec_sync_init(odp_instance_t *inst) -{ - int rc; - - suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; - suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_INLINE; - - rc = ipsec_init(inst, ODP_IPSEC_OP_MODE_INLINE); - if (rc != 0) - return rc; - - suite_context.pool = odp_pool_lookup("packet_pool"); - if (suite_context.pool == ODP_POOL_INVALID) - return -1; - suite_context.pktio = odp_pktio_lookup("loop"); - if (suite_context.pktio == ODP_PKTIO_INVALID) - return -1; - - return ipsec_config(*inst); -} - -odp_suiteinfo_t ipsec_suites[] = { - {"IPsec-plain-out", ipsec_suite_plain_init, ipsec_suite_term, - ipsec_out_suite}, - {"IPsec-sched-out", ipsec_suite_sched_init, ipsec_suite_term, - ipsec_out_suite}, - ODP_SUITE_INFO_NULL, -}; - -int main(int argc, char *argv[]) -{ - int ret; - - /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) - return -1; - - odp_cunit_register_global_init(ipsec_sync_init); - odp_cunit_register_global_term(ipsec_term); - - ret = odp_cunit_register(ipsec_suites); - if (ret == 0) - ret = odp_cunit_run(); - - return ret; -} diff --git a/test/validation/api/ipsec/ipsec_inline_out.sh b/test/validation/api/ipsec/ipsec_inline_out.sh new file mode 100755 index 000000000..91ebccac7 --- /dev/null +++ b/test/validation/api/ipsec/ipsec_inline_out.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/ipsec" +$TEST_DIR/ipsec_main$EXEEXT inline-out diff --git a/test/validation/api/ipsec/ipsec_main.c b/test/validation/api/ipsec/ipsec_main.c new file mode 100644 index 000000000..2c343f063 --- /dev/null +++ b/test/validation/api/ipsec/ipsec_main.c @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2017-2018 Linaro Limited + * Copyright (c) 2024 Nokia + */ + +#include <odp_api.h> +#include <odp_cunit_common.h> +#include "ipsec.h" + +odp_suiteinfo_t ipsec_sync_suites[] = { + {"IPsec-in", ipsec_suite_sync_init, ipsec_suite_term, ipsec_in_suite}, + {"IPsec-out", ipsec_suite_sync_init, ipsec_suite_term, ipsec_out_suite}, + ODP_SUITE_INFO_NULL, +}; + +odp_suiteinfo_t ipsec_async_suites[] = { + {"IPsec-plain-in", ipsec_suite_plain_init, ipsec_suite_term, ipsec_in_suite}, + {"IPsec-sched-in", ipsec_suite_sched_init, ipsec_suite_term, ipsec_in_suite}, + {"IPsec-plain-out", ipsec_suite_plain_init, ipsec_suite_term, ipsec_out_suite}, + {"IPsec-sched-out", ipsec_suite_sched_init, ipsec_suite_term, ipsec_out_suite}, + ODP_SUITE_INFO_NULL, +}; + +odp_suiteinfo_t ipsec_inline_in_suites[] = { + {"IPsec-plain-in", ipsec_suite_plain_init, ipsec_suite_term, ipsec_in_suite}, + {"IPsec-sched-in", ipsec_suite_sched_init, ipsec_suite_term, ipsec_in_suite}, + ODP_SUITE_INFO_NULL, +}; + +odp_suiteinfo_t ipsec_inline_out_suites[] = { + {"IPsec-plain-out", ipsec_suite_plain_init, ipsec_suite_term, ipsec_out_suite}, + {"IPsec-sched-out", ipsec_suite_sched_init, ipsec_suite_term, ipsec_out_suite}, + ODP_SUITE_INFO_NULL, +}; + +static void usage_exit(void) +{ + fprintf(stderr, "Usage: ipsec_main {sync|async|inline-in|inline-out}\n"); + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) +{ + char *test_mode; + odp_suiteinfo_t *suites = NULL; + int ret; + + if (odp_cunit_parse_options(&argc, argv)) + return EXIT_FAILURE; + + if (argc < 2) + usage_exit(); + test_mode = argv[1]; + + if ((!strcmp(test_mode, "sync"))) { + suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_SYNC; + suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_SYNC; + suites = ipsec_sync_suites; + } else if ((!strcmp(test_mode, "async"))) { + suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; + suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; + suites = ipsec_async_suites; + } else if ((!strcmp(test_mode, "inline-in"))) { + suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_INLINE; + suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; + suites = ipsec_inline_in_suites; + } else if ((!strcmp(test_mode, "inline-out"))) { + suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_ASYNC; + suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_INLINE; + suites = ipsec_inline_out_suites; + } else { + usage_exit(); + } + + odp_cunit_register_global_init(ipsec_init); + odp_cunit_register_global_term(ipsec_term); + ret = odp_cunit_register(suites); + + if (ret == 0) + ret = odp_cunit_run(); + + return ret; +} diff --git a/test/validation/api/ipsec/ipsec_sync.c b/test/validation/api/ipsec/ipsec_sync.c deleted file mode 100644 index 74ae1fe19..000000000 --- a/test/validation/api/ipsec/ipsec_sync.c +++ /dev/null @@ -1,49 +0,0 @@ -/* Copyright (c) 2017-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include "ipsec.h" - -static int ipsec_sync_init(odp_instance_t *inst) -{ - int rc; - - suite_context.inbound_op_mode = ODP_IPSEC_OP_MODE_SYNC; - suite_context.outbound_op_mode = ODP_IPSEC_OP_MODE_SYNC; - - rc = ipsec_init(inst, ODP_IPSEC_OP_MODE_SYNC); - if (rc != 0) - return rc; - - suite_context.pool = odp_pool_lookup("packet_pool"); - if (suite_context.pool == ODP_POOL_INVALID) - return -1; - - return ipsec_config(*inst); -} - -odp_suiteinfo_t ipsec_suites[] = { - {"IPsec-in", ipsec_suite_sync_init, ipsec_in_term, ipsec_in_suite}, - {"IPsec-out", ipsec_suite_sync_init, ipsec_out_term, ipsec_out_suite}, - ODP_SUITE_INFO_NULL, -}; - -int main(int argc, char *argv[]) -{ - int ret; - - /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) - return -1; - - odp_cunit_register_global_init(ipsec_sync_init); - odp_cunit_register_global_term(ipsec_term); - - ret = odp_cunit_register(ipsec_suites); - if (ret == 0) - ret = odp_cunit_run(); - - return ret; -} diff --git a/test/validation/api/ipsec/ipsec_sync.sh b/test/validation/api/ipsec/ipsec_sync.sh new file mode 100755 index 000000000..2653ddc78 --- /dev/null +++ b/test/validation/api/ipsec/ipsec_sync.sh @@ -0,0 +1,3 @@ +#!/bin/sh +TEST_DIR="${TEST_DIR:-$(dirname $0)/..}/ipsec" +$TEST_DIR/ipsec_main$EXEEXT sync diff --git a/test/validation/api/lock/lock.c b/test/validation/api/lock/lock.c index 78c6ee79b..a4e6932c4 100644 --- a/test/validation/api/lock/lock.c +++ b/test/validation/api/lock/lock.c @@ -1246,7 +1246,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(lock_init); diff --git a/test/validation/api/ml/.gitignore b/test/validation/api/ml/.gitignore new file mode 100644 index 000000000..be2347720 --- /dev/null +++ b/test/validation/api/ml/.gitignore @@ -0,0 +1 @@ +ml_main diff --git a/test/validation/api/ml/Makefile.am b/test/validation/api/ml/Makefile.am new file mode 100644 index 000000000..b7946a963 --- /dev/null +++ b/test/validation/api/ml/Makefile.am @@ -0,0 +1,4 @@ +include ../Makefile.inc + +test_PROGRAMS = ml_main +ml_main_SOURCES = ml.c diff --git a/test/validation/api/ml/ml.c b/test/validation/api/ml/ml.c new file mode 100644 index 000000000..5f8be1b64 --- /dev/null +++ b/test/validation/api/ml/ml.c @@ -0,0 +1,572 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2023 Nokia + */ + +#include <odp_api.h> +#include <odp/helper/odph_api.h> +#include "odp_cunit_common.h" + +#define UAREA 0xaa +#define NUM_COMPL 10u +#define COMPL_POOL_NAME "ML compl pool" + +typedef struct global_t { + int disabled; + uint32_t num_compl; + odp_ml_capability_t ml_capa; +} global_t; + +typedef struct { + uint32_t count; + uint8_t mark[NUM_COMPL]; +} uarea_init_t; + +static global_t global; + +static int ml_suite_init(void) +{ + memset(&global, 0, sizeof(global_t)); + + if (odp_ml_capability(&global.ml_capa)) { + ODPH_ERR("ML capability failed\n"); + return -1; + } + + if (global.ml_capa.max_models == 0) { + global.disabled = 1; + ODPH_DBG("ML test disabled\n"); + return 0; + } + + global.num_compl = ODPH_MIN(NUM_COMPL, global.ml_capa.pool.max_num); + + return 0; +} + +static int check_ml_support(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + return ODP_TEST_ACTIVE; +} + +static void test_ml_capability(void) +{ + odp_ml_capability_t ml_capa; + + memset(&ml_capa, 0, sizeof(odp_ml_capability_t)); + CU_ASSERT(odp_ml_capability(&ml_capa) == 0); + + if (ml_capa.max_models == 0) + return; + + CU_ASSERT(ml_capa.max_model_size > 0); + CU_ASSERT(ml_capa.max_models_loaded > 0); + CU_ASSERT(ml_capa.max_inputs > 0); + CU_ASSERT(ml_capa.max_outputs > 0); + CU_ASSERT(ml_capa.max_segs_per_input > 0); + CU_ASSERT(ml_capa.max_segs_per_output > 0); + CU_ASSERT(ml_capa.min_input_align > 0); + CU_ASSERT(ml_capa.min_output_align > 0); + + if ((ml_capa.load.compl_mode_mask | ml_capa.run.compl_mode_mask) & + ODP_ML_COMPL_MODE_EVENT) { + odp_pool_capability_t pool_capa; + + CU_ASSERT_FATAL(odp_pool_capability(&pool_capa) == 0); + + CU_ASSERT(ml_capa.pool.max_pools > 0); + CU_ASSERT(ml_capa.pool.max_pools <= pool_capa.max_pools); + CU_ASSERT(ml_capa.pool.max_num > 0); + CU_ASSERT(ml_capa.pool.max_cache_size >= ml_capa.pool.min_cache_size); + } + + CU_ASSERT(ml_capa.load.compl_mode_mask); + CU_ASSERT(ml_capa.run.compl_mode_mask); + + if (ml_capa.load.compl_mode_mask & ODP_ML_COMPL_MODE_EVENT) + CU_ASSERT(ml_capa.load.compl_queue_plain || ml_capa.load.compl_queue_sched); + + if (ml_capa.run.compl_mode_mask & ODP_ML_COMPL_MODE_EVENT) + CU_ASSERT(ml_capa.run.compl_queue_plain || ml_capa.run.compl_queue_sched); +} + +static void test_ml_param(uint8_t fill) +{ + odp_ml_config_t config; + odp_ml_model_param_t model_param; + odp_ml_compl_pool_param_t pool_param; + odp_ml_compl_param_t compl_param; + odp_ml_run_param_t run_param; + + memset(&config, fill, sizeof(config)); + odp_ml_config_init(&config); + CU_ASSERT(config.max_models_created == 1); + CU_ASSERT(config.max_models_loaded == 1); + CU_ASSERT(config.load_mode_mask == 0); + CU_ASSERT(config.run_mode_mask == 0); + + memset(&model_param, fill, sizeof(model_param)); + odp_ml_model_param_init(&model_param); + CU_ASSERT(model_param.max_compl_id == 0); + CU_ASSERT(!model_param.extra_stat_enable); + CU_ASSERT(model_param.extra_param == NULL); + CU_ASSERT(model_param.extra_info.num_inputs == 0); + CU_ASSERT(model_param.extra_info.num_outputs == 0); + CU_ASSERT(model_param.extra_info.input_format == NULL); + CU_ASSERT(model_param.extra_info.output_format == NULL); + + memset(&pool_param, fill, sizeof(pool_param)); + odp_ml_compl_pool_param_init(&pool_param); + CU_ASSERT(pool_param.uarea_size == 0); + CU_ASSERT(pool_param.uarea_init.args == NULL); + CU_ASSERT(pool_param.uarea_init.init_fn == NULL); + CU_ASSERT(pool_param.cache_size <= global.ml_capa.pool.max_cache_size); + CU_ASSERT(pool_param.cache_size >= global.ml_capa.pool.min_cache_size); + + memset(&compl_param, fill, sizeof(compl_param)); + odp_ml_compl_param_init(&compl_param); + CU_ASSERT(compl_param.user_ptr == NULL); + + memset(&run_param, fill, sizeof(run_param)); + odp_ml_run_param_init(&run_param); + CU_ASSERT(run_param.batch_size == 0); + CU_ASSERT(run_param.result == NULL); +} + +static void test_ml_param_init(void) +{ + test_ml_param(0x00); + test_ml_param(0xff); +} + +static void test_ml_debug(void) +{ + odp_ml_print(); +} + +static void ml_compl_pool_create_max_pools(void) +{ + int ret; + uint32_t i, j; + odp_ml_compl_pool_param_t ml_pool_param; + uint32_t max_pools = global.ml_capa.pool.max_pools; + odp_pool_t compl_pools[max_pools]; + + odp_ml_compl_pool_param_init(&ml_pool_param); + ml_pool_param.num = global.num_compl; + for (i = 0; i < max_pools; i++) { + compl_pools[i] = odp_ml_compl_pool_create(NULL, &ml_pool_param); + + if (compl_pools[i] == ODP_POOL_INVALID) + break; + } + + CU_ASSERT(i == max_pools); + + /* Destroy the created valid pools */ + for (j = 0; j < i; j++) { + ret = odp_pool_destroy(compl_pools[j]); + CU_ASSERT(ret == 0); + + if (ret == -1) + ODPH_ERR("ML completion pool destroy failed: %u / %u\n", j, i); + } +} + +static void compl_pool_info(void) +{ + odp_pool_t pool; + odp_pool_t compl_pool; + odp_pool_info_t pool_info; + odp_ml_compl_pool_param_t pool_param; + + /* Create an ML job completion pool */ + odp_ml_compl_pool_param_init(&pool_param); + pool_param.num = global.num_compl; + + compl_pool = odp_ml_compl_pool_create(COMPL_POOL_NAME, &pool_param); + CU_ASSERT_FATAL(compl_pool != ODP_POOL_INVALID); + + /* Verify info about the created ML completion pool compl_pool */ + pool = odp_pool_lookup(COMPL_POOL_NAME); + CU_ASSERT(pool == compl_pool); + + memset(&pool_info, 0x66, sizeof(odp_pool_info_t)); + CU_ASSERT_FATAL(odp_pool_info(compl_pool, &pool_info) == 0); + + CU_ASSERT(!strcmp(pool_info.name, COMPL_POOL_NAME)); + CU_ASSERT(pool_info.pool_ext == 0); + CU_ASSERT(pool_info.type == ODP_POOL_ML_COMPL); + CU_ASSERT(pool_info.ml_pool_param.num == NUM_COMPL); + CU_ASSERT(pool_info.ml_pool_param.uarea_size == 0); + CU_ASSERT(pool_info.ml_pool_param.cache_size == pool_param.cache_size); + + CU_ASSERT_FATAL(odp_pool_destroy(compl_pool) == 0); +} + +static void compl_alloc_max(void) +{ + uint64_t u64; + odp_event_t event; + odp_pool_t compl_pool; + odp_ml_compl_pool_param_t pool_param; + const int num = ODPH_MIN(global.ml_capa.pool.max_num, 1000000u); + odp_ml_compl_t *compl = malloc(num * sizeof(odp_ml_compl_t)); + + CU_ASSERT_FATAL(compl != NULL); + + /* Create an ML job completion pool */ + odp_ml_compl_pool_param_init(&pool_param); + pool_param.num = num; + + compl_pool = odp_ml_compl_pool_create(COMPL_POOL_NAME, &pool_param); + CU_ASSERT_FATAL(compl_pool != ODP_POOL_INVALID); + + for (int i = 0; i < num; i++) { + compl[i] = odp_ml_compl_alloc(compl_pool); + CU_ASSERT_FATAL(compl[i] != ODP_ML_COMPL_INVALID); + + u64 = odp_ml_compl_to_u64(compl[i]); + CU_ASSERT(u64 != odp_ml_compl_to_u64(ODP_ML_COMPL_INVALID)); + + event = odp_ml_compl_to_event(compl[i]); + CU_ASSERT(odp_event_type(event) == ODP_EVENT_ML_COMPL); + } + + for (int i = 0; i < num; i++) + odp_ml_compl_free(compl[i]); + + free(compl); + + CU_ASSERT_FATAL(odp_pool_destroy(compl_pool) == 0); +} + +static void compl_pool_lookup(void) +{ + odp_pool_t pool, pool_a, pool_b; + odp_ml_compl_pool_param_t pool_param; + + /* Create an ML job completion pool */ + odp_ml_compl_pool_param_init(&pool_param); + pool_param.num = global.num_compl; + + pool_a = odp_ml_compl_pool_create(COMPL_POOL_NAME, &pool_param); + CU_ASSERT_FATAL(pool_a != ODP_POOL_INVALID); + + pool = odp_pool_lookup(COMPL_POOL_NAME); + CU_ASSERT(pool == pool_a); + + /* Second pool with the same name */ + pool_b = odp_ml_compl_pool_create(COMPL_POOL_NAME, &pool_param); + CU_ASSERT_FATAL(pool_b != ODP_POOL_INVALID); + + pool = odp_pool_lookup(COMPL_POOL_NAME); + CU_ASSERT(pool == pool_a || pool == pool_b); + + CU_ASSERT(odp_pool_destroy(pool_a) == 0); + CU_ASSERT(odp_pool_destroy(pool_b) == 0); +} + +static int check_event_user_area(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + if (((global.ml_capa.load.compl_mode_mask | global.ml_capa.run.compl_mode_mask) & + ODP_ML_COMPL_MODE_EVENT) && + (global.ml_capa.pool.max_uarea_size > 0)) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + +static void test_ml_compl_user_area(void) +{ + uint32_t i; + void *addr; + void *prev; + odp_pool_t pool; + odp_ml_compl_pool_param_t pool_param; + uint32_t size = global.ml_capa.pool.max_uarea_size; + uint32_t num = global.num_compl; + odp_ml_compl_t compl_evs[num]; + + odp_ml_compl_pool_param_init(&pool_param); + pool_param.num = num; + pool_param.uarea_size = size; + pool = odp_ml_compl_pool_create(NULL, &pool_param); + CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); + + prev = NULL; + for (i = 0; i < num; i++) { + odp_event_t ev; + int flag = 0; + + compl_evs[i] = odp_ml_compl_alloc(pool); + + if (compl_evs[i] == ODP_ML_COMPL_INVALID) + break; + + addr = odp_ml_compl_user_area(compl_evs[i]); + + CU_ASSERT_FATAL(addr != NULL); + CU_ASSERT(prev != addr); + + memset(addr, 0, size); + + ev = odp_ml_compl_to_event(compl_evs[i]); + CU_ASSERT(odp_event_user_area(ev) == addr); + CU_ASSERT(odp_event_user_area_and_flag(ev, &flag) == addr); + CU_ASSERT(flag < 0); + + prev = addr; + } + CU_ASSERT(i == num); + + for (uint32_t j = 0; j < i; j++) + odp_ml_compl_free(compl_evs[j]); + + CU_ASSERT(odp_pool_destroy(pool) == 0); +} + +static int check_event_user_area_init(void) +{ + if (global.disabled) + return ODP_TEST_INACTIVE; + + if (global.ml_capa.pool.max_uarea_size > 0 && global.ml_capa.pool.uarea_persistence) + return ODP_TEST_ACTIVE; + + return ODP_TEST_INACTIVE; +} + +static void init_event_uarea(void *uarea, uint32_t size, void *args, uint32_t index) +{ + uarea_init_t *data = args; + + data->count++; + data->mark[index] = 1; + memset(uarea, UAREA, size); +} + +static void test_ml_compl_user_area_init(void) +{ + odp_ml_compl_pool_param_t pool_param; + uint32_t num = global.num_compl, i; + odp_pool_t pool; + uarea_init_t data; + odp_ml_compl_t compl_evs[num]; + uint8_t *uarea; + + memset(&data, 0, sizeof(uarea_init_t)); + odp_ml_compl_pool_param_init(&pool_param); + pool_param.uarea_init.init_fn = init_event_uarea; + pool_param.uarea_init.args = &data; + pool_param.num = num; + pool_param.uarea_size = 1; + pool = odp_ml_compl_pool_create(NULL, &pool_param); + + CU_ASSERT_FATAL(pool != ODP_POOL_INVALID); + CU_ASSERT(data.count == num); + + for (i = 0; i < num; i++) { + CU_ASSERT(data.mark[i] == 1); + + compl_evs[i] = odp_ml_compl_alloc(pool); + + CU_ASSERT(compl_evs[i] != ODP_ML_COMPL_INVALID); + + if (compl_evs[i] == ODP_ML_COMPL_INVALID) + break; + + uarea = odp_ml_compl_user_area(compl_evs[i]); + + CU_ASSERT(*uarea == UAREA); + } + + for (uint32_t j = 0; j < i; j++) + odp_ml_compl_free(compl_evs[j]); + + odp_pool_destroy(pool); +} + +static void test_ml_fp32_to_uint8(void) +{ + uint8_t u8[8]; + float fp[8] = {-20.f, -16.4f, -14.6f, -12.5f, 0, 31.4f, 80.f, 96.3f}; + uint8_t expected[8] = {0, 0, 4, 10, 43, 127, 255, 255}; + float scale = 0.3746f; + uint8_t zero_point = 43; + + odp_ml_fp32_to_uint8(u8, fp, 8, scale, zero_point); + for (uint32_t i = 0; i < 8; i++) + CU_ASSERT(u8[i] == expected[i]); +} + +static void test_ml_fp32_from_uint8(void) +{ + float fp[4]; + float scale = 0.4f; + uint8_t zero_point = 43; + uint8_t u8[4] = {0, 43, 145, 255}; + float expected[4] = {-17.2f, 0.0f, 40.8f, 84.8f}; + + odp_ml_fp32_from_uint8(fp, u8, 4, scale, zero_point); + for (uint32_t i = 0; i < 4; i++) + CU_ASSERT(fp[i] == expected[i]); +} + +static void test_ml_fp32_to_int8(void) +{ + int8_t i8[5]; + float scale = 0.0223f; + int8_t zero_point = 0; + float fp32[5] = {-3.4f, -2.5f, 0, 1.4f, 2.9f}; + int8_t i8_expected[5] = {-127, -112, 0, 63, 127}; + + odp_ml_fp32_to_int8(i8, fp32, 5, scale, zero_point); + + for (uint32_t i = 0; i < 5; i++) + CU_ASSERT(i8[i] == i8_expected[i]); +} + +static void test_ml_fp32_to_int8_positive_zp(void) +{ + int8_t i8[6]; + float scale = 0.0223f; + int8_t zero_point = 56; + float fp32[6] = {-4.1f, -3.4f, -2.5f, 0, 1.4f, 2.9f}; + int8_t i8_expected[6] = {-127, -96, -56, 56, 119, 127}; + + odp_ml_fp32_to_int8(i8, fp32, 6, scale, zero_point); + + for (uint32_t i = 0; i < 6; i++) + CU_ASSERT(i8[i] == i8_expected[i]); +} + +static void test_ml_fp32_to_int8_negative_zp(void) +{ + int8_t i8[6]; + float scale = 0.0223f; + int8_t zero_point = -56; + float fp32[6] = {-3.4f, -2.5f, 0, 1.4f, 2.9f, 4.1f}; + int8_t i8_expected[6] = {-127, -127, -56, 7, 74, 127}; + + odp_ml_fp32_to_int8(i8, fp32, 6, scale, zero_point); + + for (uint32_t i = 0; i < 6; i++) + CU_ASSERT(i8[i] == i8_expected[i]); +} + +static void test_ml_fp32_from_int8(void) +{ + float fp32[6]; + float scale = 0.05f; + int8_t zero_point = 56; + int8_t i8[6] = {-128, 46, 0, 56, 85, 127}; + float fp32_expected[6] = {-9.2f, -0.5f, -2.8f, 0.0f, 1.45f, 3.55f}; + + odp_ml_fp32_from_int8(fp32, i8, 6, scale, zero_point); + + for (uint32_t i = 0; i < 6; i++) + CU_ASSERT(fp32[i] == fp32_expected[i]); +} + +static int approx_equal(double a, double b) +{ + const double tolerance = .01; + + if (a < 0 && b < 0) { + a = -a; + b = -b; + } + + if (a < b) { + double tmp = a; + + a = b; + b = tmp; + } + + return (a * (1 - tolerance) < b && a * (1 + tolerance) > b); +} + +static void test_ml_fp32_fp16(void) +{ + float fp32[4]; + uint16_t fp16[4]; + float big = 1, small = 1; + const float phi = 1.618033988749; + + fp32[0] = 0.0; + fp32[1] = -0.0; + memset(fp16, 1, sizeof(fp16)); + odp_ml_fp32_to_fp16(fp16, fp32, 2); + memset(fp32, 1, sizeof(fp32)); + odp_ml_fp32_from_fp16(fp32, fp16, 2); + CU_ASSERT(fp32[0] == 0); + CU_ASSERT(fp32[1] == 0); + + /* + * 65504 is the largest normal number for fp16 with 5 exponent bits (IEEE 754-2008). + */ + while (big < 65504 / 2) { + fp32[0] = big; + fp32[1] = -big; + fp32[2] = small; + fp32[3] = -small; + memset(fp16, 0, sizeof(fp16)); + odp_ml_fp32_to_fp16(fp16, fp32, 4); + memset(fp32, 0, sizeof(fp32)); + odp_ml_fp32_from_fp16(fp32, fp16, 4); + CU_ASSERT(approx_equal(fp32[0], big)); + CU_ASSERT(approx_equal(fp32[1], -big)); + CU_ASSERT(approx_equal(fp32[2], small)); + CU_ASSERT(approx_equal(fp32[3], -small)); + big *= phi; + small /= phi; + } +} + +odp_testinfo_t ml_suite[] = { + ODP_TEST_INFO(test_ml_capability), + ODP_TEST_INFO_CONDITIONAL(test_ml_param_init, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_debug, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(ml_compl_pool_create_max_pools, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(compl_pool_info, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(compl_alloc_max, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(compl_pool_lookup, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_compl_user_area, check_event_user_area), + ODP_TEST_INFO_CONDITIONAL(test_ml_compl_user_area_init, check_event_user_area_init), + ODP_TEST_INFO_CONDITIONAL(test_ml_fp32_to_uint8, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_fp32_from_uint8, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_fp32_to_int8, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_fp32_to_int8_positive_zp, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_fp32_to_int8_negative_zp, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_fp32_from_int8, check_ml_support), + ODP_TEST_INFO_CONDITIONAL(test_ml_fp32_fp16, check_ml_support), + ODP_TEST_INFO_NULL +}; + +odp_suiteinfo_t ml_suites[] = { + {"ML", ml_suite_init, NULL, ml_suite}, + ODP_SUITE_INFO_NULL +}; + +int main(int argc, char *argv[]) +{ + int ret; + + /* parse common options: */ + if (odp_cunit_parse_options(&argc, argv)) + return -1; + + ret = odp_cunit_register(ml_suites); + + if (ret == 0) + ret = odp_cunit_run(); + + return ret; +} diff --git a/test/validation/api/packet/packet.c b/test/validation/api/packet/packet.c index 77a0f8494..ca9c73f17 100644 --- a/test/validation/api/packet/packet.c +++ b/test/validation/api/packet/packet.c @@ -4585,7 +4585,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(packet_suites); diff --git a/test/validation/api/pktio/lso.c b/test/validation/api/pktio/lso.c index 3e033ee8b..832c08859 100644 --- a/test/validation/api/pktio/lso.c +++ b/test/validation/api/pktio/lso.c @@ -689,9 +689,9 @@ static void lso_send_custom_eth(const uint8_t *test_packet, uint32_t pkt_len, ui ODPH_DBG("\n Sent payload length: %u bytes\n", sent_payload); - /* Wait 1 sec to receive all created segments. Timeout and MAX_NUM_SEG values should be + /* Wait a bit to receive all created segments. Timeout and MAX_NUM_SEG values should be * large enough to ensure that we receive all created segments. */ - num = recv_packets(pktio_b, ODP_TIME_SEC_IN_NS, pkt_out, MAX_NUM_SEG); + num = recv_packets(pktio_b, 100 * ODP_TIME_MSEC_IN_NS, pkt_out, MAX_NUM_SEG); CU_ASSERT(num > 0); CU_ASSERT(num < MAX_NUM_SEG); @@ -814,9 +814,9 @@ static void lso_send_ipv4(const uint8_t *test_packet, uint32_t pkt_len, uint32_t ODPH_DBG("\n Sent payload length: %u bytes\n", sent_payload); - /* Wait 1 sec to receive all created segments. Timeout and MAX_NUM_SEG values should be + /* Wait a bit to receive all created segments. Timeout and MAX_NUM_SEG values should be * large enough to ensure that we receive all created segments. */ - num = recv_packets(pktio_b, ODP_TIME_SEC_IN_NS, packet, MAX_NUM_SEG); + num = recv_packets(pktio_b, 100 * ODP_TIME_MSEC_IN_NS, packet, MAX_NUM_SEG); CU_ASSERT(num > 0); CU_ASSERT(num < MAX_NUM_SEG); diff --git a/test/validation/api/pktio/pktio.c b/test/validation/api/pktio/pktio.c index 780d83066..deef4895a 100644 --- a/test/validation/api/pktio/pktio.c +++ b/test/validation/api/pktio/pktio.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2020-2023, Nokia + * Copyright (c) 2020-2024, Nokia * Copyright (c) 2020, Marvell * All rights reserved. * @@ -11,6 +11,7 @@ #include <odp/helper/odph_api.h> +#include <inttypes.h> #include <stdlib.h> #include "parser.h" #include "lso.h" @@ -3303,7 +3304,7 @@ static void pktio_test_pktin_ts(void) odp_packet_t pkt_tbl[TX_BATCH_LEN]; uint32_t pkt_seq[TX_BATCH_LEN]; uint64_t ns1, ns2; - uint64_t res, res_ns; + uint64_t res, res_ns, input_delay; odp_time_t ts_prev; odp_time_t ts; int num_rx = 0; @@ -3366,6 +3367,18 @@ static void pktio_test_pktin_ts(void) 1, TXRX_MODE_SINGLE, ODP_TIME_SEC_IN_NS, false); if (ret != 1) break; + + /* Compare to packet IO time to input timestamp */ + ts = odp_pktio_time(pktio_rx_info.id, NULL); + CU_ASSERT_FATAL(odp_packet_has_ts(pkt_tbl[i])); + ts_prev = odp_packet_ts(pkt_tbl[i]); + CU_ASSERT(odp_time_cmp(ts, ts_prev) >= 0); + input_delay = odp_time_diff_ns(ts, ts_prev); + if (input_delay > 100 * ODP_TIME_MSEC_IN_NS) { + printf(" Test packet %d input delay: %" PRIu64 "ns\n", i, input_delay); + CU_FAIL("Packet input delay too long"); + } + odp_time_wait_ns(PKTIO_TS_INTERVAL); } num_rx = i; @@ -5097,7 +5110,7 @@ static void pktio_test_pktin_event_queue(odp_pktin_mode_t pktin_mode) int num_buf = 0; int num_bad = 0; odp_pktio_t pktio[MAX_NUM_IFACES] = {0}; - uint64_t wait_sec = odp_schedule_wait_time(ODP_TIME_SEC_IN_NS); + uint64_t wait_time = odp_schedule_wait_time(100 * ODP_TIME_MSEC_IN_NS); CU_ASSERT_FATAL(num_ifaces >= 1); @@ -5170,9 +5183,9 @@ static void pktio_test_pktin_event_queue(odp_pktin_mode_t pktin_mode) /* Receive events */ while (1) { - /* Break after 1 sec of inactivity */ + /* Break after a period of inactivity */ if (pktin_mode == ODP_PKTIN_MODE_SCHED) { - ev = odp_schedule(&from, wait_sec); + ev = odp_schedule(&from, wait_time); if (ev == ODP_EVENT_INVALID) break; @@ -5492,7 +5505,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(pktio_suites); diff --git a/test/validation/api/pool/pool.c b/test/validation/api/pool/pool.c index ee8aa4b67..86a47230a 100644 --- a/test/validation/api/pool/pool.c +++ b/test/validation/api/pool/pool.c @@ -1718,7 +1718,8 @@ static void test_packet_pool_ext_capa(void) odp_pool_ext_capability_t capa; odp_pool_type_t type; const odp_pool_type_t unsupported_types[] = {ODP_POOL_BUFFER, ODP_POOL_TIMEOUT, - ODP_POOL_VECTOR, ODP_POOL_DMA_COMPL}; + ODP_POOL_VECTOR, ODP_POOL_DMA_COMPL, + ODP_POOL_ML_COMPL}; const int num_types = ODPH_ARRAY_SIZE(unsupported_types); /* Verify operation for unsupported pool types */ @@ -2373,7 +2374,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(pool_suites); diff --git a/test/validation/api/queue/queue.c b/test/validation/api/queue/queue.c index cb2b147ac..4b5ccde65 100644 --- a/test/validation/api/queue/queue.c +++ b/test/validation/api/queue/queue.c @@ -1164,7 +1164,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(queue_suites); diff --git a/test/validation/api/random/random.c b/test/validation/api/random/random.c index 97e367678..551fe775d 100644 --- a/test/validation/api/random/random.c +++ b/test/validation/api/random/random.c @@ -313,11 +313,10 @@ static void random_test_independence(odp_random_kind_t kind) for (int i = 0; i < 256; i++) { for (int j = 0; j < 256; j++) { - double expected = - (double)row[i] * (double)col[j] / (double)num; - double diff = - (double)freq[i][j] - expected; - chisq += diff * diff / expected; + double expect = (double)row[i] * (double)col[j] / (double)num; + double diff = (double)freq[i][j] - expect; + + chisq += diff * diff / expect; } } @@ -527,7 +526,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(random_suites); diff --git a/test/validation/api/scheduler/scheduler.c b/test/validation/api/scheduler/scheduler.c index 8f450a057..8dddd8d8f 100644 --- a/test/validation/api/scheduler/scheduler.c +++ b/test/validation/api/scheduler/scheduler.c @@ -1,5 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited - * Copyright (c) 2019-2022, Nokia + * Copyright (c) 2019-2024, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -54,9 +54,9 @@ #define CHAOS_PTR_TO_NDX(p) ((uint64_t)(uint32_t)(uintptr_t)p) #define CHAOS_NDX_TO_PTR(n) ((void *)(uintptr_t)n) -#define WAIT_TIMEOUT (1000 * ODP_TIME_MSEC_IN_NS) +#define WAIT_TIMEOUT (100 * ODP_TIME_MSEC_IN_NS) #define WAIT_ROUNDS 5 -#define WAIT_TOLERANCE (150 * ODP_TIME_MSEC_IN_NS) +#define WAIT_TOLERANCE (15 * ODP_TIME_MSEC_IN_NS) #define WAIT_1MS_RETRIES 1000 #define SCHED_AND_PLAIN_ROUNDS 10000 @@ -1355,7 +1355,7 @@ static void chaos_run(unsigned int qtype) test_globals_t *globals; thread_args_t *args; odp_shm_t shm; - int i, rc, num_thr; + int i, rc; void *arg_ptr; odp_schedule_sync_t sync[] = {ODP_SCHED_SYNC_PARALLEL, ODP_SCHED_SYNC_ATOMIC, @@ -1418,16 +1418,10 @@ static void chaos_run(unsigned int qtype) CU_ASSERT_FATAL(rc == 0); } - /* Test runs also on the main thread */ - num_thr = globals->num_workers - 1; arg_ptr = args; - if (num_thr > 0) - odp_cunit_thread_create(num_thr, chaos_thread, &arg_ptr, 0, 0); + odp_cunit_thread_create(globals->num_workers, chaos_thread, &arg_ptr, 0, 0); - chaos_thread(args); - - if (num_thr > 0) - odp_cunit_thread_join(num_thr); + odp_cunit_thread_join(globals->num_workers); if (CHAOS_DEBUG) printf("Thread %d returning from chaos threads..cleaning up\n", @@ -1475,8 +1469,7 @@ static int schedule_common_(void *arg) odp_pool_t pool; int locked; int num; - odp_event_t ev; - odp_buffer_t buf, buf_cpy; + odp_buffer_t buf; odp_queue_t from; globals = args->globals; @@ -1559,7 +1552,8 @@ static int schedule_common_(void *arg) odp_event_free(events[j]); } } else { - ev = odp_schedule(&from, ODP_SCHED_NO_WAIT); + odp_event_t ev = odp_schedule(&from, ODP_SCHED_NO_WAIT); + if (ev == ODP_EVENT_INVALID) continue; @@ -1570,6 +1564,7 @@ static int schedule_common_(void *arg) uint32_t ndx; uint32_t ndx_max; int rc; + odp_buffer_t buf_cpy; ndx_max = odp_queue_lock_count(from); CU_ASSERT_FATAL(ndx_max > 0); @@ -1830,7 +1825,6 @@ static void parallel_execute(odp_schedule_sync_t sync, int num_queues, test_globals_t *globals; thread_args_t *args; void *arg_ptr; - int num; shm = odp_shm_lookup(GLOBALS_SHM_NAME); CU_ASSERT_FATAL(shm != ODP_SHM_INVALID); @@ -1860,18 +1854,11 @@ static void parallel_execute(odp_schedule_sync_t sync, int num_queues, odp_schedule_print(); /* Create and launch worker threads */ - - /* Test runs also on the main thread */ - num = globals->num_workers - 1; arg_ptr = args; - if (num > 0) - odp_cunit_thread_create(num, schedule_common_, &arg_ptr, 0, 0); - - schedule_common_(args); + odp_cunit_thread_create(globals->num_workers, schedule_common_, &arg_ptr, 0, 0); /* Wait for worker threads to terminate */ - if (num > 0) - odp_cunit_thread_join(num); + odp_cunit_thread_join(globals->num_workers); /* Cleanup ordered queues for next pass */ if (sync == ODP_SCHED_SYNC_ORDERED) @@ -2495,16 +2482,6 @@ out: CU_ASSERT(odp_queue_destroy(queue) == 0); } -static int check_2_workers(void) -{ - if (globals->num_workers < 2) { - printf("\nTest: scheduler_test_order_wait_2_threads: SKIPPED\n"); - return ODP_TEST_INACTIVE; - } - - return ODP_TEST_ACTIVE; -} - static int sched_and_plain_thread(void *arg) { odp_event_t ev1, ev2; @@ -2590,7 +2567,7 @@ static void scheduler_test_sched_and_plain(odp_schedule_sync_t sync) uint64_t wait = odp_schedule_wait_time(100 * ODP_TIME_MSEC_IN_NS); uint32_t events_per_queue = BUFS_PER_QUEUE / 2; uint32_t prev_seq; - int first, num; + int first; void *arg_ptr; CU_ASSERT_FATAL(!odp_schedule_capability(&sched_capa)); @@ -2673,16 +2650,10 @@ static void scheduler_test_sched_and_plain(odp_schedule_sync_t sync) } CU_ASSERT_FATAL(seq > 2); - /* Test runs also on the main thread */ - num = globals->num_workers - 1; arg_ptr = args; - if (num > 0) - odp_cunit_thread_create(num, sched_and_plain_thread, &arg_ptr, 0, 0); - - sched_and_plain_thread(args); + odp_cunit_thread_create(globals->num_workers, sched_and_plain_thread, &arg_ptr, 0, 0); - if (num > 0) - odp_cunit_thread_join(num); + odp_cunit_thread_join(globals->num_workers); /* Check plain queue sequence numbers and free events */ first = 1; @@ -2979,15 +2950,10 @@ static void scheduler_fifo_mt(odp_schedule_sync_t sync, int multi) for (i = 0; i < num_thr; i++) arg[i] = i; - if (num_thr > 1) - odp_cunit_thread_create(num_thr - 1, scheduler_fifo_test, (void **)&arg[1], 1, 0); - - /* Main thread runs as thread 0 */ - scheduler_fifo_test(0); + odp_cunit_thread_create(num_thr, scheduler_fifo_test, (void **)&arg[0], 1, 0); /* Wait for worker threads to terminate */ - if (num_thr > 1) - odp_cunit_thread_join(num_thr - 1); + odp_cunit_thread_join(num_thr); } static void scheduler_fifo_mt_parallel_single(void) @@ -3083,7 +3049,7 @@ static void scheduler_test_atomicity(void) odp_pool_t pool; odp_queue_t queue; odp_queue_param_t queue_param; - int i, num; + int i; void *arg_ptr; shm = odp_shm_lookup(GLOBALS_SHM_NAME); @@ -3121,18 +3087,12 @@ static void scheduler_test_atomicity(void) odp_atomic_init_u32(&globals->atomicity_q.state, 0); /* Create and launch worker threads */ - /* Test runs also on the main thread */ args->num_workers = globals->num_workers; - num = globals->num_workers - 1; arg_ptr = args; - if (num > 0) - odp_cunit_thread_create(num, atomicity_test_run, &arg_ptr, 0, 0); - - atomicity_test_run(args); + odp_cunit_thread_create(globals->num_workers, atomicity_test_run, &arg_ptr, 0, 0); /* Wait for worker threads to terminate */ - if (num > 0) - odp_cunit_thread_join(num); + odp_cunit_thread_join(globals->num_workers); odp_queue_destroy(globals->atomicity_q.handle); } @@ -3720,7 +3680,7 @@ odp_testinfo_t scheduler_basic_suite[] = { ODP_TEST_INFO(scheduler_test_pause_enqueue), ODP_TEST_INFO(scheduler_test_ordered_lock), ODP_TEST_INFO(scheduler_test_order_wait_1_thread), - ODP_TEST_INFO_CONDITIONAL(scheduler_test_order_wait_2_threads, check_2_workers), + ODP_TEST_INFO(scheduler_test_order_wait_2_threads), ODP_TEST_INFO_CONDITIONAL(scheduler_test_flow_aware, check_flow_aware_support), ODP_TEST_INFO(scheduler_test_parallel), @@ -3795,7 +3755,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(global_init); diff --git a/test/validation/api/scheduler/scheduler_no_predef_groups.c b/test/validation/api/scheduler/scheduler_no_predef_groups.c index 1b941ac61..ad6f6d3a2 100644 --- a/test/validation/api/scheduler/scheduler_no_predef_groups.c +++ b/test/validation/api/scheduler/scheduler_no_predef_groups.c @@ -212,7 +212,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(global_init); diff --git a/test/validation/api/shmem/shmem.c b/test/validation/api/shmem/shmem.c index 257e0214e..9e91dab35 100644 --- a/test/validation/api/shmem/shmem.c +++ b/test/validation/api/shmem/shmem.c @@ -1163,7 +1163,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(shmem_suites); diff --git a/test/validation/api/stash/stash.c b/test/validation/api/stash/stash.c index 67c336a2f..162697ba9 100644 --- a/test/validation/api/stash/stash.c +++ b/test/validation/api/stash/stash.c @@ -1385,7 +1385,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(stash_suites); diff --git a/test/validation/api/std/std.c b/test/validation/api/std/std.c index 56d05a4b4..161ee87cf 100644 --- a/test/validation/api/std/std.c +++ b/test/validation/api/std/std.c @@ -97,7 +97,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(std_suites); diff --git a/test/validation/api/system/system.c b/test/validation/api/system/system.c index 89a46a7f2..3f7e0497d 100644 --- a/test/validation/api/system/system.c +++ b/test/validation/api/system/system.c @@ -202,23 +202,27 @@ static void system_test_cpu_cycles_long_period(void) { int i; int periods = PERIODS_100_MSEC; + uint64_t max_period_duration = 100 * ODP_TIME_MSEC_IN_NS + periods - 1; uint64_t c2, c1, c3, max; uint64_t tmp, diff, res; res = odp_cpu_cycles_resolution(); max = odp_cpu_cycles_max(); + c3 = odp_cpu_cycles(); + + CU_ASSERT(c3 <= max); /* - * We can virtually never see a 64 bit cycle counter wrap around, - * so let's not even try. Use small a number of periods to speed - * up testing in this common case. + * If the cycle counter is not close to wrapping around during + * the test, then speed up the test by not trying to see the wrap + * around too hard. Assume cycle counter frequency of less than 10 GHz. */ - if (max == UINT64_MAX) + CU_ASSERT(odp_cpu_hz_max() < 10ULL * ODP_TIME_SEC_IN_NS); + if (max - c3 > 10 * periods * max_period_duration) periods = 10; printf("\n Testing CPU cycles for %i seconds... ", periods / 10); - c3 = odp_cpu_cycles(); for (i = 0; i < periods; i++) { c1 = odp_cpu_cycles(); odp_time_wait_ns(100 * ODP_TIME_MSEC_IN_NS + i); @@ -241,7 +245,7 @@ static void system_test_cpu_cycles_long_period(void) } /* wrap was detected, no need to continue */ - if (i < PERIODS_100_MSEC) { + if (i < periods) { printf("wrap was detected.\n"); return; } @@ -683,7 +687,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; ret = odp_cunit_register(system_suites); diff --git a/test/validation/api/thread/thread.c b/test/validation/api/thread/thread.c index 840256fcf..ad9ffa745 100644 --- a/test/validation/api/thread/thread.c +++ b/test/validation/api/thread/thread.c @@ -255,7 +255,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(thread_global_init); diff --git a/test/validation/api/time/time.c b/test/validation/api/time/time.c index 8d3481e26..cfef7f619 100644 --- a/test/validation/api/time/time.c +++ b/test/validation/api/time/time.c @@ -1,5 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited - * Copyright (c) 2019-2023, Nokia + * Copyright (c) 2019-2024, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -24,6 +24,7 @@ #define TIME_SAMPLES 2 #define TIME_TOLERANCE_NS 1000000 #define TIME_TOLERANCE_CI_NS 40000000 +#define TIME_TOLERANCE_1CPU_NS 40000000 #define GLOBAL_SHM_NAME "GlobalTimeTest" #define YEAR_IN_NS (365 * 24 * ODP_TIME_HOUR_IN_NS) @@ -858,8 +859,7 @@ static void time_test_global_sync(const int ctrl) odph_thread_common_param_t thr_common; odph_thread_param_t thr_param; odph_thread_t thread_tbl[MAX_WORKERS]; - const uint64_t tolerance = - odp_cunit_ci() ? TIME_TOLERANCE_CI_NS : TIME_TOLERANCE_NS; + uint64_t tolerance = odp_cunit_ci() ? TIME_TOLERANCE_CI_NS : TIME_TOLERANCE_NS; const int num = ctrl ? 2 : global_mem->num_threads; if (num < 2) { @@ -879,14 +879,23 @@ static void time_test_global_sync(const int ctrl) if (ctrl) { /* Test sync between one control and one worker thread. */ + int control_cpu; + int worker_cpu; + odp_cpumask_default_control(&cpumask, 1); thr_common.cpumask = &cpumask; thr_param.thr_type = ODP_THREAD_CONTROL; + control_cpu = odp_cpumask_first(&cpumask); int r = odph_thread_create(&thread_tbl[thr++], &thr_common, &thr_param, 1); CU_ASSERT_FATAL(r == 1); odp_cpumask_default_worker(&cpumask, 1); + worker_cpu = odp_cpumask_first(&cpumask); + if (control_cpu == worker_cpu) { + printf(" single CPU, relaxing tolerance. "); + tolerance = TIME_TOLERANCE_1CPU_NS; + } } else { /* Test sync between num worker threads. */ odp_cpumask_default_worker(&cpumask, num); @@ -1007,7 +1016,7 @@ int main(int argc, char *argv[]) int ret; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; odp_cunit_register_global_init(time_global_init); diff --git a/test/validation/api/timer/timer.c b/test/validation/api/timer/timer.c index 28b1399a2..3678d0cb2 100644 --- a/test/validation/api/timer/timer.c +++ b/test/validation/api/timer/timer.c @@ -192,8 +192,8 @@ static int timer_global_init(odp_instance_t *instance) global_mem->periodic_support = capa.periodic.max_pools > 0; - /* By default 20 msec resolution */ - res_ns = 20 * ODP_TIME_MSEC_IN_NS; + /* By default 2 msec resolution */ + res_ns = 2 * ODP_TIME_MSEC_IN_NS; if (res_ns < capa.max_res.res_ns) res_ns = capa.max_res.res_ns; @@ -1131,42 +1131,42 @@ static void timer_single_shot(odp_queue_type_t queue_type, odp_timer_tick_type_t static void timer_plain_rel_wait(void) { - timer_single_shot(ODP_QUEUE_TYPE_PLAIN, RELATIVE, START, TIMEOUT, 2, 500 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_PLAIN, RELATIVE, START, TIMEOUT, 2, 50 * MSEC); } static void timer_plain_abs_wait(void) { - timer_single_shot(ODP_QUEUE_TYPE_PLAIN, ABSOLUTE, START, TIMEOUT, 2, 500 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_PLAIN, ABSOLUTE, START, TIMEOUT, 2, 50 * MSEC); } static void timer_plain_rel_cancel(void) { - timer_single_shot(ODP_QUEUE_TYPE_PLAIN, RELATIVE, START, CANCEL, 5, 1000 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_PLAIN, RELATIVE, START, CANCEL, 5, 100 * MSEC); } static void timer_plain_abs_cancel(void) { - timer_single_shot(ODP_QUEUE_TYPE_PLAIN, ABSOLUTE, START, CANCEL, 5, 1000 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_PLAIN, ABSOLUTE, START, CANCEL, 5, 100 * MSEC); } static void timer_plain_rel_restart_wait(void) { - timer_single_shot(ODP_QUEUE_TYPE_PLAIN, RELATIVE, RESTART, TIMEOUT, 2, 600 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_PLAIN, RELATIVE, RESTART, TIMEOUT, 2, 60 * MSEC); } static void timer_plain_abs_restart_wait(void) { - timer_single_shot(ODP_QUEUE_TYPE_PLAIN, ABSOLUTE, RESTART, TIMEOUT, 2, 600 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_PLAIN, ABSOLUTE, RESTART, TIMEOUT, 2, 60 * MSEC); } static void timer_plain_rel_restart_cancel(void) { - timer_single_shot(ODP_QUEUE_TYPE_PLAIN, RELATIVE, RESTART, CANCEL, 5, 1000 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_PLAIN, RELATIVE, RESTART, CANCEL, 5, 100 * MSEC); } static void timer_plain_abs_restart_cancel(void) { - timer_single_shot(ODP_QUEUE_TYPE_PLAIN, ABSOLUTE, RESTART, CANCEL, 5, 1000 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_PLAIN, ABSOLUTE, RESTART, CANCEL, 5, 100 * MSEC); } static void timer_plain_abs_wait_3sec(void) @@ -1176,42 +1176,42 @@ static void timer_plain_abs_wait_3sec(void) static void timer_sched_rel_wait(void) { - timer_single_shot(ODP_QUEUE_TYPE_SCHED, RELATIVE, START, TIMEOUT, 2, 500 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_SCHED, RELATIVE, START, TIMEOUT, 2, 50 * MSEC); } static void timer_sched_abs_wait(void) { - timer_single_shot(ODP_QUEUE_TYPE_SCHED, ABSOLUTE, START, TIMEOUT, 2, 500 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_SCHED, ABSOLUTE, START, TIMEOUT, 2, 50 * MSEC); } static void timer_sched_rel_cancel(void) { - timer_single_shot(ODP_QUEUE_TYPE_SCHED, RELATIVE, START, CANCEL, 5, 1000 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_SCHED, RELATIVE, START, CANCEL, 5, 100 * MSEC); } static void timer_sched_abs_cancel(void) { - timer_single_shot(ODP_QUEUE_TYPE_SCHED, ABSOLUTE, START, CANCEL, 5, 1000 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_SCHED, ABSOLUTE, START, CANCEL, 5, 100 * MSEC); } static void timer_sched_rel_restart_wait(void) { - timer_single_shot(ODP_QUEUE_TYPE_SCHED, RELATIVE, RESTART, TIMEOUT, 2, 600 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_SCHED, RELATIVE, RESTART, TIMEOUT, 2, 60 * MSEC); } static void timer_sched_abs_restart_wait(void) { - timer_single_shot(ODP_QUEUE_TYPE_SCHED, ABSOLUTE, RESTART, TIMEOUT, 2, 600 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_SCHED, ABSOLUTE, RESTART, TIMEOUT, 2, 60 * MSEC); } static void timer_sched_rel_restart_cancel(void) { - timer_single_shot(ODP_QUEUE_TYPE_SCHED, RELATIVE, RESTART, CANCEL, 5, 1000 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_SCHED, RELATIVE, RESTART, CANCEL, 5, 100 * MSEC); } static void timer_sched_abs_restart_cancel(void) { - timer_single_shot(ODP_QUEUE_TYPE_SCHED, ABSOLUTE, RESTART, CANCEL, 5, 1000 * MSEC); + timer_single_shot(ODP_QUEUE_TYPE_SCHED, ABSOLUTE, RESTART, CANCEL, 5, 100 * MSEC); } static void timer_sched_abs_wait_3sec(void) @@ -2403,7 +2403,6 @@ static int worker_entrypoint(void *arg ODP_UNUSED) } } else { uint64_t cur_tick; - uint64_t tck; int reset_timer = 0; if (tt[i].ev != ODP_EVENT_INVALID) { @@ -2862,9 +2861,9 @@ static void timer_test_periodic(odp_queue_type_t queue_type, int use_first, int int num_tmo; int done; const int num = 200; - /* Test frequency: 1x 100Hz, or 1x min/max_base_freq */ + /* Test frequency: 1x 1000 Hz, or 1x min/max_base_freq */ const uint64_t multiplier = 1; - odp_fract_u64_t base_freq = {100, 0, 0}; + odp_fract_u64_t base_freq = {1000, 0, 0}; odp_timer_clk_src_t clk_src = test_global->clk_src; memset(&timer_capa, 0, sizeof(odp_timer_capability_t)); @@ -3262,7 +3261,7 @@ int main(int argc, char *argv[]) int ret = 0; /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; if (global_init()) diff --git a/test/validation/api/traffic_mngr/traffic_mngr.c b/test/validation/api/traffic_mngr/traffic_mngr.c index ed07db751..b7f546dcd 100644 --- a/test/validation/api/traffic_mngr/traffic_mngr.c +++ b/test/validation/api/traffic_mngr/traffic_mngr.c @@ -5025,7 +5025,7 @@ odp_suiteinfo_t traffic_mngr_suites[] = { int main(int argc, char *argv[]) { /* parse common options: */ - if (odp_cunit_parse_options(argc, argv)) + if (odp_cunit_parse_options(&argc, argv)) return -1; int ret = odp_cunit_register(traffic_mngr_suites); |