aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMaxim Uvarov <maxim.uvarov@linaro.org>2018-09-19 21:10:56 +0300
committerMaxim Uvarov <maxim.uvarov@linaro.org>2018-09-19 21:10:56 +0300
commite9ad0a254722115450199fe4a72942be9b5cf4e1 (patch)
treedf3287580c436b67de3738ae87ad879142201bae
parentd0bd42aa817eb79dffeab77cbe3ea2ea6f5e3db4 (diff)
parent6bbd53f2f407b81255d6cef969da8037cff2fa3e (diff)
downloadodp-dpdk-master.tar.gz
merge https://github.com/Linaro/odp-dpdk/pull/58HEADmaster
Signed-off-by: Maxim Uvarov <maxim.uvarov@linaro.org>
-rw-r--r--example/traffic_mgmt/odp_traffic_mgmt.c64
-rw-r--r--helper/Makefile.am1
-rw-r--r--helper/chksum.c75
-rw-r--r--helper/include/odp/helper/chksum.h42
-rw-r--r--helper/include/odp/helper/ip.h2
-rw-r--r--helper/include/odp/helper/odph_api.h1
-rw-r--r--helper/include/odp/helper/sctp.h49
-rw-r--r--include/Makefile.am1
-rw-r--r--include/odp/api/abi-default/classification.h4
-rw-r--r--include/odp/api/abi-default/ipsec.h2
-rw-r--r--include/odp/api/abi-default/packet.h2
-rw-r--r--include/odp/api/abi-default/pool.h2
-rw-r--r--include/odp/api/abi-default/timer.h6
-rw-r--r--include/odp/api/spec/queue.h277
-rw-r--r--include/odp/api/spec/queue_types.h311
-rw-r--r--m4/odp_dpdk.m44
-rw-r--r--platform/linux-dpdk/Makefile.am2
-rw-r--r--platform/linux-dpdk/include-abi/odp/api/abi/packet.h2
-rw-r--r--platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h7
-rw-r--r--platform/linux-dpdk/include/odp_buffer_internal.h5
-rw-r--r--platform/linux-dpdk/include/odp_packet_internal.h3
-rw-r--r--platform/linux-dpdk/include/odp_packet_io_internal.h1
-rw-r--r--platform/linux-dpdk/include/odp_pool_internal.h2
-rw-r--r--platform/linux-dpdk/include/odp_ring_mpmc_internal.h89
-rw-r--r--platform/linux-dpdk/odp_crypto.c1
-rw-r--r--platform/linux-dpdk/odp_packet.c105
-rw-r--r--platform/linux-dpdk/odp_packet_dpdk.c2
-rw-r--r--platform/linux-dpdk/odp_pool.c2
-rw-r--r--platform/linux-dpdk/odp_queue_basic.c487
-rw-r--r--platform/linux-dpdk/odp_queue_if.c52
-rw-r--r--platform/linux-dpdk/odp_queue_spsc.c29
-rw-r--r--platform/linux-dpdk/odp_timer.c2
-rw-r--r--platform/linux-generic/Makefile.am2
-rw-r--r--platform/linux-generic/include-abi/odp/api/abi/classification.h4
-rw-r--r--platform/linux-generic/include-abi/odp/api/abi/ipsec.h2
-rw-r--r--platform/linux-generic/include-abi/odp/api/abi/pool.h2
-rw-r--r--platform/linux-generic/include-abi/odp/api/abi/timer.h6
-rw-r--r--platform/linux-generic/include/odp/api/plat/packet_inline_types.h7
-rw-r--r--platform/linux-generic/include/odp/api/plat/packet_inlines.h4
-rw-r--r--platform/linux-generic/include/odp/api/plat/queue_inline_types.h26
-rw-r--r--platform/linux-generic/include/odp/api/plat/queue_inlines.h29
-rw-r--r--platform/linux-generic/include/odp_buffer_internal.h5
-rw-r--r--platform/linux-generic/include/odp_packet_internal.h4
-rw-r--r--platform/linux-generic/include/odp_packet_io_internal.h1
-rw-r--r--platform/linux-generic/include/odp_pool_internal.h2
-rw-r--r--platform/linux-generic/include/odp_queue_basic_internal.h34
-rw-r--r--platform/linux-generic/include/odp_queue_if.h57
-rw-r--r--platform/linux-generic/include/odp_queue_scalable_internal.h9
-rw-r--r--platform/linux-generic/include/odp_ring_mpmc_internal.h169
-rw-r--r--platform/linux-generic/include/odp_ring_spsc_internal.h33
-rw-r--r--platform/linux-generic/include/odp_ring_st_internal.h34
-rw-r--r--platform/linux-generic/include/odp_schedule_if.h2
-rw-r--r--platform/linux-generic/include/odp_traffic_mngr_internal.h2
-rw-r--r--platform/linux-generic/include/protocols/sctp.h49
-rw-r--r--platform/linux-generic/odp_classification.c65
-rw-r--r--platform/linux-generic/odp_crypto_null.c1
-rw-r--r--platform/linux-generic/odp_crypto_openssl.c1
-rw-r--r--platform/linux-generic/odp_ipsec.c15
-rw-r--r--platform/linux-generic/odp_ipsec_events.c1
-rw-r--r--platform/linux-generic/odp_ipsec_sad.c4
-rw-r--r--platform/linux-generic/odp_packet.c103
-rw-r--r--platform/linux-generic/odp_packet_io.c108
-rw-r--r--platform/linux-generic/odp_pool.c2
-rw-r--r--platform/linux-generic/odp_queue_basic.c486
-rw-r--r--platform/linux-generic/odp_queue_if.c56
-rw-r--r--platform/linux-generic/odp_queue_lf.c16
-rw-r--r--platform/linux-generic/odp_queue_scalable.c58
-rw-r--r--platform/linux-generic/odp_queue_spsc.c40
-rw-r--r--platform/linux-generic/odp_schedule_basic.c20
-rw-r--r--platform/linux-generic/odp_schedule_iquery.c18
-rw-r--r--platform/linux-generic/odp_schedule_scalable.c2
-rw-r--r--platform/linux-generic/odp_schedule_sp.c4
-rw-r--r--platform/linux-generic/odp_timer.c9
-rw-r--r--platform/linux-generic/odp_traffic_mngr.c42
-rw-r--r--platform/linux-generic/pktio/dpdk.c4
-rw-r--r--platform/linux-generic/pktio/loop.c28
-rw-r--r--platform/linux-generic/pktio/netmap.c4
-rwxr-xr-xscripts/ci/build.sh22
-rwxr-xr-xscripts/ci/build_arm64.sh14
-rwxr-xr-xscripts/ci/build_armhf.sh16
-rwxr-xr-xscripts/ci/build_i386.sh15
-rwxr-xr-xscripts/ci/build_powerpc.sh15
-rwxr-xr-xscripts/ci/build_x86_64.sh8
-rwxr-xr-xscripts/ci/check.sh17
-rwxr-xr-xscripts/ci/coverage.sh31
-rwxr-xr-xscripts/ci/distcheck.sh21
-rw-r--r--test/common/test_packet_parser.h26
-rw-r--r--test/performance/odp_pool_perf.c2
-rw-r--r--test/performance/odp_queue_perf.c380
-rw-r--r--test/performance/odp_sched_perf.c67
-rw-r--r--test/validation/api/pktio/parser.c47
-rw-r--r--test/validation/api/pktio/pktio.c428
92 files changed, 3051 insertions, 1262 deletions
diff --git a/example/traffic_mgmt/odp_traffic_mgmt.c b/example/traffic_mgmt/odp_traffic_mgmt.c
index 2af106ab..dbc65958 100644
--- a/example/traffic_mgmt/odp_traffic_mgmt.c
+++ b/example/traffic_mgmt/odp_traffic_mgmt.c
@@ -230,7 +230,7 @@ static odp_tm_t odp_tm_test;
static odp_pool_t odp_pool;
-static odp_tm_queue_t queue_num_tbls[NUM_SVC_CLASSES][TM_QUEUES_PER_CLASS + 1];
+static odp_tm_queue_t queue_num_tbls[NUM_SVC_CLASSES][TM_QUEUES_PER_CLASS];
static uint32_t next_queue_nums[NUM_SVC_CLASSES];
static uint8_t random_buf[RANDOM_BUF_LEN];
@@ -434,7 +434,7 @@ static int config_example_user(odp_tm_node_t cos_tm_node,
return rc;
svc_class_queue_num = next_queue_nums[svc_class]++;
- queue_num_tbls[svc_class][svc_class_queue_num + 1] =
+ queue_num_tbls[svc_class][svc_class_queue_num] =
tm_queue;
}
}
@@ -633,7 +633,7 @@ static int traffic_generator(uint32_t pkts_to_send)
while (pkt_cnt < pkts_to_send) {
svc_class = pkt_service_class();
queue_num = random_16() & (TM_QUEUES_PER_CLASS - 1);
- tm_queue = queue_num_tbls[svc_class][queue_num + 1];
+ tm_queue = queue_num_tbls[svc_class][queue_num];
pkt_len = ((uint32_t)((random_8() & 0x7F) + 2)) * 32;
pkt_len = MIN(pkt_len, 1500);
pkt = make_odp_packet(pkt_len);
@@ -743,6 +743,47 @@ static void signal_handler(int signal)
abort();
}
+static int destroy_tm_queues(void)
+{
+ int i;
+ int class;
+ int ret;
+
+ for (i = 0; i < NUM_SVC_CLASSES; i++)
+ for (class = 0; class < TM_QUEUES_PER_CLASS; class++) {
+ odp_tm_queue_t tm_queue;
+ odp_tm_queue_info_t info;
+
+ tm_queue = queue_num_tbls[i][class];
+
+ ret = odp_tm_queue_info(tm_queue, &info);
+ if (ret) {
+ printf("Err: odp_tm_queue_info %d\n", ret);
+ return -1;
+ }
+
+ ret = odp_tm_node_disconnect(info.next_tm_node);
+ if (ret) {
+ printf("Err: odp_tm_node_disconnect %d\n", ret);
+ return -1;
+ }
+
+ ret = odp_tm_queue_disconnect(tm_queue);
+ if (ret) {
+ printf("odp_tm_queue_disconnect %d\n", ret);
+ return -1;
+ }
+
+ ret = odp_tm_queue_destroy(tm_queue);
+ if (ret) {
+ printf("odp_tm_queue_destroy %d\n", ret);
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
int main(int argc, char *argv[])
{
struct sigaction signal_action;
@@ -796,6 +837,12 @@ int main(int argc, char *argv[])
odp_tm_stats_print(odp_tm_test);
+ rc = destroy_tm_queues();
+ if (rc != 0) {
+ printf("Error: destroy_tm_queues() failed, rc = %d\n", rc);
+ return -1;
+ }
+
rc = odp_pool_destroy(odp_pool);
if (rc != 0) {
printf("Error: odp_pool_destroy() failed, rc = %d\n", rc);
@@ -814,11 +861,12 @@ int main(int argc, char *argv[])
return -1;
}
- /* Trying to keep this example as simple as possible we avoid
- * clean termination of TM queues. This will error on global
- * termination code
- */
- (void)odp_term_global(instance);
+ rc = odp_term_global(instance);
+ if (rc != 0) {
+ printf("Error: odp_term_global() failed, rc = %d\n", rc);
+ return -1;
+ }
+ printf("Quit\n");
return 0;
}
diff --git a/helper/Makefile.am b/helper/Makefile.am
index 15948cc4..29c56739 100644
--- a/helper/Makefile.am
+++ b/helper/Makefile.am
@@ -24,6 +24,7 @@ helperinclude_HEADERS = \
include/odp/helper/odph_hashtable.h\
include/odp/helper/odph_iplookuptable.h\
include/odp/helper/odph_lineartable.h\
+ include/odp/helper/sctp.h \
include/odp/helper/strong_types.h\
include/odp/helper/tcp.h\
include/odp/helper/table.h\
diff --git a/helper/chksum.c b/helper/chksum.c
index da5625a7..7c7c20a2 100644
--- a/helper/chksum.c
+++ b/helper/chksum.c
@@ -9,6 +9,7 @@
#include <odp.h>
#include <odp/helper/ip.h>
#include <odp/helper/udp.h>
+#include <odp/helper/sctp.h>
#include <odp/helper/tcp.h>
#include <odp/helper/chksum.h>
#include <stddef.h>
@@ -351,3 +352,77 @@ int odph_udp_tcp_chksum(odp_packet_t odp_pkt,
return ret_code;
}
+
+static uint32_t odph_packet_crc32c(odp_packet_t pkt,
+ uint32_t offset,
+ uint32_t length,
+ uint32_t init_val)
+{
+ uint32_t sum = init_val;
+
+ if (offset + length > odp_packet_len(pkt))
+ return sum;
+
+ while (length > 0) {
+ uint32_t seg_len;
+ void *data = odp_packet_offset(pkt, offset, &seg_len, NULL);
+
+ if (seg_len > length)
+ seg_len = length;
+
+ sum = odp_hash_crc32c(data, seg_len, sum);
+ length -= seg_len;
+ offset += seg_len;
+ }
+
+ return sum;
+}
+
+int odph_sctp_chksum_set(odp_packet_t pkt)
+{
+ uint32_t l4_offset = odp_packet_l4_offset(pkt);
+ uint32_t sum = 0;
+
+ if (!odp_packet_has_sctp(pkt))
+ return -1;
+
+ if (l4_offset == ODP_PACKET_OFFSET_INVALID)
+ return -1;
+
+ odp_packet_copy_from_mem(pkt,
+ l4_offset + ODPH_SCTPHDR_LEN - 4,
+ 4,
+ &sum);
+
+ sum = ~odph_packet_crc32c(pkt, l4_offset,
+ odp_packet_len(pkt) - l4_offset,
+ ~0);
+ return odp_packet_copy_from_mem(pkt,
+ l4_offset + ODPH_SCTPHDR_LEN - 4,
+ 4,
+ &sum);
+}
+
+int odph_sctp_chksum_verify(odp_packet_t pkt)
+{
+ uint32_t l4_offset = odp_packet_l4_offset(pkt);
+ uint32_t sum;
+ uint32_t temp = 0;
+
+ if (!odp_packet_has_sctp(pkt))
+ return -1;
+
+ sum = odph_packet_crc32c(pkt, l4_offset,
+ ODPH_SCTPHDR_LEN - 4,
+ ~0);
+ sum = odp_hash_crc32c(&temp, 4, sum);
+ sum = ~odph_packet_crc32c(pkt, l4_offset + ODPH_SCTPHDR_LEN,
+ odp_packet_len(pkt) - l4_offset -
+ ODPH_SCTPHDR_LEN,
+ sum);
+
+ odp_packet_copy_to_mem(pkt, l4_offset + ODPH_SCTPHDR_LEN - 4,
+ 4, &temp);
+
+ return (temp == sum) ? 0 : 2;
+}
diff --git a/helper/include/odp/helper/chksum.h b/helper/include/odp/helper/chksum.h
index c03abeb7..148b9860 100644
--- a/helper/include/odp/helper/chksum.h
+++ b/helper/include/odp/helper/chksum.h
@@ -190,6 +190,48 @@ static inline int odph_udp_chksum_verify(odp_packet_t odp_pkt)
}
/**
+ * Generate SCTP checksum
+ *
+ * This function supports SCTP over either IPv4 or IPV6 - including handling
+ * any IPv4 header options and any IPv6 extension headers. However it
+ * does not handle tunneled pkts (i.e. any case where there is more than
+ * one IPv4/IPv6 header).
+ * This function also handles non-contiguous pkts. In particular it can
+ * handle arbitrary packet segmentation, including cases where the segments
+ * are not 2 byte aligned, nor have a length that is a multiple of 2. This
+ * function also can handle jumbo frames (at least up to 10K).
+ *
+ * This function will insert the calculated CRC32-c checksum into the proper
+ * location in the SCTP header.
+ *
+ * @param odp_pkt Calculate and insert chksum for this SCTP pkt, which can
+ * be over IPv4 or IPv6.
+ * @retval 0 on success
+ * @retval <0 on failure
+ */
+int odph_sctp_chksum_set(odp_packet_t odp_pkt);
+
+/**
+ * Verify SCTP checksum
+ *
+ * This function supports SCTP over either IPv4 or IPV6 - including handling
+ * any IPv4 header options and any IPv6 extension headers. However it
+ * does not handle tunneled pkts (i.e. any case where there is more than
+ * one IPv4/IPv6 header).
+ * This function also handles non-contiguous pkts. In particular it can
+ * handle arbitrary packet segmentation, including cases where the segments
+ * are not 2 byte aligned, nor have a length that is a multiple of 2. This
+ * function also can handle jumbo frames (at least up to 10K).
+ *
+ * @param odp_pkt Calculate and compare the chksum for this SCTP pkt,
+ * which can be over IPv4 or IPv6.
+ * @retval <0 on failure
+ * @retval 0 if the incoming chksum field is correct
+ * @retval 2 when the chksum field is incorrect
+ */
+int odph_sctp_chksum_verify(odp_packet_t odp_pkt);
+
+/**
* @}
*/
diff --git a/helper/include/odp/helper/ip.h b/helper/include/odp/helper/ip.h
index cdc43062..b5bfed78 100644
--- a/helper/include/odp/helper/ip.h
+++ b/helper/include/odp/helper/ip.h
@@ -259,6 +259,8 @@ typedef struct ODP_PACKED {
#define ODPH_IPPROTO_AH 0x33 /**< Authentication Header (51) */
#define ODPH_IPPROTO_ESP 0x32 /**< Encapsulating Security Payload (50) */
#define ODPH_IPPROTO_ICMPV6 0x3A /**< Internet Control Message Protocol (58) */
+#define ODPH_IPPROTO_SCTP 0x84 /**< Stream Control Transmission protocol
+ (132) */
#define ODPH_IPPROTO_INVALID 0xFF /**< Reserved invalid by IANA */
/**@}*/
diff --git a/helper/include/odp/helper/odph_api.h b/helper/include/odp/helper/odph_api.h
index 8ad8a27f..d46ab2ad 100644
--- a/helper/include/odp/helper/odph_api.h
+++ b/helper/include/odp/helper/odph_api.h
@@ -27,6 +27,7 @@ extern "C" {
#include <odp/helper/ipsec.h>
#include <odp/helper/odph_lineartable.h>
#include <odp/helper/odph_iplookuptable.h>
+#include <odp/helper/sctp.h>
#include <odp/helper/strong_types.h>
#include <odp/helper/tcp.h>
#include <odp/helper/table.h>
diff --git a/helper/include/odp/helper/sctp.h b/helper/include/odp/helper/sctp.h
new file mode 100644
index 00000000..c81bd631
--- /dev/null
+++ b/helper/include/odp/helper/sctp.h
@@ -0,0 +1,49 @@
+/* Copyright (c) 2014-2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODP SCTP header
+ */
+
+#ifndef ODPH_SCTP_H_
+#define ODPH_SCTP_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp_api.h>
+
+/** @addtogroup odph_header ODPH HEADER
+ * @{
+ */
+
+/** SCTP header length */
+#define ODPH_SCTPHDR_LEN 12
+
+/** SCTP header */
+typedef struct ODP_PACKED {
+ odp_u16be_t src_port; /**< Source port */
+ odp_u16be_t dst_port; /**< Destination port */
+ odp_u32be_t tag; /**< Verification tag */
+ odp_u32be_t chksum; /**< SCTP header and data checksum */
+} odph_sctphdr_t;
+
+/** @internal Compile time assert */
+ODP_STATIC_ASSERT(sizeof(odph_sctphdr_t) == ODPH_SCTPHDR_LEN,
+ "ODPH_SCTPHDR_T__SIZE_ERROR");
+
+/**
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/include/Makefile.am b/include/Makefile.am
index 512002f8..97f52840 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -82,6 +82,7 @@ odpapispecinclude_HEADERS = \
odp/api/spec/packet_io_stats.h \
odp/api/spec/pool.h \
odp/api/spec/queue.h \
+ odp/api/spec/queue_types.h \
odp/api/spec/random.h \
odp/api/spec/rwlock.h \
odp/api/spec/rwlock_recursive.h \
diff --git a/include/odp/api/abi-default/classification.h b/include/odp/api/abi-default/classification.h
index 67c26517..ef102731 100644
--- a/include/odp/api/abi-default/classification.h
+++ b/include/odp/api/abi-default/classification.h
@@ -24,8 +24,8 @@ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_pmr_t;
typedef _odp_abi_cos_t *odp_cos_t;
typedef _odp_abi_pmr_t *odp_pmr_t;
-#define ODP_COS_INVALID ((odp_cos_t)~0)
-#define ODP_PMR_INVAL ((odp_pmr_t)~0)
+#define ODP_COS_INVALID ((odp_cos_t)0)
+#define ODP_PMR_INVAL ((odp_pmr_t)0)
#define ODP_COS_NAME_LEN 32
diff --git a/include/odp/api/abi-default/ipsec.h b/include/odp/api/abi-default/ipsec.h
index 7ec43372..2c95fd4f 100644
--- a/include/odp/api/abi-default/ipsec.h
+++ b/include/odp/api/abi-default/ipsec.h
@@ -28,7 +28,7 @@ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_ipsec_sa_t;
typedef _odp_abi_ipsec_sa_t *odp_ipsec_sa_t;
-#define ODP_IPSEC_SA_INVALID ((odp_ipsec_sa_t)0xffffffff)
+#define ODP_IPSEC_SA_INVALID ((odp_ipsec_sa_t)0)
/**
* @}
diff --git a/include/odp/api/abi-default/packet.h b/include/odp/api/abi-default/packet.h
index 97c0d088..b5380b04 100644
--- a/include/odp/api/abi-default/packet.h
+++ b/include/odp/api/abi-default/packet.h
@@ -27,7 +27,7 @@ typedef _odp_abi_packet_t *odp_packet_t;
typedef _odp_abi_packet_seg_t *odp_packet_seg_t;
#define ODP_PACKET_INVALID ((odp_packet_t)0)
-#define ODP_PACKET_SEG_INVALID ((odp_packet_seg_t)0xffffffff)
+#define ODP_PACKET_SEG_INVALID ((odp_packet_seg_t)0)
#define ODP_PACKET_OFFSET_INVALID 0xffff
typedef uint8_t odp_proto_l2_type_t;
diff --git a/include/odp/api/abi-default/pool.h b/include/odp/api/abi-default/pool.h
index f3489ff9..4c67e309 100644
--- a/include/odp/api/abi-default/pool.h
+++ b/include/odp/api/abi-default/pool.h
@@ -22,7 +22,7 @@ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_pool_t;
typedef _odp_abi_pool_t *odp_pool_t;
-#define ODP_POOL_INVALID ((odp_pool_t)0xffffffff)
+#define ODP_POOL_INVALID ((odp_pool_t)0)
#define ODP_POOL_NAME_LEN 32
diff --git a/include/odp/api/abi-default/timer.h b/include/odp/api/abi-default/timer.h
index 21a8961d..a7da8ff7 100644
--- a/include/odp/api/abi-default/timer.h
+++ b/include/odp/api/abi-default/timer.h
@@ -31,17 +31,17 @@ struct timer_pool_s; /**< Forward declaration */
typedef struct timer_pool_s *odp_timer_pool_t;
-#define ODP_TIMER_POOL_INVALID NULL
+#define ODP_TIMER_POOL_INVALID ((odp_timer_pool_t)0)
#define ODP_TIMER_POOL_NAME_LEN 32
typedef _odp_abi_timer_t *odp_timer_t;
-#define ODP_TIMER_INVALID ((odp_timer_t)0xffffffff)
+#define ODP_TIMER_INVALID ((odp_timer_t)0)
typedef _odp_abi_timeout_t *odp_timeout_t;
-#define ODP_TIMEOUT_INVALID ((odp_timeout_t)NULL)
+#define ODP_TIMEOUT_INVALID ((odp_timeout_t)0)
/**
* @}
diff --git a/include/odp/api/spec/queue.h b/include/odp/api/spec/queue.h
index 3015d779..2f5e1230 100644
--- a/include/odp/api/spec/queue.h
+++ b/include/odp/api/spec/queue.h
@@ -19,8 +19,8 @@
extern "C" {
#endif
-#include <odp/api/schedule_types.h>
#include <odp/api/event.h>
+#include <odp/api/spec/queue_types.h>
/** @defgroup odp_queue ODP QUEUE
* Macros and operation on a queue.
@@ -43,272 +43,6 @@ extern "C" {
*/
/**
- * Queue type
- */
-typedef enum odp_queue_type_t {
- /** Plain queue
- *
- * Plain queues offer simple FIFO storage of events. Application may
- * dequeue directly from these queues. */
- ODP_QUEUE_TYPE_PLAIN = 0,
-
- /** Scheduled queue
- *
- * Scheduled queues are connected to the scheduler. Application must
- * not dequeue events directly from these queues but use the scheduler
- * instead. */
- ODP_QUEUE_TYPE_SCHED
-} odp_queue_type_t;
-
-/**
- * Queue operation mode
- */
-typedef enum odp_queue_op_mode_t {
- /** Multithread safe operation
- *
- * Queue operation (enqueue or dequeue) is multithread safe. Any
- * number of application threads may perform the operation
- * concurrently. */
- ODP_QUEUE_OP_MT = 0,
-
- /** Not multithread safe operation
- *
- * Queue operation (enqueue or dequeue) may not be multithread safe.
- * Application ensures synchronization between threads so that
- * simultaneously only single thread attempts the operation on
- * the same queue. */
- ODP_QUEUE_OP_MT_UNSAFE,
-
- /** Disabled
- *
- * Direct enqueue or dequeue operation from application is disabled.
- * An attempt to enqueue/dequeue directly will result undefined
- * behaviour. Various ODP functions (e.g. packet input, timer,
- * crypto, scheduler, etc) are able to perform enqueue or
- * dequeue operations normally on the queue.
- * */
- ODP_QUEUE_OP_DISABLED
-
-} odp_queue_op_mode_t;
-
-/**
- * Non-blocking level
- *
- * A non-blocking level defines implementation guarantees for application
- * progress when multiple threads operate on the same resource (e.g. a queue)
- * simultaneously. The first level (ODP_BLOCKING) does not have any block
- * freedom guarantees, but a suspending thread may block the other threads for
- * the entire time it remains suspended (infinitely if crashed).
- * On the contrary, actual non-blocking levels provide guarantees of progress:
- *
- * ODP_NONBLOCKING_LF: A non-blocking and lock-free implementation guarantees
- * that at least one of the threads successfully completes
- * its operations, regardless of what other threads do.
- * Application progress is guaranteed, but individual
- * threads may starve while trying to execute their
- * operations on the shared resource.
- *
- * ODP_NONBLOCKING_WF: A non-blocking and wait-free implementation guarantees
- * application progress with starvation freedom. All
- * threads are guaranteed to complete their operations in
- * a bounded number of steps, regardless of what other
- * threads do.
- *
- * Non-blocking levels are listed from the weakest to the strongest guarantee of
- * block freedom. Performance of a non-blocking implementation may be lower than
- * the blocking one. Non-blocking guarantees are important e.g. for real-time
- * applications when real-time and non real-time threads share a resource.
- */
-typedef enum odp_nonblocking_t {
- /** Blocking implementation. A suspeding thread may block all other
- * threads, i.e. no block freedom guarantees. This is the lowest level.
- */
- ODP_BLOCKING = 0,
-
- /** Non-blocking and lock-free implementation. Other threads can make
- * progress while a thread is suspended. Starvation freedom is not
- * guaranteed.
- */
- ODP_NONBLOCKING_LF,
-
- /** Non-blocking and wait-free implementation. Other threads can make
- * progress while a thread is suspended. Starvation freedom is
- * guaranteed.
- */
- ODP_NONBLOCKING_WF
-
-} odp_nonblocking_t;
-
-/**
- * Queue capabilities
- */
-typedef struct odp_queue_capability_t {
- /** Maximum number of event queues of any type (default size). Use
- * this in addition to queue type specific 'max_num', if both queue
- * types are used simultaneously. */
- uint32_t max_queues;
-
- /** Maximum number of ordered locks per queue */
- uint32_t max_ordered_locks;
-
- /** Maximum number of scheduling groups */
- unsigned max_sched_groups;
-
- /** Number of scheduling priorities */
- unsigned sched_prios;
-
- /** Plain queue capabilities */
- struct {
- /** Maximum number of plain (ODP_BLOCKING) queues of the
- * default size. */
- uint32_t max_num;
-
- /** Maximum number of events a plain (ODP_BLOCKING) queue can
- * store simultaneously. The value of zero means that plain
- * queues do not have a size limit, but a single queue can
- * store all available events. */
- uint32_t max_size;
-
- /** Lock-free (ODP_NONBLOCKING_LF) implementation capabilities.
- * The specification is the same as for the blocking
- * implementation. */
- struct {
- /** Maximum number of queues. Lock-free queues are not
- * supported when zero. */
- uint32_t max_num;
-
- /** Maximum queue size */
- uint32_t max_size;
-
- } lockfree;
-
- /** Wait-free (ODP_NONBLOCKING_WF) implementation capabilities.
- * The specification is the same as for the blocking
- * implementation. */
- struct {
- /** Maximum number of queues. Wait-free queues are not
- * supported when zero. */
- uint32_t max_num;
-
- /** Maximum queue size */
- uint32_t max_size;
-
- } waitfree;
-
- } plain;
-
- /** Scheduled queue capabilities */
- struct {
- /** Maximum number of scheduled (ODP_BLOCKING) queues of the
- * default size. */
- uint32_t max_num;
-
- /** Maximum number of events a scheduled (ODP_BLOCKING) queue
- * can store simultaneously. The value of zero means that
- * scheduled queues do not have a size limit, but a single
- * queue can store all available events. */
- uint32_t max_size;
-
- /** Lock-free (ODP_NONBLOCKING_LF) implementation capabilities.
- * The specification is the same as for the blocking
- * implementation. */
- struct {
- /** Maximum number of queues. Lock-free queues are not
- * supported when zero. */
- uint32_t max_num;
-
- /** Maximum queue size */
- uint32_t max_size;
-
- } lockfree;
-
- /** Wait-free (ODP_NONBLOCKING_WF) implementation capabilities.
- * The specification is the same as for the blocking
- * implementation. */
- struct {
- /** Maximum number of queues. Wait-free queues are not
- * supported when zero. */
- uint32_t max_num;
-
- /** Maximum queue size */
- uint32_t max_size;
-
- } waitfree;
-
- } sched;
-
-} odp_queue_capability_t;
-
-/**
- * ODP Queue parameters
- */
-typedef struct odp_queue_param_t {
- /** Queue type
- *
- * Valid values for other parameters in this structure depend on
- * the queue type. */
- odp_queue_type_t type;
-
- /** Enqueue mode
- *
- * Default value for both queue types is ODP_QUEUE_OP_MT. Application
- * may enable performance optimizations by defining MT_UNSAFE or
- * DISABLED modes when applicable. */
- odp_queue_op_mode_t enq_mode;
-
- /** Dequeue mode
- *
- * For PLAIN queues, the default value is ODP_QUEUE_OP_MT. Application
- * may enable performance optimizations by defining MT_UNSAFE or
- * DISABLED modes when applicable. However, when a plain queue is input
- * to the implementation (e.g. a queue for packet output), the
- * parameter is ignored in queue creation and the value is
- * ODP_QUEUE_OP_DISABLED.
- *
- * For SCHED queues, the parameter is ignored in queue creation and
- * the value is ODP_QUEUE_OP_DISABLED. */
- odp_queue_op_mode_t deq_mode;
-
- /** Scheduler parameters
- *
- * These parameters are considered only when queue type is
- * ODP_QUEUE_TYPE_SCHED. */
- odp_schedule_param_t sched;
-
- /** Non-blocking level
- *
- * Queue implementation must guarantee at least this level of block
- * freedom for queue enqueue and dequeue/schedule operations.
- * The default value is ODP_BLOCKING. */
- odp_nonblocking_t nonblocking;
-
- /** Queue context pointer
- *
- * User defined context pointer associated with the queue. The same
- * pointer can be accessed with odp_queue_context() and
- * odp_queue_context_set() calls. The implementation may read the
- * pointer for prefetching the context data. Default value of the
- * pointer is NULL. */
- void *context;
-
- /** Queue context data length
- *
- * User defined context data length in bytes for prefetching.
- * The implementation may use this value as a hint for the number of
- * context data bytes to prefetch. Default value is zero (no hint). */
- uint32_t context_len;
-
- /** Queue size
- *
- * The queue must be able to store at minimum this many events
- * simultaneously. The value must not exceed 'max_size' queue
- * capability. The value of zero means implementation specific
- * default size. */
- uint32_t size;
-
-} odp_queue_param_t;
-
-/**
* Queue create
*
* Create a queue according to the queue parameters. Queue type is specified by
@@ -528,15 +262,6 @@ uint64_t odp_queue_to_u64(odp_queue_t hdl);
void odp_queue_param_init(odp_queue_param_t *param);
/**
- * Queue information
- * Retrieve information about a queue with odp_queue_info()
- */
-typedef struct odp_queue_info_t {
- const char *name; /**< queue name */
- odp_queue_param_t param; /**< queue parameters */
-} odp_queue_info_t;
-
-/**
* Retrieve information about a queue
*
* Invalid queue handles or handles to free/destroyed queues leads to
diff --git a/include/odp/api/spec/queue_types.h b/include/odp/api/spec/queue_types.h
new file mode 100644
index 00000000..be7e79a8
--- /dev/null
+++ b/include/odp/api/spec/queue_types.h
@@ -0,0 +1,311 @@
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODP queue types
+ */
+
+#ifndef ODP_API_SPEC_QUEUE_TYPES_H_
+#define ODP_API_SPEC_QUEUE_TYPES_H_
+#include <odp/visibility_begin.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp/api/schedule_types.h>
+
+/** @addtogroup odp_queue
+ * @{
+ */
+
+/**
+ * Queue type
+ */
+typedef enum odp_queue_type_t {
+ /** Plain queue
+ *
+ * Plain queues offer simple FIFO storage of events. Application may
+ * dequeue directly from these queues. */
+ ODP_QUEUE_TYPE_PLAIN = 0,
+
+ /** Scheduled queue
+ *
+ * Scheduled queues are connected to the scheduler. Application must
+ * not dequeue events directly from these queues but use the scheduler
+ * instead. */
+ ODP_QUEUE_TYPE_SCHED
+} odp_queue_type_t;
+
+/**
+ * Queue operation mode
+ */
+typedef enum odp_queue_op_mode_t {
+ /** Multithread safe operation
+ *
+ * Queue operation (enqueue or dequeue) is multithread safe. Any
+ * number of application threads may perform the operation
+ * concurrently. */
+ ODP_QUEUE_OP_MT = 0,
+
+ /** Not multithread safe operation
+ *
+ * Queue operation (enqueue or dequeue) may not be multithread safe.
+ * Application ensures synchronization between threads so that
+ * simultaneously only single thread attempts the operation on
+ * the same queue. */
+ ODP_QUEUE_OP_MT_UNSAFE,
+
+ /** Disabled
+ *
+ * Direct enqueue or dequeue operation from application is disabled.
+ * An attempt to enqueue/dequeue directly will result undefined
+ * behaviour. Various ODP functions (e.g. packet input, timer,
+ * crypto, scheduler, etc) are able to perform enqueue or
+ * dequeue operations normally on the queue.
+ * */
+ ODP_QUEUE_OP_DISABLED
+
+} odp_queue_op_mode_t;
+
+/**
+ * Non-blocking level
+ *
+ * A non-blocking level defines implementation guarantees for application
+ * progress when multiple threads operate on the same resource (e.g. a queue)
+ * simultaneously. The first level (ODP_BLOCKING) does not have any block
+ * freedom guarantees, but a suspending thread may block the other threads for
+ * the entire time it remains suspended (infinitely if crashed).
+ * On the contrary, actual non-blocking levels provide guarantees of progress:
+ *
+ * ODP_NONBLOCKING_LF: A non-blocking and lock-free implementation guarantees
+ * that at least one of the threads successfully completes
+ * its operations, regardless of what other threads do.
+ * Application progress is guaranteed, but individual
+ * threads may starve while trying to execute their
+ * operations on the shared resource.
+ *
+ * ODP_NONBLOCKING_WF: A non-blocking and wait-free implementation guarantees
+ * application progress with starvation freedom. All
+ * threads are guaranteed to complete their operations in
+ * a bounded number of steps, regardless of what other
+ * threads do.
+ *
+ * Non-blocking levels are listed from the weakest to the strongest guarantee of
+ * block freedom. Performance of a non-blocking implementation may be lower than
+ * the blocking one. Non-blocking guarantees are important e.g. for real-time
+ * applications when real-time and non real-time threads share a resource.
+ */
+typedef enum odp_nonblocking_t {
+ /** Blocking implementation. A suspeding thread may block all other
+ * threads, i.e. no block freedom guarantees. This is the lowest level.
+ */
+ ODP_BLOCKING = 0,
+
+ /** Non-blocking and lock-free implementation. Other threads can make
+ * progress while a thread is suspended. Starvation freedom is not
+ * guaranteed.
+ */
+ ODP_NONBLOCKING_LF,
+
+ /** Non-blocking and wait-free implementation. Other threads can make
+ * progress while a thread is suspended. Starvation freedom is
+ * guaranteed.
+ */
+ ODP_NONBLOCKING_WF
+
+} odp_nonblocking_t;
+
+/**
+ * Queue capabilities
+ */
+typedef struct odp_queue_capability_t {
+ /** Maximum number of event queues of any type (default size). Use
+ * this in addition to queue type specific 'max_num', if both queue
+ * types are used simultaneously. */
+ uint32_t max_queues;
+
+ /** Maximum number of ordered locks per queue */
+ uint32_t max_ordered_locks;
+
+ /** Maximum number of scheduling groups */
+ unsigned max_sched_groups;
+
+ /** Number of scheduling priorities */
+ unsigned sched_prios;
+
+ /** Plain queue capabilities */
+ struct {
+ /** Maximum number of plain (ODP_BLOCKING) queues of the
+ * default size. */
+ uint32_t max_num;
+
+ /** Maximum number of events a plain (ODP_BLOCKING) queue can
+ * store simultaneously. The value of zero means that plain
+ * queues do not have a size limit, but a single queue can
+ * store all available events. */
+ uint32_t max_size;
+
+ /** Lock-free (ODP_NONBLOCKING_LF) implementation capabilities.
+ * The specification is the same as for the blocking
+ * implementation. */
+ struct {
+ /** Maximum number of queues. Lock-free queues are not
+ * supported when zero. */
+ uint32_t max_num;
+
+ /** Maximum queue size */
+ uint32_t max_size;
+
+ } lockfree;
+
+ /** Wait-free (ODP_NONBLOCKING_WF) implementation capabilities.
+ * The specification is the same as for the blocking
+ * implementation. */
+ struct {
+ /** Maximum number of queues. Wait-free queues are not
+ * supported when zero. */
+ uint32_t max_num;
+
+ /** Maximum queue size */
+ uint32_t max_size;
+
+ } waitfree;
+
+ } plain;
+
+ /** Scheduled queue capabilities */
+ struct {
+ /** Maximum number of scheduled (ODP_BLOCKING) queues of the
+ * default size. */
+ uint32_t max_num;
+
+ /** Maximum number of events a scheduled (ODP_BLOCKING) queue
+ * can store simultaneously. The value of zero means that
+ * scheduled queues do not have a size limit, but a single
+ * queue can store all available events. */
+ uint32_t max_size;
+
+ /** Lock-free (ODP_NONBLOCKING_LF) implementation capabilities.
+ * The specification is the same as for the blocking
+ * implementation. */
+ struct {
+ /** Maximum number of queues. Lock-free queues are not
+ * supported when zero. */
+ uint32_t max_num;
+
+ /** Maximum queue size */
+ uint32_t max_size;
+
+ } lockfree;
+
+ /** Wait-free (ODP_NONBLOCKING_WF) implementation capabilities.
+ * The specification is the same as for the blocking
+ * implementation. */
+ struct {
+ /** Maximum number of queues. Wait-free queues are not
+ * supported when zero. */
+ uint32_t max_num;
+
+ /** Maximum queue size */
+ uint32_t max_size;
+
+ } waitfree;
+
+ } sched;
+
+} odp_queue_capability_t;
+
+/**
+ * ODP Queue parameters
+ */
+typedef struct odp_queue_param_t {
+ /** Queue type
+ *
+ * Valid values for other parameters in this structure depend on
+ * the queue type. */
+ odp_queue_type_t type;
+
+ /** Enqueue mode
+ *
+ * Default value for both queue types is ODP_QUEUE_OP_MT. Application
+ * may enable performance optimizations by defining MT_UNSAFE or
+ * DISABLED modes when applicable. */
+ odp_queue_op_mode_t enq_mode;
+
+ /** Dequeue mode
+ *
+ * For PLAIN queues, the default value is ODP_QUEUE_OP_MT. Application
+ * may enable performance optimizations by defining MT_UNSAFE or
+ * DISABLED modes when applicable. However, when a plain queue is input
+ * to the implementation (e.g. a queue for packet output), the
+ * parameter is ignored in queue creation and the value is
+ * ODP_QUEUE_OP_DISABLED.
+ *
+ * For SCHED queues, the parameter is ignored in queue creation and
+ * the value is ODP_QUEUE_OP_DISABLED. */
+ odp_queue_op_mode_t deq_mode;
+
+ /** Scheduler parameters
+ *
+ * These parameters are considered only when queue type is
+ * ODP_QUEUE_TYPE_SCHED. */
+ odp_schedule_param_t sched;
+
+ /** Non-blocking level
+ *
+ * Queue implementation must guarantee at least this level of block
+ * freedom for queue enqueue and dequeue/schedule operations.
+ * The default value is ODP_BLOCKING. */
+ odp_nonblocking_t nonblocking;
+
+ /** Queue context pointer
+ *
+ * User defined context pointer associated with the queue. The same
+ * pointer can be accessed with odp_queue_context() and
+ * odp_queue_context_set() calls. The implementation may read the
+ * pointer for prefetching the context data. Default value of the
+ * pointer is NULL. */
+ void *context;
+
+ /** Queue context data length
+ *
+ * User defined context data length in bytes for prefetching.
+ * The implementation may use this value as a hint for the number of
+ * context data bytes to prefetch. Default value is zero (no hint). */
+ uint32_t context_len;
+
+ /** Queue size
+ *
+ * The queue must be able to store at minimum this many events
+ * simultaneously. The value must not exceed 'max_size' queue
+ * capability. The value of zero means implementation specific
+ * default size. */
+ uint32_t size;
+
+} odp_queue_param_t;
+
+/**
+ * Queue information
+ * Retrieve information about a queue with odp_queue_info()
+ */
+typedef struct odp_queue_info_t {
+ const char *name; /**< queue name */
+ odp_queue_param_t param; /**< queue parameters */
+} odp_queue_info_t;
+
+/**
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <odp/visibility_end.h>
+#endif
diff --git a/m4/odp_dpdk.m4 b/m4/odp_dpdk.m4
index b3500edd..7e635a3a 100644
--- a/m4/odp_dpdk.m4
+++ b/m4/odp_dpdk.m4
@@ -128,9 +128,9 @@ AC_DEFUN([ODP_DPDK], [dnl
AS_IF([test "x$1" = "xsystem"], [dnl
DPDK_CPPFLAGS="-isystem /usr/include/dpdk"
DPDK_LDFLAGS=""
- DPDK_LIB_PATH="`$CC --print-file-name=libdpdk.so`"
+ DPDK_LIB_PATH="`$CC $CFLAGS $LDFLAGS --print-file-name=libdpdk.so`"
if test "$DPDK_LIB_PATH" = "libdpdk.so" ; then
- DPDK_LIB_PATH="`$CC --print-file-name=libdpdk.a`"
+ DPDK_LIB_PATH="`$CC $CFLAGS $LDFLAGS --print-file-name=libdpdk.a`"
AS_IF([test "$DPDK_LIB_PATH" = "libdpdk.a"],
[AC_MSG_FAILURE([Could not locate system DPDK library directory])])
else
diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am
index 0b75c630..9a15b85a 100644
--- a/platform/linux-dpdk/Makefile.am
+++ b/platform/linux-dpdk/Makefile.am
@@ -114,6 +114,7 @@ noinst_HEADERS = \
${top_srcdir}/platform/linux-generic/include/odp_queue_if.h \
${top_srcdir}/platform/linux-generic/include/odp_queue_lf.h \
${top_srcdir}/platform/linux-generic/include/odp_ring_internal.h \
+ include/odp_ring_mpmc_internal.h \
include/odp_ring_spsc_internal.h \
include/odp_ring_st_internal.h \
${top_srcdir}/platform/linux-generic/include/odp_schedule_if.h \
@@ -126,6 +127,7 @@ noinst_HEADERS = \
include/protocols/eth.h \
include/protocols/ip.h \
include/protocols/ipsec.h \
+ include/protocols/sctp.h \
include/protocols/tcp.h \
include/protocols/thash.h \
include/protocols/udp.h \
diff --git a/platform/linux-dpdk/include-abi/odp/api/abi/packet.h b/platform/linux-dpdk/include-abi/odp/api/abi/packet.h
index a8744cb8..3847f3c0 100644
--- a/platform/linux-dpdk/include-abi/odp/api/abi/packet.h
+++ b/platform/linux-dpdk/include-abi/odp/api/abi/packet.h
@@ -32,7 +32,7 @@ typedef ODP_HANDLE_T(odp_packet_t);
typedef ODP_HANDLE_T(odp_packet_seg_t);
-#define ODP_PACKET_SEG_INVALID _odp_cast_scalar(odp_packet_seg_t, NULL)
+#define ODP_PACKET_SEG_INVALID _odp_cast_scalar(odp_packet_seg_t, 0)
typedef uint8_t odp_proto_l2_type_t;
diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h b/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h
index e5f981ba..d1cad9f9 100644
--- a/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h
+++ b/platform/linux-dpdk/include/odp/api/plat/packet_inline_types.h
@@ -110,7 +110,7 @@ typedef union {
uint32_t all_flags;
struct {
- uint32_t reserved1: 11;
+ uint32_t reserved1: 10;
/*
* Init flags
@@ -134,6 +134,7 @@ typedef union {
uint32_t l3_chksum_err: 1; /* L3 checksum error */
uint32_t tcp_err: 1; /* TCP error */
uint32_t udp_err: 1; /* UDP error */
+ uint32_t sctp_err: 1; /* SCTP error */
uint32_t l4_chksum_err: 1; /* L4 checksum error */
uint32_t ipsec_err: 1; /* IPsec error */
uint32_t crypto_err: 1; /* Crypto packet operation error */
@@ -141,9 +142,9 @@ typedef union {
/* Flag groups */
struct {
- uint32_t reserved2: 11;
+ uint32_t reserved2: 10;
uint32_t other: 13; /* All other flags */
- uint32_t error: 8; /* All error flags */
+ uint32_t error: 9; /* All error flags */
} all;
} _odp_packet_flags_t;
diff --git a/platform/linux-dpdk/include/odp_buffer_internal.h b/platform/linux-dpdk/include/odp_buffer_internal.h
index 15ef8c12..235fc46f 100644
--- a/platform/linux-dpdk/include/odp_buffer_internal.h
+++ b/platform/linux-dpdk/include/odp_buffer_internal.h
@@ -115,11 +115,6 @@ static inline void _odp_buffer_event_type_set(odp_buffer_t buf, int ev)
buf_hdl_to_hdr(buf)->event_type = ev;
}
-static inline odp_event_t event_from_buf_hdr(odp_buffer_hdr_t *hdr)
-{
- return (odp_event_t)hdr;
-}
-
#ifdef __cplusplus
}
#endif
diff --git a/platform/linux-dpdk/include/odp_packet_internal.h b/platform/linux-dpdk/include/odp_packet_internal.h
index baaa2e66..e9e867b6 100644
--- a/platform/linux-dpdk/include/odp_packet_internal.h
+++ b/platform/linux-dpdk/include/odp_packet_internal.h
@@ -115,7 +115,7 @@ typedef struct {
odp_time_t timestamp;
/* Classifier destination queue */
- void *dst_queue;
+ odp_queue_t dst_queue;
/* Result for crypto packet op */
odp_crypto_packet_result_t crypto_op_result;
@@ -265,6 +265,7 @@ int _odp_packet_cmp_data(odp_packet_t pkt, uint32_t offset,
int _odp_packet_ipv4_chksum_insert(odp_packet_t pkt);
int _odp_packet_tcp_chksum_insert(odp_packet_t pkt);
int _odp_packet_udp_chksum_insert(odp_packet_t pkt);
+int _odp_packet_sctp_chksum_insert(odp_packet_t pkt);
/* We can't enforce tailroom reservation for received packets */
ODP_STATIC_ASSERT(CONFIG_PACKET_TAILROOM == 0,
diff --git a/platform/linux-dpdk/include/odp_packet_io_internal.h b/platform/linux-dpdk/include/odp_packet_io_internal.h
index a9a83a48..a6a93ba1 100644
--- a/platform/linux-dpdk/include/odp_packet_io_internal.h
+++ b/platform/linux-dpdk/include/odp_packet_io_internal.h
@@ -86,7 +86,6 @@ struct pktio_entry {
struct {
odp_queue_t queue;
- void *queue_int;
odp_pktin_queue_t pktin;
} in_queue[PKTIO_MAX_QUEUES];
diff --git a/platform/linux-dpdk/include/odp_pool_internal.h b/platform/linux-dpdk/include/odp_pool_internal.h
index a94afd7f..e063034c 100644
--- a/platform/linux-dpdk/include/odp_pool_internal.h
+++ b/platform/linux-dpdk/include/odp_pool_internal.h
@@ -73,7 +73,7 @@ static inline pool_t *pool_entry(uint32_t pool_idx)
static inline pool_t *pool_entry_from_hdl(odp_pool_t pool_hdl)
{
- return &pool_tbl->pool[_odp_typeval(pool_hdl)];
+ return &pool_tbl->pool[_odp_typeval(pool_hdl) - 1];
}
static inline void buffer_free_multi(odp_buffer_hdr_t *buf_hdr[], int num)
diff --git a/platform/linux-dpdk/include/odp_ring_mpmc_internal.h b/platform/linux-dpdk/include/odp_ring_mpmc_internal.h
new file mode 100644
index 00000000..30b7b506
--- /dev/null
+++ b/platform/linux-dpdk/include/odp_ring_mpmc_internal.h
@@ -0,0 +1,89 @@
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_RING_MPMC_INTERNAL_H_
+#define ODP_RING_MPMC_INTERNAL_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp/api/queue.h>
+
+#include <odp_debug_internal.h>
+
+#include <rte_config.h>
+#include <rte_ring.h>
+#include <rte_errno.h>
+
+/* Lock-free ring for multi-producer / multi-consumer usage.
+ *
+ * Enqueue and dequeue operations can be done concurrently.
+ */
+typedef struct rte_ring *ring_mpmc_t;
+
+static void ring_mpmc_name_to_mz_name(const char *name, char *ring_name)
+{
+ int i = 0;
+ int max_len = ODP_QUEUE_NAME_LEN < RTE_RING_NAMESIZE ?
+ ODP_QUEUE_NAME_LEN : RTE_RING_NAMESIZE;
+
+ do {
+ snprintf(ring_name, max_len, "%d-mpmc-%s", i++, name);
+ ring_name[max_len - 1] = 0;
+ } while (rte_ring_lookup(ring_name) != NULL);
+}
+
+/* Initialize ring. Ring size must be a power of two. */
+static inline ring_mpmc_t ring_mpmc_create(const char *name, uint32_t size)
+{
+ struct rte_ring *rte_ring;
+ char ring_name[RTE_RING_NAMESIZE];
+
+ /* Ring name must be unique */
+ ring_mpmc_name_to_mz_name(name, ring_name);
+
+ rte_ring = rte_ring_create(ring_name, size, rte_socket_id(), 0);
+ if (rte_ring == NULL) {
+ ODP_ERR("Creating DPDK ring failed: %s\n",
+ rte_strerror(rte_errno));
+ return NULL;
+ }
+
+ return rte_ring;
+}
+
+/* Free all memory used by the ring. */
+static inline void ring_mpmc_free(ring_mpmc_t ring)
+{
+ rte_ring_free(ring);
+}
+
+/* Dequeue data from the ring head. Max_num is smaller than ring size.*/
+static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t ring, void **data,
+ uint32_t max_num)
+{
+ return rte_ring_mc_dequeue_burst(ring, data, max_num, NULL);
+}
+
+/* Enqueue data into the ring tail. Num_data is smaller than ring size. */
+static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t ring, void **data,
+ uint32_t num_data)
+{
+ return rte_ring_mp_enqueue_burst(ring, data, num_data, NULL);
+}
+
+/* Check if ring is empty */
+static inline int ring_mpmc_is_empty(ring_mpmc_t ring)
+{
+ return rte_ring_empty(ring);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c
index 4016ac6d..9324c590 100644
--- a/platform/linux-dpdk/odp_crypto.c
+++ b/platform/linux-dpdk/odp_crypto.c
@@ -24,6 +24,7 @@
/* Inlined API functions */
#include <odp/api/plat/event_inlines.h>
+#include <odp/api/plat/queue_inlines.h>
#include <rte_config.h>
#include <rte_crypto.h>
diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c
index 7b19dac9..be01c4e8 100644
--- a/platform/linux-dpdk/odp_packet.c
+++ b/platform/linux-dpdk/odp_packet.c
@@ -21,6 +21,7 @@
#include <protocols/eth.h>
#include <protocols/ip.h>
+#include <protocols/sctp.h>
#include <protocols/tcp.h>
#include <protocols/udp.h>
@@ -1244,6 +1245,33 @@ static uint16_t packet_sum_ones_comp16(odp_packet_hdr_t *pkt_hdr,
return sum;
}
+static uint32_t packet_sum_crc32c(odp_packet_hdr_t *pkt_hdr,
+ uint32_t offset,
+ uint32_t len,
+ uint32_t init_val)
+{
+ uint32_t sum = init_val;
+
+ if (offset + len > odp_packet_len(packet_handle(pkt_hdr)))
+ return sum;
+
+ while (len > 0) {
+ uint32_t seglen = 0; /* GCC */
+
+ void *mapaddr = odp_packet_offset(packet_handle(pkt_hdr),
+ offset, &seglen, NULL);
+
+ if (seglen > len)
+ seglen = len;
+
+ sum = odp_hash_crc32c(mapaddr, seglen, sum);
+ len -= seglen;
+ offset += seglen;
+ }
+
+ return sum;
+}
+
/** Parser helper function for Ethernet packets */
static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr,
uint32_t *offset, uint32_t frame_len)
@@ -1526,6 +1554,34 @@ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr,
*parseptr += sizeof(_odp_udphdr_t);
}
+/**
+ * Parser helper function for SCTP
+ */
+static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr,
+ uint16_t sctp_len,
+ odp_proto_chksums_t chksums,
+ uint32_t *l4_part_sum)
+{
+ if (odp_unlikely(sctp_len < sizeof(_odp_sctphdr_t))) {
+ prs->flags.sctp_err = 1;
+ return;
+ }
+
+ if (chksums.chksum.sctp &&
+ !prs->input_flags.ipfrag) {
+ const _odp_sctphdr_t *sctp =
+ (const _odp_sctphdr_t *)*parseptr;
+ uint32_t crc = ~0;
+ uint32_t zero = 0;
+
+ crc = odp_hash_crc32c(sctp, sizeof(*sctp) - 4, crc);
+ crc = odp_hash_crc32c(&zero, 4, crc);
+ *l4_part_sum = crc;
+ }
+
+ *parseptr += sizeof(_odp_sctphdr_t);
+}
+
static inline
int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr,
uint32_t offset,
@@ -1616,6 +1672,8 @@ int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr,
case _ODP_IPPROTO_SCTP:
prs->input_flags.sctp = 1;
+ parse_sctp(prs, &parseptr, frame_len - prs->l4_offset, chksums,
+ l4_part_sum);
break;
case _ODP_IPPROTO_NO_NEXT:
@@ -1813,6 +1871,30 @@ int _odp_packet_udp_chksum_insert(odp_packet_t pkt)
return _odp_packet_tcp_udp_chksum_insert(pkt, _ODP_IPPROTO_UDP);
}
+/**
+ * Calculate and fill in SCTP checksum
+ *
+ * @param pkt ODP packet
+ *
+ * @retval 0 on success
+ * @retval <0 on failure
+ */
+int _odp_packet_sctp_chksum_insert(odp_packet_t pkt)
+{
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+ uint32_t sum;
+ uint32_t len = odp_packet_len(pkt);
+
+ if (pkt_hdr->p.l4_offset == ODP_PACKET_OFFSET_INVALID)
+ return -1;
+
+ sum = 0;
+ odp_packet_copy_from_mem(pkt, pkt_hdr->p.l4_offset + 8, 4, &sum);
+ sum = ~packet_sum_crc32c(pkt_hdr, pkt_hdr->p.l4_offset,
+ len - pkt_hdr->p.l4_offset, ~0);
+ return odp_packet_copy_from_mem(pkt, pkt_hdr->p.l4_offset + 8, 4, &sum);
+}
+
static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
odp_proto_chksums_t chksums,
uint32_t l4_part_sum)
@@ -1855,6 +1937,29 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
}
}
+ if (chksums.chksum.sctp &&
+ pkt_hdr->p.input_flags.sctp &&
+ !pkt_hdr->p.input_flags.ipfrag) {
+ uint32_t sum = ~packet_sum_crc32c(pkt_hdr,
+ pkt_hdr->p.l4_offset +
+ _ODP_SCTPHDR_LEN,
+ len -
+ pkt_hdr->p.l4_offset -
+ _ODP_SCTPHDR_LEN,
+ l4_part_sum);
+ _odp_sctphdr_t *sctp = odp_packet_offset(packet_handle(pkt_hdr),
+ pkt_hdr->p.l4_offset,
+ NULL, NULL);
+
+ pkt_hdr->p.input_flags.l4_chksum_done = 1;
+ if (sum != sctp->chksum) {
+ pkt_hdr->p.flags.l4_chksum_err = 1;
+ pkt_hdr->p.flags.sctp_err = 1;
+ ODP_DBG("SCTP chksum fail (%x/%x)!\n", sum,
+ sctp->chksum);
+ }
+ }
+
return pkt_hdr->p.flags.all_flags != 0;
}
diff --git a/platform/linux-dpdk/odp_packet_dpdk.c b/platform/linux-dpdk/odp_packet_dpdk.c
index f7df892b..9db5bbf3 100644
--- a/platform/linux-dpdk/odp_packet_dpdk.c
+++ b/platform/linux-dpdk/odp_packet_dpdk.c
@@ -561,7 +561,7 @@ static int start_pkt_dpdk(pktio_entry_t *pktio_entry)
for (i = 0; i < nbtxq; i++) {
ret = rte_eth_tx_queue_setup(port_id, i,
- pkt_dpdk->opt.num_rx_desc,
+ pkt_dpdk->opt.num_tx_desc,
socket_id, txconf);
if (ret < 0) {
ODP_ERR("txq:err=%d, port=%" PRIu16 "\n", ret, port_id);
diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c
index ee1877a3..62e8ed78 100644
--- a/platform/linux-dpdk/odp_pool.c
+++ b/platform/linux-dpdk/odp_pool.c
@@ -62,7 +62,7 @@ const _odp_pool_inline_offset_t ODP_ALIGNED_CACHE _odp_pool_inline = {
static inline odp_pool_t pool_index_to_handle(uint32_t pool_idx)
{
- return _odp_cast_scalar(odp_pool_t, pool_idx);
+ return _odp_cast_scalar(odp_pool_t, pool_idx + 1);
}
int odp_pool_init_global(void)
diff --git a/platform/linux-dpdk/odp_queue_basic.c b/platform/linux-dpdk/odp_queue_basic.c
index 2a33e194..670489da 100644
--- a/platform/linux-dpdk/odp_queue_basic.c
+++ b/platform/linux-dpdk/odp_queue_basic.c
@@ -48,11 +48,6 @@ static int queue_init(queue_entry_t *queue, const char *name,
queue_global_t *queue_glb;
extern _odp_queue_inline_offset_t _odp_queue_inline_offset;
-static inline queue_entry_t *qentry_from_handle(odp_queue_t handle)
-{
- return (queue_entry_t *)(uintptr_t)handle;
-}
-
static int queue_capa(odp_queue_capability_t *capa, int sched)
{
memset(capa, 0, sizeof(odp_queue_capability_t));
@@ -294,9 +289,9 @@ static odp_queue_t queue_create(const char *name,
if (!queue->s.spsc &&
param->nonblocking == ODP_NONBLOCKING_LF) {
- queue_lf_func_t *lf_func;
+ queue_lf_func_t *lf_fn;
- lf_func = &queue_glb->queue_lf_func;
+ lf_fn = &queue_glb->queue_lf_func;
queue_lf = queue_lf_create(queue);
@@ -306,10 +301,11 @@ static odp_queue_t queue_create(const char *name,
}
queue->s.queue_lf = queue_lf;
- queue->s.enqueue = lf_func->enq;
- queue->s.enqueue_multi = lf_func->enq_multi;
- queue->s.dequeue = lf_func->deq;
- queue->s.dequeue_multi = lf_func->deq_multi;
+ queue->s.enqueue = lf_fn->enq;
+ queue->s.enqueue_multi = lf_fn->enq_multi;
+ queue->s.dequeue = lf_fn->deq;
+ queue->s.dequeue_multi = lf_fn->deq_multi;
+ queue->s.orig_dequeue_multi = lf_fn->deq_multi;
}
type = queue->s.type;
@@ -388,8 +384,10 @@ static int queue_destroy(odp_queue_t handle)
if (queue->s.spsc)
empty = ring_spsc_is_empty(queue->s.ring_spsc);
- else
+ else if (queue->s.type == ODP_QUEUE_TYPE_SCHED)
empty = ring_st_is_empty(queue->s.ring_st);
+ else
+ empty = ring_mpmc_is_empty(queue->s.ring_mpmc);
if (!empty) {
UNLOCK(queue);
@@ -397,9 +395,11 @@ static int queue_destroy(odp_queue_t handle)
return -1;
}
if (queue->s.spsc)
- ring_spsc_free(queue->s.ring_st);
- else
+ ring_spsc_free(queue->s.ring_spsc);
+ else if (queue->s.type == ODP_QUEUE_TYPE_SCHED)
ring_st_free(queue->s.ring_st);
+ else
+ ring_mpmc_free(queue->s.ring_mpmc);
switch (queue->s.status) {
case QUEUE_STATUS_READY:
@@ -457,59 +457,50 @@ static odp_queue_t queue_lookup(const char *name)
return ODP_QUEUE_INVALID;
}
-static inline int enq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[],
- int num)
+static inline int _plain_queue_enq_multi(odp_queue_t handle,
+ odp_buffer_hdr_t *buf_hdr[], int num)
{
- int sched = 0;
- int ret;
queue_entry_t *queue;
- int num_enq;
+ int ret, num_enq;
+ ring_mpmc_t ring_mpmc;
- queue = q_int;
+ queue = qentry_from_handle(handle);
+ ring_mpmc = queue->s.ring_mpmc;
- if (sched_fn->ord_enq_multi(q_int, (void **)buf_hdr, num, &ret))
+ if (sched_fn->ord_enq_multi(handle, (void **)buf_hdr, num, &ret))
return ret;
- LOCK(queue);
-
- if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) {
- UNLOCK(queue);
- ODP_ERR("Bad queue status\n");
- return -1;
- }
-
- num_enq = ring_st_enq_multi(queue->s.ring_st, (void **)buf_hdr, num);
+ num_enq = ring_mpmc_enq_multi(ring_mpmc, (void **)buf_hdr, num);
- if (odp_unlikely(num_enq == 0)) {
- UNLOCK(queue);
- return 0;
- }
+ return num_enq;
+}
- if (queue->s.status == QUEUE_STATUS_NOTSCHED) {
- queue->s.status = QUEUE_STATUS_SCHED;
- sched = 1;
- }
+static inline int _plain_queue_deq_multi(odp_queue_t handle,
+ odp_buffer_hdr_t *buf_hdr[], int num)
+{
+ int num_deq;
+ queue_entry_t *queue;
+ ring_mpmc_t ring_mpmc;
- UNLOCK(queue);
+ queue = qentry_from_handle(handle);
+ ring_mpmc = queue->s.ring_mpmc;
- /* Add queue to scheduling */
- if (sched && sched_fn->sched_queue(queue->s.index))
- ODP_ABORT("schedule_queue failed\n");
+ num_deq = ring_mpmc_deq_multi(ring_mpmc, (void **)buf_hdr, num);
- return num_enq;
+ return num_deq;
}
-static int queue_int_enq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[],
- int num)
+static int plain_queue_enq_multi(odp_queue_t handle,
+ odp_buffer_hdr_t *buf_hdr[], int num)
{
- return enq_multi(q_int, buf_hdr, num);
+ return _plain_queue_enq_multi(handle, buf_hdr, num);
}
-static int queue_int_enq(void *q_int, odp_buffer_hdr_t *buf_hdr)
+static int plain_queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr)
{
int ret;
- ret = enq_multi(q_int, &buf_hdr, 1);
+ ret = _plain_queue_enq_multi(handle, &buf_hdr, 1);
if (ret == 1)
return 0;
@@ -517,62 +508,18 @@ static int queue_int_enq(void *q_int, odp_buffer_hdr_t *buf_hdr)
return -1;
}
-static int queue_enq_multi(odp_queue_t handle, const odp_event_t ev[], int num)
-{
- queue_entry_t *queue = qentry_from_handle(handle);
-
- if (odp_unlikely(num == 0))
- return 0;
-
- if (num > QUEUE_MULTI_MAX)
- num = QUEUE_MULTI_MAX;
-
- return queue->s.enqueue_multi(queue, (odp_buffer_hdr_t **)(uintptr_t)ev,
- num);
-}
-
-static int queue_enq(odp_queue_t handle, odp_event_t ev)
-{
- queue_entry_t *queue = qentry_from_handle(handle);
-
- return queue->s.enqueue(queue, (odp_buffer_hdr_t *)(uintptr_t)ev);
-}
-
-static inline int plain_queue_deq(queue_entry_t *queue,
- odp_buffer_hdr_t *buf_hdr[], int num)
-{
- int num_deq;
-
- LOCK(queue);
-
- if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) {
- /* Bad queue, or queue has been destroyed. */
- UNLOCK(queue);
- return -1;
- }
-
- num_deq = ring_st_deq_multi(queue->s.ring_st, (void **)buf_hdr, num);
-
- UNLOCK(queue);
-
- return num_deq;
-}
-
-static int queue_int_deq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[],
- int num)
+static int plain_queue_deq_multi(odp_queue_t handle,
+ odp_buffer_hdr_t *buf_hdr[], int num)
{
- queue_entry_t *queue = q_int;
-
- return plain_queue_deq(queue, buf_hdr, num);
+ return _plain_queue_deq_multi(handle, buf_hdr, num);
}
-static odp_buffer_hdr_t *queue_int_deq(void *q_int)
+static odp_buffer_hdr_t *plain_queue_deq(odp_queue_t handle)
{
- queue_entry_t *queue = q_int;
odp_buffer_hdr_t *buf_hdr = NULL;
int ret;
- ret = plain_queue_deq(queue, &buf_hdr, 1);
+ ret = _plain_queue_deq_multi(handle, &buf_hdr, 1);
if (ret == 1)
return buf_hdr;
@@ -580,36 +527,50 @@ static odp_buffer_hdr_t *queue_int_deq(void *q_int)
return NULL;
}
-static int queue_deq_multi(odp_queue_t handle, odp_event_t ev[], int num)
+static int error_enqueue(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr)
{
- queue_entry_t *queue = qentry_from_handle(handle);
+ (void)buf_hdr;
- if (num > QUEUE_MULTI_MAX)
- num = QUEUE_MULTI_MAX;
+ ODP_ERR("Enqueue not supported (" PRIu64 ")\n",
+ odp_queue_to_u64(handle));
- return queue->s.dequeue_multi(queue, (odp_buffer_hdr_t **)ev, num);
+ return -1;
}
-static odp_event_t queue_deq(odp_queue_t handle)
+static int error_enqueue_multi(odp_queue_t handle,
+ odp_buffer_hdr_t *buf_hdr[], int num)
+
{
- queue_entry_t *queue = qentry_from_handle(handle);
+ (void)buf_hdr;
+ (void)num;
+
+ ODP_ERR("Enqueue multi not supported (" PRIu64 ")\n",
+ odp_queue_to_u64(handle));
- return (odp_event_t)queue->s.dequeue(queue);
+ return -1;
}
-static int timer_queue_deq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[],
- int num)
+static int timer_queue_deq_multi(odp_queue_t handle,
+ odp_buffer_hdr_t *buf_hdr[], int num)
{
timer_run();
- return queue_int_deq_multi(q_int, buf_hdr, num);
+ return _plain_queue_deq_multi(handle, buf_hdr, num);
}
-static odp_buffer_hdr_t *timer_queue_deq(void *q_int)
+static odp_buffer_hdr_t *timer_queue_deq(odp_queue_t handle)
{
+ odp_buffer_hdr_t *buf_hdr = NULL;
+ int ret;
+
timer_run();
- return queue_int_deq(q_int);
+ ret = _plain_queue_deq_multi(handle, &buf_hdr, 1);
+
+ if (ret == 1)
+ return buf_hdr;
+ else
+ return NULL;
}
/* Enable timer polling on dequeue call */
@@ -627,67 +588,24 @@ void queue_enable_timer_poll(odp_queue_t handle)
UNLOCK(queue);
}
-static int queue_init(queue_entry_t *queue, const char *name,
- const odp_queue_param_t *param)
+static odp_buffer_hdr_t *error_dequeue(odp_queue_t handle)
{
- uint32_t queue_size;
- int spsc;
-
- if (name == NULL) {
- queue->s.name[0] = 0;
- } else {
- strncpy(queue->s.name, name, ODP_QUEUE_NAME_LEN - 1);
- queue->s.name[ODP_QUEUE_NAME_LEN - 1] = 0;
- }
- memcpy(&queue->s.param, param, sizeof(odp_queue_param_t));
- if (queue->s.param.sched.lock_count > sched_fn->max_ordered_locks())
- return -1;
-
- if (param->type == ODP_QUEUE_TYPE_SCHED)
- queue->s.param.deq_mode = ODP_QUEUE_OP_DISABLED;
-
- queue->s.type = queue->s.param.type;
-
- queue->s.pktin = PKTIN_INVALID;
- queue->s.pktout = PKTOUT_INVALID;
-
- /* Use default size for all small queues to quarantee performance
- * level. */
- queue_size = queue_glb->config.default_queue_size;
- if (param->size > queue_glb->config.default_queue_size)
- queue_size = param->size;
-
- /* Round up if not already a power of two */
- queue_size = ROUNDUP_POWER2_U32(queue_size);
-
- if (queue_size > queue_glb->config.max_queue_size) {
- ODP_ERR("Too large queue size %u\n", queue_size);
- return -1;
- }
+ ODP_ERR("Dequeue not supported (" PRIu64 ")\n",
+ odp_queue_to_u64(handle));
- /* Single-producer / single-consumer plain queue has simple and
- * lock-free implementation */
- spsc = (param->type == ODP_QUEUE_TYPE_PLAIN) &&
- (param->enq_mode == ODP_QUEUE_OP_MT_UNSAFE) &&
- (param->deq_mode == ODP_QUEUE_OP_MT_UNSAFE);
-
- queue->s.spsc = spsc;
- queue->s.queue_lf = NULL;
+ return NULL;
+}
- if (spsc) {
- queue_spsc_init(queue, queue_size);
- } else {
- queue->s.enqueue = queue_int_enq;
- queue->s.dequeue = queue_int_deq;
- queue->s.enqueue_multi = queue_int_enq_multi;
- queue->s.dequeue_multi = queue_int_deq_multi;
+static int error_dequeue_multi(odp_queue_t handle,
+ odp_buffer_hdr_t *buf_hdr[], int num)
+{
+ (void)buf_hdr;
+ (void)num;
- queue->s.ring_st = ring_st_create(queue->s.name, queue_size);
- if (queue->s.ring_st == NULL)
- return -1;
- }
+ ODP_ERR("Dequeue multi not supported (" PRIu64 ")\n",
+ odp_queue_to_u64(handle));
- return 0;
+ return -1;
}
static void queue_param_init(odp_queue_param_t *params)
@@ -741,6 +659,50 @@ static int queue_info(odp_queue_t handle, odp_queue_info_t *info)
return 0;
}
+static inline int _sched_queue_enq_multi(odp_queue_t handle,
+ odp_buffer_hdr_t *buf_hdr[], int num)
+{
+ int sched = 0;
+ int ret;
+ queue_entry_t *queue;
+ int num_enq;
+ ring_st_t ring_st;
+
+ queue = qentry_from_handle(handle);
+ ring_st = queue->s.ring_st;
+
+ if (sched_fn->ord_enq_multi(handle, (void **)buf_hdr, num, &ret))
+ return ret;
+
+ LOCK(queue);
+
+ if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) {
+ UNLOCK(queue);
+ ODP_ERR("Bad queue status\n");
+ return -1;
+ }
+
+ num_enq = ring_st_enq_multi(ring_st, (void **)buf_hdr, num);
+
+ if (odp_unlikely(num_enq == 0)) {
+ UNLOCK(queue);
+ return 0;
+ }
+
+ if (queue->s.status == QUEUE_STATUS_NOTSCHED) {
+ queue->s.status = QUEUE_STATUS_SCHED;
+ sched = 1;
+ }
+
+ UNLOCK(queue);
+
+ /* Add queue to scheduling */
+ if (sched && sched_fn->sched_queue(queue->s.index))
+ ODP_ABORT("schedule_queue failed\n");
+
+ return num_enq;
+}
+
int sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num,
int update_status)
{
@@ -784,6 +746,24 @@ int sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num,
return num_deq;
}
+static int sched_queue_enq_multi(odp_queue_t handle,
+ odp_buffer_hdr_t *buf_hdr[], int num)
+{
+ return _sched_queue_enq_multi(handle, buf_hdr, num);
+}
+
+static int sched_queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr)
+{
+ int ret;
+
+ ret = _sched_queue_enq_multi(handle, &buf_hdr, 1);
+
+ if (ret == 1)
+ return 0;
+ else
+ return -1;
+}
+
int sched_queue_empty(uint32_t queue_index)
{
queue_entry_t *queue = qentry_from_index(queue_index);
@@ -810,48 +790,137 @@ int sched_queue_empty(uint32_t queue_index)
return ret;
}
+static int queue_init(queue_entry_t *queue, const char *name,
+ const odp_queue_param_t *param)
+{
+ uint32_t queue_size;
+ odp_queue_type_t queue_type;
+ int spsc;
+
+ queue_type = param->type;
+
+ if (name == NULL) {
+ queue->s.name[0] = 0;
+ } else {
+ strncpy(queue->s.name, name, ODP_QUEUE_NAME_LEN - 1);
+ queue->s.name[ODP_QUEUE_NAME_LEN - 1] = 0;
+ }
+ memcpy(&queue->s.param, param, sizeof(odp_queue_param_t));
+ if (queue->s.param.sched.lock_count > sched_fn->max_ordered_locks())
+ return -1;
+
+ if (queue_type == ODP_QUEUE_TYPE_SCHED)
+ queue->s.param.deq_mode = ODP_QUEUE_OP_DISABLED;
+
+ queue->s.type = queue_type;
+
+ queue->s.pktin = PKTIN_INVALID;
+ queue->s.pktout = PKTOUT_INVALID;
+
+ /* Use default size for all small queues to quarantee performance
+ * level. */
+ queue_size = queue_glb->config.default_queue_size;
+ if (param->size > queue_glb->config.default_queue_size)
+ queue_size = param->size;
+
+ /* Round up if not already a power of two */
+ queue_size = ROUNDUP_POWER2_U32(queue_size);
+
+ if (queue_size > queue_glb->config.max_queue_size) {
+ ODP_ERR("Too large queue size %u\n", queue_size);
+ return -1;
+ }
+
+ /* Single-producer / single-consumer plain queue has simple and
+ * lock-free implementation */
+ spsc = (queue_type == ODP_QUEUE_TYPE_PLAIN) &&
+ (param->enq_mode == ODP_QUEUE_OP_MT_UNSAFE) &&
+ (param->deq_mode == ODP_QUEUE_OP_MT_UNSAFE);
+
+ queue->s.spsc = spsc;
+ queue->s.queue_lf = NULL;
+
+ /* Default to error functions */
+ queue->s.enqueue = error_enqueue;
+ queue->s.enqueue_multi = error_enqueue_multi;
+ queue->s.dequeue = error_dequeue;
+ queue->s.dequeue_multi = error_dequeue_multi;
+ queue->s.orig_dequeue_multi = error_dequeue_multi;
+
+ if (spsc) {
+ queue_spsc_init(queue, queue_size);
+ } else {
+ if (queue_type == ODP_QUEUE_TYPE_PLAIN) {
+ queue->s.enqueue = plain_queue_enq;
+ queue->s.enqueue_multi = plain_queue_enq_multi;
+ queue->s.dequeue = plain_queue_deq;
+ queue->s.dequeue_multi = plain_queue_deq_multi;
+ queue->s.orig_dequeue_multi = plain_queue_deq_multi;
+
+ queue->s.ring_mpmc = ring_mpmc_create(queue->s.name,
+ queue_size);
+ if (queue->s.ring_mpmc == NULL) {
+ ODP_ERR("Creating MPMC ring failed\n");
+ return -1;
+ }
+ } else {
+ queue->s.enqueue = sched_queue_enq;
+ queue->s.enqueue_multi = sched_queue_enq_multi;
+
+ queue->s.ring_st = ring_st_create(queue->s.name,
+ queue_size);
+ if (queue->s.ring_st == NULL) {
+ ODP_ERR("Creating ST ring failed\n");
+ return -1;
+ }
+ }
+ }
+
+ return 0;
+}
+
static uint64_t queue_to_u64(odp_queue_t hdl)
{
return _odp_pri(hdl);
}
-static odp_pktout_queue_t queue_get_pktout(void *q_int)
+static odp_pktout_queue_t queue_get_pktout(odp_queue_t handle)
{
- queue_entry_t *qentry = q_int;
+ queue_entry_t *qentry = qentry_from_handle(handle);
return qentry->s.pktout;
}
-static void queue_set_pktout(void *q_int, odp_pktio_t pktio, int index)
+static void queue_set_pktout(odp_queue_t handle, odp_pktio_t pktio, int index)
{
- queue_entry_t *qentry = q_int;
+ queue_entry_t *qentry = qentry_from_handle(handle);
qentry->s.pktout.pktio = pktio;
qentry->s.pktout.index = index;
}
-static odp_pktin_queue_t queue_get_pktin(void *q_int)
+static odp_pktin_queue_t queue_get_pktin(odp_queue_t handle)
{
- queue_entry_t *qentry = q_int;
+ queue_entry_t *qentry = qentry_from_handle(handle);
return qentry->s.pktin;
}
-static void queue_set_pktin(void *q_int, odp_pktio_t pktio, int index)
+static void queue_set_pktin(odp_queue_t handle, odp_pktio_t pktio, int index)
{
- queue_entry_t *qentry = q_int;
+ queue_entry_t *qentry = qentry_from_handle(handle);
qentry->s.pktin.pktio = pktio;
qentry->s.pktin.index = index;
}
-static void queue_set_enq_deq_func(void *q_int,
+static void queue_set_enq_deq_func(odp_queue_t handle,
queue_enq_fn_t enq,
queue_enq_multi_fn_t enq_multi,
queue_deq_fn_t deq,
queue_deq_multi_fn_t deq_multi)
{
- queue_entry_t *qentry = q_int;
+ queue_entry_t *qentry = qentry_from_handle(handle);
if (enq)
qentry->s.enqueue = enq;
@@ -866,29 +935,66 @@ static void queue_set_enq_deq_func(void *q_int,
qentry->s.dequeue_multi = deq_multi;
}
-static void *queue_from_ext(odp_queue_t handle)
+static int queue_orig_multi(odp_queue_t handle,
+ odp_buffer_hdr_t **buf_hdr, int num)
+{
+ queue_entry_t *queue = qentry_from_handle(handle);
+
+ return queue->s.orig_dequeue_multi(handle, buf_hdr, num);
+}
+
+static int queue_api_enq_multi(odp_queue_t handle,
+ const odp_event_t ev[], int num)
+{
+ queue_entry_t *queue = qentry_from_handle(handle);
+
+ if (odp_unlikely(num == 0))
+ return 0;
+
+ if (num > QUEUE_MULTI_MAX)
+ num = QUEUE_MULTI_MAX;
+
+ return queue->s.enqueue_multi(handle,
+ (odp_buffer_hdr_t **)(uintptr_t)ev, num);
+}
+
+static int queue_api_enq(odp_queue_t handle, odp_event_t ev)
{
- return qentry_from_handle(handle);
+ queue_entry_t *queue = qentry_from_handle(handle);
+
+ return queue->s.enqueue(handle,
+ (odp_buffer_hdr_t *)(uintptr_t)ev);
}
-static odp_queue_t queue_to_ext(void *q_int)
+static int queue_api_deq_multi(odp_queue_t handle, odp_event_t ev[], int num)
{
- queue_entry_t *qentry = q_int;
+ queue_entry_t *queue = qentry_from_handle(handle);
+
+ if (num > QUEUE_MULTI_MAX)
+ num = QUEUE_MULTI_MAX;
+
+ return queue->s.dequeue_multi(handle,
+ (odp_buffer_hdr_t **)ev, num);
+}
+
+static odp_event_t queue_api_deq(odp_queue_t handle)
+{
+ queue_entry_t *queue = qentry_from_handle(handle);
- return qentry->s.handle;
+ return (odp_event_t)queue->s.dequeue(handle);
}
/* API functions */
-queue_api_t queue_basic_api = {
+_odp_queue_api_fn_t queue_basic_api = {
.queue_create = queue_create,
.queue_destroy = queue_destroy,
.queue_lookup = queue_lookup,
.queue_capability = queue_capability,
.queue_context_set = queue_context_set,
- .queue_enq = queue_enq,
- .queue_enq_multi = queue_enq_multi,
- .queue_deq = queue_deq,
- .queue_deq_multi = queue_deq_multi,
+ .queue_enq = queue_api_enq,
+ .queue_enq_multi = queue_api_enq_multi,
+ .queue_deq = queue_api_deq,
+ .queue_deq_multi = queue_api_deq_multi,
.queue_type = queue_type,
.queue_sched_type = queue_sched_type,
.queue_sched_prio = queue_sched_prio,
@@ -905,15 +1011,10 @@ queue_fn_t queue_basic_fn = {
.term_global = queue_term_global,
.init_local = queue_init_local,
.term_local = queue_term_local,
- .from_ext = queue_from_ext,
- .to_ext = queue_to_ext,
- .enq = queue_int_enq,
- .enq_multi = queue_int_enq_multi,
- .deq = queue_int_deq,
- .deq_multi = queue_int_deq_multi,
.get_pktout = queue_get_pktout,
.set_pktout = queue_set_pktout,
.get_pktin = queue_get_pktin,
.set_pktin = queue_set_pktin,
- .set_enq_deq_fn = queue_set_enq_deq_func
+ .set_enq_deq_fn = queue_set_enq_deq_func,
+ .orig_deq_multi = queue_orig_multi
};
diff --git a/platform/linux-dpdk/odp_queue_if.c b/platform/linux-dpdk/odp_queue_if.c
index bb9a8e1d..3095faae 100644
--- a/platform/linux-dpdk/odp_queue_if.c
+++ b/platform/linux-dpdk/odp_queue_if.c
@@ -19,98 +19,78 @@
#include <odp/visibility_begin.h>
_odp_queue_inline_offset_t ODP_ALIGNED_CACHE _odp_queue_inline_offset;
+const _odp_queue_api_fn_t *_odp_queue_api;
#include <odp/visibility_end.h>
-extern const queue_api_t queue_basic_api;
+extern const _odp_queue_api_fn_t queue_basic_api;
extern const queue_fn_t queue_basic_fn;
-const queue_api_t *queue_api;
const queue_fn_t *queue_fn;
odp_queue_t odp_queue_create(const char *name, const odp_queue_param_t *param)
{
- return queue_api->queue_create(name, param);
+ return _odp_queue_api->queue_create(name, param);
}
int odp_queue_destroy(odp_queue_t queue)
{
- return queue_api->queue_destroy(queue);
+ return _odp_queue_api->queue_destroy(queue);
}
odp_queue_t odp_queue_lookup(const char *name)
{
- return queue_api->queue_lookup(name);
+ return _odp_queue_api->queue_lookup(name);
}
int odp_queue_capability(odp_queue_capability_t *capa)
{
- return queue_api->queue_capability(capa);
+ return _odp_queue_api->queue_capability(capa);
}
int odp_queue_context_set(odp_queue_t queue, void *context, uint32_t len)
{
- return queue_api->queue_context_set(queue, context, len);
-}
-
-int odp_queue_enq(odp_queue_t queue, odp_event_t ev)
-{
- return queue_api->queue_enq(queue, ev);
-}
-
-int odp_queue_enq_multi(odp_queue_t queue, const odp_event_t events[], int num)
-{
- return queue_api->queue_enq_multi(queue, events, num);
-}
-
-odp_event_t odp_queue_deq(odp_queue_t queue)
-{
- return queue_api->queue_deq(queue);
-}
-
-int odp_queue_deq_multi(odp_queue_t queue, odp_event_t events[], int num)
-{
- return queue_api->queue_deq_multi(queue, events, num);
+ return _odp_queue_api->queue_context_set(queue, context, len);
}
odp_queue_type_t odp_queue_type(odp_queue_t queue)
{
- return queue_api->queue_type(queue);
+ return _odp_queue_api->queue_type(queue);
}
odp_schedule_sync_t odp_queue_sched_type(odp_queue_t queue)
{
- return queue_api->queue_sched_type(queue);
+ return _odp_queue_api->queue_sched_type(queue);
}
odp_schedule_prio_t odp_queue_sched_prio(odp_queue_t queue)
{
- return queue_api->queue_sched_prio(queue);
+ return _odp_queue_api->queue_sched_prio(queue);
}
odp_schedule_group_t odp_queue_sched_group(odp_queue_t queue)
{
- return queue_api->queue_sched_group(queue);
+ return _odp_queue_api->queue_sched_group(queue);
}
uint32_t odp_queue_lock_count(odp_queue_t queue)
{
- return queue_api->queue_lock_count(queue);
+ return _odp_queue_api->queue_lock_count(queue);
}
uint64_t odp_queue_to_u64(odp_queue_t hdl)
{
- return queue_api->queue_to_u64(hdl);
+ return _odp_queue_api->queue_to_u64(hdl);
}
void odp_queue_param_init(odp_queue_param_t *param)
{
- return queue_api->queue_param_init(param);
+ return _odp_queue_api->queue_param_init(param);
}
int odp_queue_info(odp_queue_t queue, odp_queue_info_t *info)
{
- return queue_api->queue_info(queue, info);
+ return _odp_queue_api->queue_info(queue, info);
}
int _odp_queue_init_global(void)
@@ -124,7 +104,7 @@ int _odp_queue_init_global(void)
!strcmp(sched, "sp") ||
!strcmp(sched, "iquery")) {
queue_fn = &queue_basic_fn;
- queue_api = &queue_basic_api;
+ _odp_queue_api = &queue_basic_api;
} else {
ODP_ABORT("Unknown scheduler specified via ODP_SCHEDULER\n");
return -1;
diff --git a/platform/linux-dpdk/odp_queue_spsc.c b/platform/linux-dpdk/odp_queue_spsc.c
index 89915eb9..26ff223f 100644
--- a/platform/linux-dpdk/odp_queue_spsc.c
+++ b/platform/linux-dpdk/odp_queue_spsc.c
@@ -9,13 +9,13 @@
#include "config.h"
#include <odp_debug_internal.h>
-static inline int spsc_enq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[],
- int num)
+static inline int spsc_enq_multi(odp_queue_t handle,
+ odp_buffer_hdr_t *buf_hdr[], int num)
{
queue_entry_t *queue;
ring_spsc_t ring_spsc;
- queue = q_int;
+ queue = qentry_from_handle(handle);
ring_spsc = queue->s.ring_spsc;
if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) {
@@ -26,13 +26,13 @@ static inline int spsc_enq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[],
return ring_spsc_enq_multi(ring_spsc, (void **)buf_hdr, num);
}
-static inline int spsc_deq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[],
- int num)
+static inline int spsc_deq_multi(odp_queue_t handle,
+ odp_buffer_hdr_t *buf_hdr[], int num)
{
queue_entry_t *queue;
ring_spsc_t ring_spsc;
- queue = q_int;
+ queue = qentry_from_handle(handle);
ring_spsc = queue->s.ring_spsc;
if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) {
@@ -43,17 +43,17 @@ static inline int spsc_deq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[],
return ring_spsc_deq_multi(ring_spsc, (void **)buf_hdr, num);
}
-static int queue_spsc_enq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[],
+static int queue_spsc_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[],
int num)
{
- return spsc_enq_multi(q_int, buf_hdr, num);
+ return spsc_enq_multi(handle, buf_hdr, num);
}
-static int queue_spsc_enq(void *q_int, odp_buffer_hdr_t *buf_hdr)
+static int queue_spsc_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr)
{
int ret;
- ret = spsc_enq_multi(q_int, &buf_hdr, 1);
+ ret = spsc_enq_multi(handle, &buf_hdr, 1);
if (ret == 1)
return 0;
@@ -61,18 +61,18 @@ static int queue_spsc_enq(void *q_int, odp_buffer_hdr_t *buf_hdr)
return -1;
}
-static int queue_spsc_deq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[],
+static int queue_spsc_deq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[],
int num)
{
- return spsc_deq_multi(q_int, buf_hdr, num);
+ return spsc_deq_multi(handle, buf_hdr, num);
}
-static odp_buffer_hdr_t *queue_spsc_deq(void *q_int)
+static odp_buffer_hdr_t *queue_spsc_deq(odp_queue_t handle)
{
odp_buffer_hdr_t *buf_hdr = NULL;
int ret;
- ret = spsc_deq_multi(q_int, &buf_hdr, 1);
+ ret = spsc_deq_multi(handle, &buf_hdr, 1);
if (ret == 1)
return buf_hdr;
@@ -86,6 +86,7 @@ void queue_spsc_init(queue_entry_t *queue, uint32_t queue_size)
queue->s.dequeue = queue_spsc_deq;
queue->s.enqueue_multi = queue_spsc_enq_multi;
queue->s.dequeue_multi = queue_spsc_deq_multi;
+ queue->s.orig_dequeue_multi = queue_spsc_deq_multi;
queue->s.ring_spsc = ring_spsc_create(queue->s.name, queue_size);
if (queue->s.ring_spsc == NULL)
diff --git a/platform/linux-dpdk/odp_timer.c b/platform/linux-dpdk/odp_timer.c
index cd04ba20..31d07009 100644
--- a/platform/linux-dpdk/odp_timer.c
+++ b/platform/linux-dpdk/odp_timer.c
@@ -11,6 +11,7 @@
#include <odp/api/shared_memory.h>
#include <odp/api/ticketlock.h>
#include <odp/api/timer.h>
+#include <odp/api/plat/queue_inlines.h>
#include <odp_init_internal.h>
#include <odp_debug_internal.h>
@@ -21,7 +22,6 @@
#include <rte_timer.h>
/* TODO: timer ABI spec needs update
- * - ODP_TIMER_INVALID: 0xffffffff -> NULL
* - Remove "struct timer_pool_s"
*/
diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am
index 13101cfd..ab0b755d 100644
--- a/platform/linux-generic/Makefile.am
+++ b/platform/linux-generic/Makefile.am
@@ -127,6 +127,7 @@ noinst_HEADERS = \
include/odp_queue_lf.h \
include/odp_queue_scalable_internal.h \
include/odp_ring_internal.h \
+ include/odp_ring_mpmc_internal.h \
include/odp_ring_spsc_internal.h \
include/odp_ring_st_internal.h \
include/odp_schedule_if.h \
@@ -142,6 +143,7 @@ noinst_HEADERS = \
include/protocols/eth.h \
include/protocols/ip.h \
include/protocols/ipsec.h \
+ include/protocols/sctp.h \
include/protocols/tcp.h \
include/protocols/thash.h \
include/protocols/udp.h
diff --git a/platform/linux-generic/include-abi/odp/api/abi/classification.h b/platform/linux-generic/include-abi/odp/api/abi/classification.h
index 1266e3f1..92b926cd 100644
--- a/platform/linux-generic/include-abi/odp/api/abi/classification.h
+++ b/platform/linux-generic/include-abi/odp/api/abi/classification.h
@@ -24,10 +24,10 @@ extern "C" {
*/
typedef ODP_HANDLE_T(odp_cos_t);
-#define ODP_COS_INVALID _odp_cast_scalar(odp_cos_t, ~0)
+#define ODP_COS_INVALID _odp_cast_scalar(odp_cos_t, 0)
typedef ODP_HANDLE_T(odp_pmr_t);
-#define ODP_PMR_INVAL _odp_cast_scalar(odp_pmr_t, ~0)
+#define ODP_PMR_INVAL _odp_cast_scalar(odp_pmr_t, 0)
#define ODP_COS_NAME_LEN 32
diff --git a/platform/linux-generic/include-abi/odp/api/abi/ipsec.h b/platform/linux-generic/include-abi/odp/api/abi/ipsec.h
index b91da795..a04bb174 100644
--- a/platform/linux-generic/include-abi/odp/api/abi/ipsec.h
+++ b/platform/linux-generic/include-abi/odp/api/abi/ipsec.h
@@ -26,7 +26,7 @@ extern "C" {
typedef ODP_HANDLE_T(odp_ipsec_sa_t);
-#define ODP_IPSEC_SA_INVALID _odp_cast_scalar(odp_ipsec_sa_t, 0xffffffff)
+#define ODP_IPSEC_SA_INVALID _odp_cast_scalar(odp_ipsec_sa_t, 0)
/**
* @}
diff --git a/platform/linux-generic/include-abi/odp/api/abi/pool.h b/platform/linux-generic/include-abi/odp/api/abi/pool.h
index cd161d5a..9ac1cf67 100644
--- a/platform/linux-generic/include-abi/odp/api/abi/pool.h
+++ b/platform/linux-generic/include-abi/odp/api/abi/pool.h
@@ -27,7 +27,7 @@ extern "C" {
typedef ODP_HANDLE_T(odp_pool_t);
-#define ODP_POOL_INVALID _odp_cast_scalar(odp_pool_t, 0xffffffff)
+#define ODP_POOL_INVALID _odp_cast_scalar(odp_pool_t, 0)
#define ODP_POOL_NAME_LEN 32
diff --git a/platform/linux-generic/include-abi/odp/api/abi/timer.h b/platform/linux-generic/include-abi/odp/api/abi/timer.h
index 49752df7..48138f72 100644
--- a/platform/linux-generic/include-abi/odp/api/abi/timer.h
+++ b/platform/linux-generic/include-abi/odp/api/abi/timer.h
@@ -28,17 +28,17 @@ struct timer_pool_s; /**< Forward declaration */
typedef struct timer_pool_s *odp_timer_pool_t;
-#define ODP_TIMER_POOL_INVALID NULL
+#define ODP_TIMER_POOL_INVALID _odp_cast_scalar(odp_timer_pool_t, 0)
#define ODP_TIMER_POOL_NAME_LEN 32
typedef ODP_HANDLE_T(odp_timer_t);
-#define ODP_TIMER_INVALID _odp_cast_scalar(odp_timer_t, 0xffffffff)
+#define ODP_TIMER_INVALID _odp_cast_scalar(odp_timer_t, 0)
typedef ODP_HANDLE_T(odp_timeout_t);
-#define ODP_TIMEOUT_INVALID _odp_cast_scalar(odp_timeout_t, NULL)
+#define ODP_TIMEOUT_INVALID _odp_cast_scalar(odp_timeout_t, 0)
/**
* @}
diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h
index 9a285fe4..25510298 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h
@@ -111,7 +111,7 @@ typedef union {
uint32_t all_flags;
struct {
- uint32_t reserved1: 11;
+ uint32_t reserved1: 10;
/*
* Init flags
@@ -135,6 +135,7 @@ typedef union {
uint32_t l3_chksum_err: 1; /* L3 checksum error */
uint32_t tcp_err: 1; /* TCP error */
uint32_t udp_err: 1; /* UDP error */
+ uint32_t sctp_err: 1; /* SCTP error */
uint32_t l4_chksum_err: 1; /* L4 checksum error */
uint32_t ipsec_err: 1; /* IPsec error */
uint32_t crypto_err: 1; /* Crypto packet operation error */
@@ -142,9 +143,9 @@ typedef union {
/* Flag groups */
struct {
- uint32_t reserved2: 11;
+ uint32_t reserved2: 10;
uint32_t other: 13; /* All other flags */
- uint32_t error: 8; /* All error flags */
+ uint32_t error: 9; /* All error flags */
} all;
} _odp_packet_flags_t;
diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h
index e2c32728..602d3344 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h
@@ -85,12 +85,12 @@ extern const _odp_pool_inline_offset_t _odp_pool_inline;
#include <odp/api/plat/strong_types.h>
static inline uint32_t _odp_packet_seg_to_ndx(odp_packet_seg_t seg)
{
- return _odp_typeval(seg);
+ return _odp_typeval(seg) - 1;
}
static inline odp_packet_seg_t _odp_packet_seg_from_ndx(uint32_t ndx)
{
- return _odp_cast_scalar(odp_packet_seg_t, ndx);
+ return _odp_cast_scalar(odp_packet_seg_t, ndx + 1);
}
#endif
diff --git a/platform/linux-generic/include/odp/api/plat/queue_inline_types.h b/platform/linux-generic/include/odp/api/plat/queue_inline_types.h
index 3a8df7a4..f81a8478 100644
--- a/platform/linux-generic/include/odp/api/plat/queue_inline_types.h
+++ b/platform/linux-generic/include/odp/api/plat/queue_inline_types.h
@@ -12,6 +12,7 @@ extern "C" {
#endif
#include <stdint.h>
+#include <odp/api/spec/queue_types.h>
/** @cond _ODP_HIDE_FROM_DOXYGEN_ */
@@ -26,6 +27,31 @@ typedef struct _odp_queue_inline_offset_t {
} _odp_queue_inline_offset_t;
+/* Queue API functions */
+typedef struct {
+ odp_queue_t (*queue_create)(const char *name,
+ const odp_queue_param_t *param);
+ int (*queue_destroy)(odp_queue_t queue);
+ odp_queue_t (*queue_lookup)(const char *name);
+ int (*queue_capability)(odp_queue_capability_t *capa);
+ int (*queue_context_set)(odp_queue_t queue, void *context,
+ uint32_t len);
+ int (*queue_enq)(odp_queue_t queue, odp_event_t ev);
+ int (*queue_enq_multi)(odp_queue_t queue, const odp_event_t events[],
+ int num);
+ odp_event_t (*queue_deq)(odp_queue_t queue);
+ int (*queue_deq_multi)(odp_queue_t queue, odp_event_t events[],
+ int num);
+ odp_queue_type_t (*queue_type)(odp_queue_t queue);
+ odp_schedule_sync_t (*queue_sched_type)(odp_queue_t queue);
+ odp_schedule_prio_t (*queue_sched_prio)(odp_queue_t queue);
+ odp_schedule_group_t (*queue_sched_group)(odp_queue_t queue);
+ uint32_t (*queue_lock_count)(odp_queue_t queue);
+ uint64_t (*queue_to_u64)(odp_queue_t queue);
+ void (*queue_param_init)(odp_queue_param_t *param);
+ int (*queue_info)(odp_queue_t queue, odp_queue_info_t *info);
+} _odp_queue_api_fn_t;
+
/** @endcond */
#ifdef __cplusplus
diff --git a/platform/linux-generic/include/odp/api/plat/queue_inlines.h b/platform/linux-generic/include/odp/api/plat/queue_inlines.h
index f802c96e..c557b4ba 100644
--- a/platform/linux-generic/include/odp/api/plat/queue_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/queue_inlines.h
@@ -12,11 +12,16 @@
/** @cond _ODP_HIDE_FROM_DOXYGEN_ */
extern _odp_queue_inline_offset_t _odp_queue_inline_offset;
+extern const _odp_queue_api_fn_t *_odp_queue_api;
#ifndef _ODP_NO_INLINE
/* Inline functions by default */
#define _ODP_INLINE static inline
- #define odp_queue_context __odp_queue_context
+ #define odp_queue_context __odp_queue_context
+ #define odp_queue_enq __odp_queue_enq
+ #define odp_queue_enq_multi __odp_queue_enq_multi
+ #define odp_queue_deq __odp_queue_deq
+ #define odp_queue_deq_multi __odp_queue_deq_multi
#else
#define _ODP_INLINE
#endif
@@ -31,6 +36,28 @@ _ODP_INLINE void *odp_queue_context(odp_queue_t handle)
return context;
}
+_ODP_INLINE int odp_queue_enq(odp_queue_t queue, odp_event_t ev)
+{
+ return _odp_queue_api->queue_enq(queue, ev);
+}
+
+_ODP_INLINE int odp_queue_enq_multi(odp_queue_t queue,
+ const odp_event_t events[], int num)
+{
+ return _odp_queue_api->queue_enq_multi(queue, events, num);
+}
+
+_ODP_INLINE odp_event_t odp_queue_deq(odp_queue_t queue)
+{
+ return _odp_queue_api->queue_deq(queue);
+}
+
+_ODP_INLINE int odp_queue_deq_multi(odp_queue_t queue,
+ odp_event_t events[], int num)
+{
+ return _odp_queue_api->queue_deq_multi(queue, events, num);
+}
+
/** @endcond */
#endif
diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h
index 48e74dd2..c6e1345d 100644
--- a/platform/linux-generic/include/odp_buffer_internal.h
+++ b/platform/linux-generic/include/odp_buffer_internal.h
@@ -120,11 +120,6 @@ static inline odp_buffer_t buf_from_buf_hdr(odp_buffer_hdr_t *hdr)
return (odp_buffer_t)hdr;
}
-static inline odp_event_t event_from_buf_hdr(odp_buffer_hdr_t *hdr)
-{
- return (odp_event_t)hdr;
-}
-
#ifdef __cplusplus
}
#endif
diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h
index cd2db2ec..0dccc97d 100644
--- a/platform/linux-generic/include/odp_packet_internal.h
+++ b/platform/linux-generic/include/odp_packet_internal.h
@@ -105,7 +105,7 @@ typedef struct {
odp_time_t timestamp;
/* Classifier destination queue */
- void *dst_queue;
+ odp_queue_t dst_queue;
/* Result for crypto packet op */
odp_crypto_packet_result_t crypto_op_result;
@@ -308,6 +308,8 @@ int _odp_packet_cmp_data(odp_packet_t pkt, uint32_t offset,
int _odp_packet_ipv4_chksum_insert(odp_packet_t pkt);
int _odp_packet_tcp_chksum_insert(odp_packet_t pkt);
int _odp_packet_udp_chksum_insert(odp_packet_t pkt);
+int _odp_packet_sctp_chksum_insert(odp_packet_t pkt);
+
#ifdef __cplusplus
}
diff --git a/platform/linux-generic/include/odp_packet_io_internal.h b/platform/linux-generic/include/odp_packet_io_internal.h
index d2446d19..08085078 100644
--- a/platform/linux-generic/include/odp_packet_io_internal.h
+++ b/platform/linux-generic/include/odp_packet_io_internal.h
@@ -105,7 +105,6 @@ struct pktio_entry {
struct {
odp_queue_t queue;
- void *queue_int;
odp_pktin_queue_t pktin;
} in_queue[PKTIO_MAX_QUEUES];
diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h
index e3de2b65..e8a04614 100644
--- a/platform/linux-generic/include/odp_pool_internal.h
+++ b/platform/linux-generic/include/odp_pool_internal.h
@@ -96,7 +96,7 @@ static inline pool_t *pool_entry(uint32_t pool_idx)
static inline pool_t *pool_entry_from_hdl(odp_pool_t pool_hdl)
{
- return &pool_tbl->pool[_odp_typeval(pool_hdl)];
+ return &pool_tbl->pool[_odp_typeval(pool_hdl) - 1];
}
static inline odp_buffer_hdr_t *buf_hdl_to_hdr(odp_buffer_t buf)
diff --git a/platform/linux-generic/include/odp_queue_basic_internal.h b/platform/linux-generic/include/odp_queue_basic_internal.h
index 654b9e31..46b74795 100644
--- a/platform/linux-generic/include/odp_queue_basic_internal.h
+++ b/platform/linux-generic/include/odp_queue_basic_internal.h
@@ -22,6 +22,7 @@ extern "C" {
#include <odp/api/hints.h>
#include <odp/api/ticketlock.h>
#include <odp_config_internal.h>
+#include <odp_ring_mpmc_internal.h>
#include <odp_ring_st_internal.h>
#include <odp_ring_spsc_internal.h>
#include <odp_queue_lf.h>
@@ -33,21 +34,29 @@ extern "C" {
#define QUEUE_STATUS_SCHED 4
struct queue_entry_s {
- odp_ticketlock_t ODP_ALIGNED_CACHE lock;
- union {
- ring_st_t ring_st;
- ring_spsc_t ring_spsc;
- };
- int status;
-
+ /* The first cache line is read only */
queue_enq_fn_t ODP_ALIGNED_CACHE enqueue;
queue_deq_fn_t dequeue;
queue_enq_multi_fn_t enqueue_multi;
queue_deq_multi_fn_t dequeue_multi;
+ uint32_t *ring_data;
+ uint32_t ring_mask;
+ uint32_t index;
+ odp_queue_t handle;
+ odp_queue_type_t type;
- uint32_t index;
- odp_queue_t handle;
- odp_queue_type_t type;
+ /* MPMC ring (2 cache lines). */
+ ring_mpmc_t ring_mpmc;
+
+ odp_ticketlock_t lock;
+ union {
+ ring_st_t ring_st;
+ ring_spsc_t ring_spsc;
+ };
+
+ int status;
+
+ queue_deq_multi_fn_t orig_dequeue_multi;
odp_queue_param_t param;
odp_pktin_queue_t pktin;
odp_pktout_queue_t pktout;
@@ -96,6 +105,11 @@ static inline odp_queue_t queue_from_index(uint32_t queue_id)
return (odp_queue_t)qentry_from_index(queue_id);
}
+static inline queue_entry_t *qentry_from_handle(odp_queue_t handle)
+{
+ return (queue_entry_t *)(uintptr_t)handle;
+}
+
void queue_spsc_init(queue_entry_t *queue, uint32_t queue_size);
/* Functions for schedulers */
diff --git a/platform/linux-generic/include/odp_queue_if.h b/platform/linux-generic/include/odp_queue_if.h
index 2eaf8771..6ec7e24c 100644
--- a/platform/linux-generic/include/odp_queue_if.h
+++ b/platform/linux-generic/include/odp_queue_if.h
@@ -18,48 +18,23 @@ extern "C" {
#define QUEUE_MULTI_MAX CONFIG_BURST_SIZE
-/* Queue API functions */
-typedef struct {
- odp_queue_t (*queue_create)(const char *name,
- const odp_queue_param_t *param);
- int (*queue_destroy)(odp_queue_t queue);
- odp_queue_t (*queue_lookup)(const char *name);
- int (*queue_capability)(odp_queue_capability_t *capa);
- int (*queue_context_set)(odp_queue_t queue, void *context,
- uint32_t len);
- int (*queue_enq)(odp_queue_t queue, odp_event_t ev);
- int (*queue_enq_multi)(odp_queue_t queue, const odp_event_t events[],
- int num);
- odp_event_t (*queue_deq)(odp_queue_t queue);
- int (*queue_deq_multi)(odp_queue_t queue, odp_event_t events[],
- int num);
- odp_queue_type_t (*queue_type)(odp_queue_t queue);
- odp_schedule_sync_t (*queue_sched_type)(odp_queue_t queue);
- odp_schedule_prio_t (*queue_sched_prio)(odp_queue_t queue);
- odp_schedule_group_t (*queue_sched_group)(odp_queue_t queue);
- uint32_t (*queue_lock_count)(odp_queue_t queue);
- uint64_t (*queue_to_u64)(odp_queue_t hdl);
- void (*queue_param_init)(odp_queue_param_t *param);
- int (*queue_info)(odp_queue_t queue, odp_queue_info_t *info);
-} queue_api_t;
-
typedef int (*queue_init_global_fn_t)(void);
typedef int (*queue_term_global_fn_t)(void);
typedef int (*queue_init_local_fn_t)(void);
typedef int (*queue_term_local_fn_t)(void);
-typedef void *(*queue_from_ext_fn_t)(odp_queue_t handle);
-typedef odp_queue_t (*queue_to_ext_fn_t)(void *q_int);
-typedef int (*queue_enq_fn_t)(void *q_int, odp_buffer_hdr_t *);
-typedef int (*queue_enq_multi_fn_t)(void *q_int, odp_buffer_hdr_t **, int);
-typedef odp_buffer_hdr_t *(*queue_deq_fn_t)(void *q_int);
-typedef int (*queue_deq_multi_fn_t)(void *q_int, odp_buffer_hdr_t **, int);
-typedef odp_pktout_queue_t (*queue_get_pktout_fn_t)(void *q_int);
-typedef void (*queue_set_pktout_fn_t)(void *q_int, odp_pktio_t pktio,
+typedef int (*queue_enq_fn_t)(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr);
+typedef int (*queue_enq_multi_fn_t)(odp_queue_t queue,
+ odp_buffer_hdr_t **buf_hdr, int num);
+typedef odp_buffer_hdr_t *(*queue_deq_fn_t)(odp_queue_t queue);
+typedef int (*queue_deq_multi_fn_t)(odp_queue_t queue,
+ odp_buffer_hdr_t **buf_hdr, int num);
+typedef odp_pktout_queue_t (*queue_get_pktout_fn_t)(odp_queue_t queue);
+typedef void (*queue_set_pktout_fn_t)(odp_queue_t queue, odp_pktio_t pktio,
int index);
-typedef odp_pktin_queue_t (*queue_get_pktin_fn_t)(void *q_int);
-typedef void (*queue_set_pktin_fn_t)(void *q_int, odp_pktio_t pktio,
+typedef odp_pktin_queue_t (*queue_get_pktin_fn_t)(odp_queue_t queue);
+typedef void (*queue_set_pktin_fn_t)(odp_queue_t queue, odp_pktio_t pktio,
int index);
-typedef void (*queue_set_enq_deq_fn_t)(void *q_int,
+typedef void (*queue_set_enq_deq_fn_t)(odp_queue_t queue,
queue_enq_fn_t enq,
queue_enq_multi_fn_t enq_multi,
queue_deq_fn_t deq,
@@ -71,17 +46,15 @@ typedef struct {
queue_term_global_fn_t term_global;
queue_init_local_fn_t init_local;
queue_term_local_fn_t term_local;
- queue_from_ext_fn_t from_ext;
- queue_to_ext_fn_t to_ext;
- queue_enq_fn_t enq;
- queue_enq_multi_fn_t enq_multi;
- queue_deq_fn_t deq;
- queue_deq_multi_fn_t deq_multi;
queue_get_pktout_fn_t get_pktout;
queue_set_pktout_fn_t set_pktout;
queue_get_pktin_fn_t get_pktin;
queue_set_pktin_fn_t set_pktin;
queue_set_enq_deq_fn_t set_enq_deq_fn;
+
+ /* Original queue dequeue multi function (before override). May be used
+ * by an overriding dequeue function. */
+ queue_deq_multi_fn_t orig_deq_multi;
} queue_fn_t;
extern const queue_fn_t *queue_fn;
diff --git a/platform/linux-generic/include/odp_queue_scalable_internal.h b/platform/linux-generic/include/odp_queue_scalable_internal.h
index 71aaa3ba..05932a36 100644
--- a/platform/linux-generic/include/odp_queue_scalable_internal.h
+++ b/platform/linux-generic/include/odp_queue_scalable_internal.h
@@ -41,6 +41,7 @@ struct queue_entry_s {
queue_deq_fn_t dequeue;
queue_enq_multi_fn_t enqueue_multi;
queue_deq_multi_fn_t dequeue_multi;
+ queue_deq_multi_fn_t orig_dequeue_multi;
uint32_t index;
odp_queue_t handle;
@@ -80,14 +81,14 @@ static inline uint32_t queue_to_id(odp_queue_t handle)
return qentry_from_ext(handle)->s.index;
}
-static inline queue_entry_t *qentry_from_int(void *handle)
+static inline queue_entry_t *qentry_from_int(odp_queue_t handle)
{
- return (queue_entry_t *)handle;
+ return (queue_entry_t *)(uintptr_t)handle;
}
-static inline void *qentry_to_int(queue_entry_t *qentry)
+static inline odp_queue_t qentry_to_int(queue_entry_t *qentry)
{
- return qentry;
+ return (odp_queue_t)qentry;
}
static inline odp_queue_t queue_get_handle(queue_entry_t *queue)
diff --git a/platform/linux-generic/include/odp_ring_mpmc_internal.h b/platform/linux-generic/include/odp_ring_mpmc_internal.h
new file mode 100644
index 00000000..74bbb8fc
--- /dev/null
+++ b/platform/linux-generic/include/odp_ring_mpmc_internal.h
@@ -0,0 +1,169 @@
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef ODP_RING_MPMC_INTERNAL_H_
+#define ODP_RING_MPMC_INTERNAL_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp/api/atomic.h>
+#include <odp/api/cpu.h>
+#include <odp/api/hints.h>
+#include <odp_align_internal.h>
+#include <odp/api/plat/atomic_inlines.h>
+#include <odp/api/plat/cpu_inlines.h>
+
+/* Ring of uint32_t data
+ *
+ * Ring stores head and tail counters. Ring indexes are formed from these
+ * counters with a mask (mask = ring_size - 1), which requires that ring size
+ * must be a power of two.
+ *
+ * 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
+ * +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
+ * | E | E | | | | | | | | | | E | E | E | E | E |
+ * +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
+ * ^ ^ ^ ^
+ * | | | |
+ * r_tail r_head w_tail w_head
+ *
+ */
+typedef struct {
+ odp_atomic_u32_t ODP_ALIGNED_CACHE r_head;
+ odp_atomic_u32_t r_tail;
+
+ odp_atomic_u32_t ODP_ALIGNED_CACHE w_head;
+ odp_atomic_u32_t w_tail;
+
+} ring_mpmc_t;
+
+static inline int ring_mpmc_cas_u32(odp_atomic_u32_t *atom,
+ uint32_t *old_val, uint32_t new_val)
+{
+ return __atomic_compare_exchange_n(&atom->v, old_val, new_val,
+ 0 /* strong */,
+ __ATOMIC_RELAXED,
+ __ATOMIC_RELAXED);
+}
+
+/* Initialize ring */
+static inline void ring_mpmc_init(ring_mpmc_t *ring)
+{
+ odp_atomic_init_u32(&ring->w_head, 0);
+ odp_atomic_init_u32(&ring->w_tail, 0);
+ odp_atomic_init_u32(&ring->r_head, 0);
+ odp_atomic_init_u32(&ring->r_tail, 0);
+}
+
+/* Dequeue data from the ring head. Num is smaller than ring size. */
+static inline uint32_t ring_mpmc_deq_multi(ring_mpmc_t *ring,
+ uint32_t *ring_data,
+ uint32_t ring_mask,
+ uint32_t data[],
+ uint32_t num)
+{
+ uint32_t old_head, new_head, w_tail, num_data, i;
+
+ /* Load acquires ensure that w_tail load happens after r_head load,
+ * and thus r_head value is always behind or equal to w_tail value.
+ * When CAS operation succeeds, this thread owns data between old
+ * and new r_head. */
+ do {
+ old_head = odp_atomic_load_acq_u32(&ring->r_head);
+ odp_prefetch(&ring_data[(old_head + 1) & ring_mask]);
+ w_tail = odp_atomic_load_acq_u32(&ring->w_tail);
+ num_data = w_tail - old_head;
+
+ /* Ring is empty */
+ if (num_data == 0)
+ return 0;
+
+ /* Try to take all available */
+ if (num > num_data)
+ num = num_data;
+
+ new_head = old_head + num;
+
+ } while (odp_unlikely(ring_mpmc_cas_u32(&ring->r_head, &old_head,
+ new_head) == 0));
+
+ /* Read data. This will not move above load acquire of r_head. */
+ for (i = 0; i < num; i++)
+ data[i] = ring_data[(old_head + 1 + i) & ring_mask];
+
+ /* Wait until other readers have updated the tail */
+ while (odp_unlikely(odp_atomic_load_u32(&ring->r_tail) != old_head))
+ odp_cpu_pause();
+
+ /* Release the new reader tail, writers acquire it. */
+ odp_atomic_store_rel_u32(&ring->r_tail, new_head);
+
+ return num;
+}
+
+/* Enqueue multiple data into the ring tail. Num is smaller than ring size. */
+static inline uint32_t ring_mpmc_enq_multi(ring_mpmc_t *ring,
+ uint32_t *ring_data,
+ uint32_t ring_mask,
+ const uint32_t data[],
+ uint32_t num)
+{
+ uint32_t old_head, new_head, r_tail, num_free, i;
+ uint32_t size = ring_mask + 1;
+
+ /* Load acquires ensure that w_head load happens after r_tail load,
+ * and thus r_tail value is always behind or equal to w_head value.
+ * When CAS operation succeeds, this thread owns data between old
+ * and new w_head. */
+ do {
+ r_tail = odp_atomic_load_acq_u32(&ring->r_tail);
+ old_head = odp_atomic_load_acq_u32(&ring->w_head);
+
+ num_free = size - (old_head - r_tail);
+
+ /* Ring is full */
+ if (num_free == 0)
+ return 0;
+
+ /* Try to use all available */
+ if (num > num_free)
+ num = num_free;
+
+ new_head = old_head + num;
+
+ } while (odp_unlikely(ring_mpmc_cas_u32(&ring->w_head, &old_head,
+ new_head) == 0));
+
+ /* Write data. This will not move above load acquire of w_head. */
+ for (i = 0; i < num; i++)
+ ring_data[(old_head + 1 + i) & ring_mask] = data[i];
+
+ /* Wait until other writers have updated the tail */
+ while (odp_unlikely(odp_atomic_load_u32(&ring->w_tail) != old_head))
+ odp_cpu_pause();
+
+ /* Release the new writer tail, readers acquire it. */
+ odp_atomic_store_rel_u32(&ring->w_tail, new_head);
+
+ return num;
+}
+
+/* Check if ring is empty */
+static inline int ring_mpmc_is_empty(ring_mpmc_t *ring)
+{
+ uint32_t head = odp_atomic_load_u32(&ring->r_head);
+ uint32_t tail = odp_atomic_load_u32(&ring->w_tail);
+
+ return head == tail;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/include/odp_ring_spsc_internal.h b/platform/linux-generic/include/odp_ring_spsc_internal.h
index e38bda1d..de122bf5 100644
--- a/platform/linux-generic/include/odp_ring_spsc_internal.h
+++ b/platform/linux-generic/include/odp_ring_spsc_internal.h
@@ -29,31 +29,27 @@ extern "C" {
typedef struct {
odp_atomic_u32_t head;
odp_atomic_u32_t tail;
- uint32_t mask;
- uint32_t *data;
} ring_spsc_t;
/* Initialize ring. Ring size must be a power of two. */
-static inline void ring_spsc_init(ring_spsc_t *ring, uint32_t *data,
- uint32_t size)
+static inline void ring_spsc_init(ring_spsc_t *ring)
{
odp_atomic_init_u32(&ring->head, 0);
odp_atomic_init_u32(&ring->tail, 0);
- ring->mask = size - 1;
- ring->data = data;
}
/* Dequeue data from the ring head. Max_num is smaller than ring size.*/
-static inline uint32_t ring_spsc_deq_multi(ring_spsc_t *ring, uint32_t data[],
+static inline uint32_t ring_spsc_deq_multi(ring_spsc_t *ring,
+ uint32_t *ring_data,
+ uint32_t ring_mask, uint32_t data[],
uint32_t max_num)
{
- uint32_t head, tail, mask, idx;
+ uint32_t head, tail, idx;
uint32_t num, i;
tail = odp_atomic_load_acq_u32(&ring->tail);
head = odp_atomic_load_u32(&ring->head);
- mask = ring->mask;
num = tail - head;
/* Empty */
@@ -63,11 +59,11 @@ static inline uint32_t ring_spsc_deq_multi(ring_spsc_t *ring, uint32_t data[],
if (num > max_num)
num = max_num;
- idx = head & mask;
+ idx = head & ring_mask;
for (i = 0; i < num; i++) {
- data[i] = ring->data[idx];
- idx = (idx + 1) & mask;
+ data[i] = ring_data[idx];
+ idx = (idx + 1) & ring_mask;
}
odp_atomic_store_rel_u32(&ring->head, head + num);
@@ -77,16 +73,17 @@ static inline uint32_t ring_spsc_deq_multi(ring_spsc_t *ring, uint32_t data[],
/* Enqueue data into the ring tail. Num_data is smaller than ring size. */
static inline uint32_t ring_spsc_enq_multi(ring_spsc_t *ring,
+ uint32_t *ring_data,
+ uint32_t ring_mask,
const uint32_t data[],
uint32_t num_data)
{
- uint32_t head, tail, mask, size, idx;
+ uint32_t head, tail, size, idx;
uint32_t num, i;
head = odp_atomic_load_acq_u32(&ring->head);
tail = odp_atomic_load_u32(&ring->tail);
- mask = ring->mask;
- size = mask + 1;
+ size = ring_mask + 1;
num = size - (tail - head);
/* Full */
@@ -96,11 +93,11 @@ static inline uint32_t ring_spsc_enq_multi(ring_spsc_t *ring,
if (num > num_data)
num = num_data;
- idx = tail & mask;
+ idx = tail & ring_mask;
for (i = 0; i < num; i++) {
- ring->data[idx] = data[i];
- idx = (idx + 1) & mask;
+ ring_data[idx] = data[i];
+ idx = (idx + 1) & ring_mask;
}
odp_atomic_store_rel_u32(&ring->tail, tail + num);
diff --git a/platform/linux-generic/include/odp_ring_st_internal.h b/platform/linux-generic/include/odp_ring_st_internal.h
index 5fb37d4e..1bc18cda 100644
--- a/platform/linux-generic/include/odp_ring_st_internal.h
+++ b/platform/linux-generic/include/odp_ring_st_internal.h
@@ -19,30 +19,25 @@ extern "C" {
typedef struct {
uint32_t head;
uint32_t tail;
- uint32_t mask;
- uint32_t *data;
-
} ring_st_t;
/* Initialize ring. Ring size must be a power of two. */
-static inline void ring_st_init(ring_st_t *ring, uint32_t *data, uint32_t size)
+static inline void ring_st_init(ring_st_t *ring)
{
ring->head = 0;
ring->tail = 0;
- ring->mask = size - 1;
- ring->data = data;
}
/* Dequeue data from the ring head. Max_num is smaller than ring size.*/
-static inline uint32_t ring_st_deq_multi(ring_st_t *ring, uint32_t data[],
+static inline uint32_t ring_st_deq_multi(ring_st_t *ring, uint32_t *ring_data,
+ uint32_t ring_mask, uint32_t data[],
uint32_t max_num)
{
- uint32_t head, tail, mask, idx;
+ uint32_t head, tail, idx;
uint32_t num, i;
head = ring->head;
tail = ring->tail;
- mask = ring->mask;
num = tail - head;
/* Empty */
@@ -52,11 +47,11 @@ static inline uint32_t ring_st_deq_multi(ring_st_t *ring, uint32_t data[],
if (num > max_num)
num = max_num;
- idx = head & mask;
+ idx = head & ring_mask;
for (i = 0; i < num; i++) {
- data[i] = ring->data[idx];
- idx = (idx + 1) & mask;
+ data[i] = ring_data[idx];
+ idx = (idx + 1) & ring_mask;
}
ring->head = head + num;
@@ -65,16 +60,17 @@ static inline uint32_t ring_st_deq_multi(ring_st_t *ring, uint32_t data[],
}
/* Enqueue data into the ring tail. Num_data is smaller than ring size. */
-static inline uint32_t ring_st_enq_multi(ring_st_t *ring, const uint32_t data[],
+static inline uint32_t ring_st_enq_multi(ring_st_t *ring, uint32_t *ring_data,
+ uint32_t ring_mask,
+ const uint32_t data[],
uint32_t num_data)
{
- uint32_t head, tail, mask, size, idx;
+ uint32_t head, tail, size, idx;
uint32_t num, i;
head = ring->head;
tail = ring->tail;
- mask = ring->mask;
- size = mask + 1;
+ size = ring_mask + 1;
num = size - (tail - head);
/* Full */
@@ -84,11 +80,11 @@ static inline uint32_t ring_st_enq_multi(ring_st_t *ring, const uint32_t data[],
if (num > num_data)
num = num_data;
- idx = tail & mask;
+ idx = tail & ring_mask;
for (i = 0; i < num; i++) {
- ring->data[idx] = data[i];
- idx = (idx + 1) & mask;
+ ring_data[idx] = data[i];
+ idx = (idx + 1) & ring_mask;
}
ring->tail = tail + num;
diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h
index 8f082aaa..30481773 100644
--- a/platform/linux-generic/include/odp_schedule_if.h
+++ b/platform/linux-generic/include/odp_schedule_if.h
@@ -32,7 +32,7 @@ typedef int (*schedule_init_queue_fn_t)(uint32_t queue_index,
typedef void (*schedule_destroy_queue_fn_t)(uint32_t queue_index);
typedef int (*schedule_sched_queue_fn_t)(uint32_t queue_index);
typedef int (*schedule_unsched_queue_fn_t)(uint32_t queue_index);
-typedef int (*schedule_ord_enq_multi_fn_t)(void *q_int,
+typedef int (*schedule_ord_enq_multi_fn_t)(odp_queue_t queue,
void *buf_hdr[], int num, int *ret);
typedef int (*schedule_init_global_fn_t)(void);
typedef int (*schedule_term_global_fn_t)(void);
diff --git a/platform/linux-generic/include/odp_traffic_mngr_internal.h b/platform/linux-generic/include/odp_traffic_mngr_internal.h
index 919831a3..770a6490 100644
--- a/platform/linux-generic/include/odp_traffic_mngr_internal.h
+++ b/platform/linux-generic/include/odp_traffic_mngr_internal.h
@@ -282,7 +282,7 @@ struct tm_queue_obj_s {
uint8_t tm_idx;
uint8_t delayed_cnt;
uint8_t blocked_cnt;
- void *tm_qentry;
+ odp_queue_t queue;
};
struct tm_node_obj_s {
diff --git a/platform/linux-generic/include/protocols/sctp.h b/platform/linux-generic/include/protocols/sctp.h
new file mode 100644
index 00000000..dea8a8a6
--- /dev/null
+++ b/platform/linux-generic/include/protocols/sctp.h
@@ -0,0 +1,49 @@
+/* Copyright (c) 2016-2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODP SCTP header
+ */
+
+#ifndef ODP_SCTP_H_
+#define ODP_SCTP_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp_api.h>
+
+/** @addtogroup odp_header ODP HEADER
+ * @{
+ */
+
+/** SCTP header length */
+#define _ODP_SCTPHDR_LEN 12
+
+/** SCTP header */
+typedef struct ODP_PACKED {
+ odp_u16be_t src_port; /**< Source port */
+ odp_u16be_t dst_port; /**< Destination port */
+ odp_u32be_t tag; /**< Verification tag */
+ odp_u32be_t chksum; /**< SCTP header and data checksum */
+} _odp_sctphdr_t;
+
+/** @internal Compile time assert */
+ODP_STATIC_ASSERT(sizeof(_odp_sctphdr_t) == _ODP_SCTPHDR_LEN,
+ "_ODP_SCTPHDR_T__SIZE_ERROR");
+
+/**
+ * @}
+ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/odp_classification.c b/platform/linux-generic/odp_classification.c
index 6e493bcd..d02fb507 100644
--- a/platform/linux-generic/odp_classification.c
+++ b/platform/linux-generic/odp_classification.c
@@ -45,16 +45,36 @@ static const rss_key default_rss = {
}
};
+static inline uint32_t _odp_cos_to_ndx(odp_cos_t cos)
+{
+ return _odp_typeval(cos) - 1;
+}
+
+static inline odp_cos_t _odp_cos_from_ndx(uint32_t ndx)
+{
+ return _odp_cast_scalar(odp_cos_t, ndx + 1);
+}
+
+static inline uint32_t _odp_pmr_to_ndx(odp_pmr_t pmr)
+{
+ return _odp_typeval(pmr) - 1;
+}
+
+static inline odp_pmr_t _odp_pmr_from_ndx(uint32_t ndx)
+{
+ return _odp_cast_scalar(odp_pmr_t, ndx + 1);
+}
+
static
-cos_t *get_cos_entry_internal(odp_cos_t cos_id)
+cos_t *get_cos_entry_internal(odp_cos_t cos)
{
- return &cos_tbl->cos_entry[_odp_typeval(cos_id)];
+ return &cos_tbl->cos_entry[_odp_cos_to_ndx(cos)];
}
static
-pmr_t *get_pmr_entry_internal(odp_pmr_t pmr_id)
+pmr_t *get_pmr_entry_internal(odp_pmr_t pmr)
{
- return &pmr_tbl->pmr[_odp_typeval(pmr_id)];
+ return &pmr_tbl->pmr[_odp_pmr_to_ndx(pmr)];
}
int odp_classification_init_global(void)
@@ -80,8 +100,7 @@ int odp_classification_init_global(void)
memset(cos_tbl, 0, sizeof(cos_tbl_t));
for (i = 0; i < CLS_COS_MAX_ENTRY; i++) {
/* init locks */
- cos_t *cos =
- get_cos_entry_internal(_odp_cast_scalar(odp_cos_t, i));
+ cos_t *cos = get_cos_entry_internal(_odp_cos_from_ndx(i));
LOCK_INIT(&cos->s.lock);
}
@@ -101,8 +120,7 @@ int odp_classification_init_global(void)
memset(pmr_tbl, 0, sizeof(pmr_tbl_t));
for (i = 0; i < CLS_PMR_MAX_ENTRY; i++) {
/* init locks */
- pmr_t *pmr =
- get_pmr_entry_internal(_odp_cast_scalar(odp_pmr_t, i));
+ pmr_t *pmr = get_pmr_entry_internal(_odp_pmr_from_ndx(i));
LOCK_INIT(&pmr->s.lock);
}
@@ -283,7 +301,7 @@ odp_cos_t odp_cls_cos_create(const char *name, odp_cls_cos_param_t *param)
odp_atomic_init_u32(&cos->s.num_rule, 0);
cos->s.index = i;
UNLOCK(&cos->s.lock);
- return _odp_cast_scalar(odp_cos_t, i);
+ return _odp_cos_from_ndx(i);
}
UNLOCK(&cos->s.lock);
}
@@ -308,7 +326,7 @@ odp_pmr_t alloc_pmr(pmr_t **pmr)
pmr_tbl->pmr[i].s.num_pmr = 0;
*pmr = &pmr_tbl->pmr[i];
/* return as locked */
- return _odp_cast_scalar(odp_pmr_t, i);
+ return _odp_pmr_from_ndx(i);
}
UNLOCK(&pmr_tbl->pmr[i].s.lock);
}
@@ -317,25 +335,27 @@ odp_pmr_t alloc_pmr(pmr_t **pmr)
}
static
-cos_t *get_cos_entry(odp_cos_t cos_id)
+cos_t *get_cos_entry(odp_cos_t cos)
{
- if (_odp_typeval(cos_id) >= CLS_COS_MAX_ENTRY ||
- cos_id == ODP_COS_INVALID)
+ uint32_t cos_id = _odp_cos_to_ndx(cos);
+
+ if (cos_id >= CLS_COS_MAX_ENTRY || cos == ODP_COS_INVALID)
return NULL;
- if (cos_tbl->cos_entry[_odp_typeval(cos_id)].s.valid == 0)
+ if (cos_tbl->cos_entry[cos_id].s.valid == 0)
return NULL;
- return &cos_tbl->cos_entry[_odp_typeval(cos_id)];
+ return &cos_tbl->cos_entry[cos_id];
}
static
-pmr_t *get_pmr_entry(odp_pmr_t pmr_id)
+pmr_t *get_pmr_entry(odp_pmr_t pmr)
{
- if (_odp_typeval(pmr_id) >= CLS_PMR_MAX_ENTRY ||
- pmr_id == ODP_PMR_INVAL)
+ uint32_t pmr_id = _odp_pmr_to_ndx(pmr);
+
+ if (pmr_id >= CLS_PMR_MAX_ENTRY || pmr == ODP_PMR_INVAL)
return NULL;
- if (pmr_tbl->pmr[_odp_typeval(pmr_id)].s.valid == 0)
+ if (pmr_tbl->pmr[pmr_id].s.valid == 0)
return NULL;
- return &pmr_tbl->pmr[_odp_typeval(pmr_id)];
+ return &pmr_tbl->pmr[pmr_id];
}
int odp_cos_destroy(odp_cos_t cos_id)
@@ -1016,7 +1036,7 @@ int cls_classify_packet(pktio_entry_t *entry, const uint8_t *base,
pkt_hdr->p.input_flags.dst_queue = 1;
if (!cos->s.queue_group) {
- pkt_hdr->dst_queue = queue_fn->from_ext(cos->s.queue);
+ pkt_hdr->dst_queue = cos->s.queue;
return 0;
}
@@ -1025,8 +1045,7 @@ int cls_classify_packet(pktio_entry_t *entry, const uint8_t *base,
hash = hash & (CLS_COS_QUEUE_MAX - 1);
tbl_index = (cos->s.index * CLS_COS_QUEUE_MAX) + (hash %
cos->s.num_queue);
- pkt_hdr->dst_queue = queue_fn->from_ext(queue_grp_tbl->
- s.queue[tbl_index]);
+ pkt_hdr->dst_queue = queue_grp_tbl->s.queue[tbl_index];
return 0;
}
diff --git a/platform/linux-generic/odp_crypto_null.c b/platform/linux-generic/odp_crypto_null.c
index 7d01f10e..13aae9bc 100644
--- a/platform/linux-generic/odp_crypto_null.c
+++ b/platform/linux-generic/odp_crypto_null.c
@@ -20,6 +20,7 @@
#include <odp/api/plat/packet_inlines.h>
#include <odp/api/plat/thread_inlines.h>
#include <odp_packet_internal.h>
+#include <odp/api/plat/queue_inlines.h>
/* Inlined API functions */
#include <odp/api/plat/event_inlines.h>
diff --git a/platform/linux-generic/odp_crypto_openssl.c b/platform/linux-generic/odp_crypto_openssl.c
index ecab6bd0..d98c87b0 100644
--- a/platform/linux-generic/odp_crypto_openssl.c
+++ b/platform/linux-generic/odp_crypto_openssl.c
@@ -20,6 +20,7 @@
#include <odp/api/plat/packet_inlines.h>
#include <odp/api/plat/thread_inlines.h>
#include <odp_packet_internal.h>
+#include <odp/api/plat/queue_inlines.h>
/* Inlined API functions */
#include <odp/api/plat/event_inlines.h>
diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c
index 5b739cc1..1bc549c2 100644
--- a/platform/linux-generic/odp_ipsec.c
+++ b/platform/linux-generic/odp_ipsec.c
@@ -18,6 +18,7 @@
#include <odp_debug_internal.h>
#include <odp_packet_internal.h>
#include <odp_ipsec_internal.h>
+#include <odp/api/plat/queue_inlines.h>
#include <protocols/eth.h>
#include <protocols/ip.h>
@@ -1305,7 +1306,8 @@ static void ipsec_out_ah_post(ipsec_state_t *state, odp_packet_t pkt)
static void ipsec_out_checksums(odp_packet_t pkt,
ipsec_state_t *state)
{
- odp_bool_t ipv4_chksum_pkt, udp_chksum_pkt, tcp_chksum_pkt;
+ odp_bool_t ipv4_chksum_pkt, udp_chksum_pkt, tcp_chksum_pkt,
+ sctp_chksum_pkt;
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
odp_ipsec_outbound_config_t outbound = ipsec_config.outbound;
@@ -1324,6 +1326,12 @@ static void ipsec_out_checksums(odp_packet_t pkt,
pkt_hdr->p.flags.l4_chksum_set,
pkt_hdr->p.flags.l4_chksum);
+ sctp_chksum_pkt = OL_TX_CHKSUM_PKT(outbound.chksum.inner_sctp,
+ state->ip_next_hdr ==
+ _ODP_IPPROTO_SCTP,
+ pkt_hdr->p.flags.l4_chksum_set,
+ pkt_hdr->p.flags.l4_chksum);
+
if (ipv4_chksum_pkt)
_odp_packet_ipv4_chksum_insert(pkt);
@@ -1332,6 +1340,9 @@ static void ipsec_out_checksums(odp_packet_t pkt,
if (udp_chksum_pkt)
_odp_packet_udp_chksum_insert(pkt);
+
+ if (sctp_chksum_pkt)
+ _odp_packet_sctp_chksum_insert(pkt);
}
static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt,
@@ -1745,7 +1756,7 @@ int _odp_ipsec_try_inline(odp_packet_t *pkt)
pkt_hdr = packet_hdr(*pkt);
pkt_hdr->p.input_flags.dst_queue = 1;
- pkt_hdr->dst_queue = queue_fn->from_ext(ipsec_sa->queue);
+ pkt_hdr->dst_queue = ipsec_sa->queue;
/* Last thing */
_odp_ipsec_sa_unuse(ipsec_sa);
diff --git a/platform/linux-generic/odp_ipsec_events.c b/platform/linux-generic/odp_ipsec_events.c
index e3977649..ea9ce961 100644
--- a/platform/linux-generic/odp_ipsec_events.c
+++ b/platform/linux-generic/odp_ipsec_events.c
@@ -17,6 +17,7 @@
/* Inlined API functions */
#include <odp/api/plat/event_inlines.h>
+#include <odp/api/plat/queue_inlines.h>
typedef struct {
/* common buffer header */
diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c
index 3b1686b8..11f37fd8 100644
--- a/platform/linux-generic/odp_ipsec_sad.c
+++ b/platform/linux-generic/odp_ipsec_sad.c
@@ -38,12 +38,12 @@ static inline ipsec_sa_t *ipsec_sa_entry(uint32_t ipsec_sa_idx)
static inline ipsec_sa_t *ipsec_sa_entry_from_hdl(odp_ipsec_sa_t ipsec_sa_hdl)
{
- return ipsec_sa_entry(_odp_typeval(ipsec_sa_hdl));
+ return ipsec_sa_entry(_odp_typeval(ipsec_sa_hdl) - 1);
}
static inline odp_ipsec_sa_t ipsec_sa_index_to_handle(uint32_t ipsec_sa_idx)
{
- return _odp_cast_scalar(odp_ipsec_sa_t, ipsec_sa_idx);
+ return _odp_cast_scalar(odp_ipsec_sa_t, ipsec_sa_idx + 1);
}
int _odp_ipsec_sad_init_global(void)
diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c
index 11f8bd43..ae63ef4a 100644
--- a/platform/linux-generic/odp_packet.c
+++ b/platform/linux-generic/odp_packet.c
@@ -22,6 +22,7 @@
#include <protocols/eth.h>
#include <protocols/ip.h>
+#include <protocols/sctp.h>
#include <protocols/tcp.h>
#include <protocols/udp.h>
@@ -2016,6 +2017,31 @@ static uint16_t packet_sum_ones_comp16(odp_packet_hdr_t *pkt_hdr,
return sum;
}
+static uint32_t packet_sum_crc32c(odp_packet_hdr_t *pkt_hdr,
+ uint32_t offset,
+ uint32_t len,
+ uint32_t init_val)
+{
+ uint32_t sum = init_val;
+
+ if (offset + len > pkt_hdr->frame_len)
+ return sum;
+
+ while (len > 0) {
+ uint32_t seglen = 0; /* GCC */
+ void *mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL);
+
+ if (seglen > len)
+ seglen = len;
+
+ sum = odp_hash_crc32c(mapaddr, seglen, sum);
+ len -= seglen;
+ offset += seglen;
+ }
+
+ return sum;
+}
+
/** Parser helper function for Ethernet packets */
static inline uint16_t parse_eth(packet_parser_t *prs, const uint8_t **parseptr,
uint32_t *offset, uint32_t frame_len)
@@ -2298,6 +2324,34 @@ static inline void parse_udp(packet_parser_t *prs, const uint8_t **parseptr,
*parseptr += sizeof(_odp_udphdr_t);
}
+/**
+ * Parser helper function for SCTP
+ */
+static inline void parse_sctp(packet_parser_t *prs, const uint8_t **parseptr,
+ uint16_t sctp_len,
+ odp_proto_chksums_t chksums,
+ uint32_t *l4_part_sum)
+{
+ if (odp_unlikely(sctp_len < sizeof(_odp_sctphdr_t))) {
+ prs->flags.sctp_err = 1;
+ return;
+ }
+
+ if (chksums.chksum.sctp &&
+ !prs->input_flags.ipfrag) {
+ const _odp_sctphdr_t *sctp =
+ (const _odp_sctphdr_t *)*parseptr;
+ uint32_t crc = ~0;
+ uint32_t zero = 0;
+
+ crc = odp_hash_crc32c(sctp, sizeof(*sctp) - 4, crc);
+ crc = odp_hash_crc32c(&zero, 4, crc);
+ *l4_part_sum = crc;
+ }
+
+ *parseptr += sizeof(_odp_sctphdr_t);
+}
+
static inline
int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr,
uint32_t offset,
@@ -2388,6 +2442,8 @@ int packet_parse_common_l3_l4(packet_parser_t *prs, const uint8_t *parseptr,
case _ODP_IPPROTO_SCTP:
prs->input_flags.sctp = 1;
+ parse_sctp(prs, &parseptr, frame_len - prs->l4_offset, chksums,
+ l4_part_sum);
break;
case _ODP_IPPROTO_NO_NEXT:
@@ -2586,6 +2642,30 @@ int _odp_packet_udp_chksum_insert(odp_packet_t pkt)
return _odp_packet_tcp_udp_chksum_insert(pkt, _ODP_IPPROTO_UDP);
}
+/**
+ * Calculate and fill in SCTP checksum
+ *
+ * @param pkt ODP packet
+ *
+ * @retval 0 on success
+ * @retval <0 on failure
+ */
+int _odp_packet_sctp_chksum_insert(odp_packet_t pkt)
+{
+ odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+ uint32_t sum;
+
+ if (pkt_hdr->p.l4_offset == ODP_PACKET_OFFSET_INVALID)
+ return -1;
+
+ sum = 0;
+ odp_packet_copy_from_mem(pkt, pkt_hdr->p.l4_offset + 8, 4, &sum);
+ sum = ~packet_sum_crc32c(pkt_hdr, pkt_hdr->p.l4_offset,
+ pkt_hdr->frame_len - pkt_hdr->p.l4_offset,
+ ~0);
+ return odp_packet_copy_from_mem(pkt, pkt_hdr->p.l4_offset + 8, 4, &sum);
+}
+
static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
odp_proto_chksums_t chksums,
uint32_t l4_part_sum)
@@ -2626,6 +2706,29 @@ static int packet_l4_chksum(odp_packet_hdr_t *pkt_hdr,
}
}
+ if (chksums.chksum.sctp &&
+ pkt_hdr->p.input_flags.sctp &&
+ !pkt_hdr->p.input_flags.ipfrag) {
+ uint32_t sum = ~packet_sum_crc32c(pkt_hdr,
+ pkt_hdr->p.l4_offset +
+ _ODP_SCTPHDR_LEN,
+ pkt_hdr->frame_len -
+ pkt_hdr->p.l4_offset -
+ _ODP_SCTPHDR_LEN,
+ l4_part_sum);
+ _odp_sctphdr_t *sctp = packet_map(pkt_hdr,
+ pkt_hdr->p.l4_offset,
+ NULL, NULL);
+
+ pkt_hdr->p.input_flags.l4_chksum_done = 1;
+ if (sum != sctp->chksum) {
+ pkt_hdr->p.flags.l4_chksum_err = 1;
+ pkt_hdr->p.flags.sctp_err = 1;
+ ODP_DBG("SCTP chksum fail (%x/%x)!\n", sum,
+ sctp->chksum);
+ }
+ }
+
return pkt_hdr->p.flags.all_flags != 0;
}
diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c
index 7759f83e..6f99bf10 100644
--- a/platform/linux-generic/odp_packet_io.c
+++ b/platform/linux-generic/odp_packet_io.c
@@ -28,6 +28,7 @@
#include <odp/api/time.h>
#include <odp/api/plat/time_inlines.h>
#include <odp_pcapng.h>
+#include <odp/api/plat/queue_inlines.h>
#include <string.h>
#include <inttypes.h>
@@ -137,7 +138,6 @@ static void init_in_queues(pktio_entry_t *entry)
for (i = 0; i < PKTIO_MAX_QUEUES; i++) {
entry->s.in_queue[i].queue = ODP_QUEUE_INVALID;
- entry->s.in_queue[i].queue_int = NULL;
entry->s.in_queue[i].pktin = PKTIN_INVALID;
}
}
@@ -327,7 +327,6 @@ static void destroy_in_queues(pktio_entry_t *entry, int num)
if (entry->s.in_queue[i].queue != ODP_QUEUE_INVALID) {
odp_queue_destroy(entry->s.in_queue[i].queue);
entry->s.in_queue[i].queue = ODP_QUEUE_INVALID;
- entry->s.in_queue[i].queue_int = NULL;
}
}
}
@@ -627,7 +626,8 @@ static inline int pktin_recv_buf(pktio_entry_t *entry, int pktin_index,
if (pkt_hdr->p.input_flags.dst_queue) {
int ret;
- ret = queue_fn->enq(pkt_hdr->dst_queue, buf_hdr);
+ ret = odp_queue_enq(pkt_hdr->dst_queue,
+ odp_packet_to_event(pkt));
if (ret < 0)
odp_packet_free(pkt);
continue;
@@ -637,47 +637,47 @@ static inline int pktin_recv_buf(pktio_entry_t *entry, int pktin_index,
return num_rx;
}
-static int pktout_enqueue(void *q_int, odp_buffer_hdr_t *buf_hdr)
+static int pktout_enqueue(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr)
{
odp_packet_t pkt = packet_from_buf_hdr(buf_hdr);
int len = 1;
int nbr;
- if (sched_fn->ord_enq_multi(q_int, (void **)buf_hdr, len, &nbr))
+ if (sched_fn->ord_enq_multi(queue, (void **)buf_hdr, len, &nbr))
return (nbr == len ? 0 : -1);
- nbr = odp_pktout_send(queue_fn->get_pktout(q_int), &pkt, len);
+ nbr = odp_pktout_send(queue_fn->get_pktout(queue), &pkt, len);
return (nbr == len ? 0 : -1);
}
-static int pktout_enq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[], int num)
+static int pktout_enq_multi(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr[],
+ int num)
{
odp_packet_t pkt_tbl[QUEUE_MULTI_MAX];
int nbr;
int i;
- if (sched_fn->ord_enq_multi(q_int, (void **)buf_hdr, num, &nbr))
+ if (sched_fn->ord_enq_multi(queue, (void **)buf_hdr, num, &nbr))
return nbr;
for (i = 0; i < num; ++i)
pkt_tbl[i] = packet_from_buf_hdr(buf_hdr[i]);
- nbr = odp_pktout_send(queue_fn->get_pktout(q_int), pkt_tbl, num);
+ nbr = odp_pktout_send(queue_fn->get_pktout(queue), pkt_tbl, num);
return nbr;
}
-static odp_buffer_hdr_t *pktin_dequeue(void *q_int)
+static odp_buffer_hdr_t *pktin_dequeue(odp_queue_t queue)
{
odp_buffer_hdr_t *buf_hdr;
odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX];
int pkts;
- odp_pktin_queue_t pktin_queue = queue_fn->get_pktin(q_int);
+ odp_pktin_queue_t pktin_queue = queue_fn->get_pktin(queue);
odp_pktio_t pktio = pktin_queue.pktio;
int pktin_index = pktin_queue.index;
pktio_entry_t *entry = get_pktio_entry(pktio);
- buf_hdr = queue_fn->deq(q_int);
- if (buf_hdr != NULL)
+ if (queue_fn->orig_deq_multi(queue, &buf_hdr, 1) == 1)
return buf_hdr;
pkts = pktin_recv_buf(entry, pktin_index, hdr_tbl, QUEUE_MULTI_MAX);
@@ -689,7 +689,8 @@ static odp_buffer_hdr_t *pktin_dequeue(void *q_int)
int num_enq;
int num = pkts - 1;
- num_enq = queue_fn->enq_multi(q_int, &hdr_tbl[1], num);
+ num_enq = odp_queue_enq_multi(queue,
+ (odp_event_t *)&hdr_tbl[1], num);
if (odp_unlikely(num_enq < num)) {
if (odp_unlikely(num_enq < 0))
@@ -705,17 +706,18 @@ static odp_buffer_hdr_t *pktin_dequeue(void *q_int)
return buf_hdr;
}
-static int pktin_deq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[], int num)
+static int pktin_deq_multi(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr[],
+ int num)
{
int nbr;
odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX];
int pkts, i, j;
- odp_pktin_queue_t pktin_queue = queue_fn->get_pktin(q_int);
+ odp_pktin_queue_t pktin_queue = queue_fn->get_pktin(queue);
odp_pktio_t pktio = pktin_queue.pktio;
int pktin_index = pktin_queue.index;
pktio_entry_t *entry = get_pktio_entry(pktio);
- nbr = queue_fn->deq_multi(q_int, buf_hdr, num);
+ nbr = queue_fn->orig_deq_multi(queue, buf_hdr, num);
if (odp_unlikely(nbr > num))
ODP_ABORT("queue_deq_multi req: %d, returned %d\n", num, nbr);
@@ -740,7 +742,7 @@ static int pktin_deq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[], int num)
if (j) {
int num_enq;
- num_enq = queue_fn->enq_multi(q_int, hdr_tbl, j);
+ num_enq = odp_queue_enq_multi(queue, (odp_event_t *)hdr_tbl, j);
if (odp_unlikely(num_enq < j)) {
if (odp_unlikely(num_enq < 0))
@@ -765,7 +767,7 @@ int sched_cb_pktin_poll_one(int pktio_index,
odp_packet_hdr_t *pkt_hdr;
odp_buffer_hdr_t *buf_hdr;
odp_packet_t packets[QUEUE_MULTI_MAX];
- void *q_int;
+ odp_queue_t queue;
if (odp_unlikely(entry->s.state != PKTIO_STATE_STARTED)) {
if (entry->s.state < PKTIO_STATE_ACTIVE ||
@@ -785,9 +787,14 @@ int sched_cb_pktin_poll_one(int pktio_index,
pkt = packets[i];
pkt_hdr = packet_hdr(pkt);
if (odp_unlikely(pkt_hdr->p.input_flags.dst_queue)) {
- q_int = pkt_hdr->dst_queue;
+ int num_enq;
+
+ queue = pkt_hdr->dst_queue;
buf_hdr = packet_to_buf_hdr(pkt);
- if (queue_fn->enq_multi(q_int, &buf_hdr, 1) < 0) {
+ num_enq = odp_queue_enq_multi(queue,
+ (odp_event_t *)&buf_hdr,
+ 1);
+ if (num_enq < 0) {
/* Queue full? */
odp_packet_free(pkt);
__atomic_fetch_add(&entry->s.stats.in_discards,
@@ -836,7 +843,7 @@ int sched_cb_pktin_poll_old(int pktio_index, int num_queue, int index[])
}
for (idx = 0; idx < num_queue; idx++) {
- void *q_int;
+ odp_queue_t queue;
int num_enq;
num = pktin_recv_buf(entry, index[idx], hdr_tbl,
@@ -850,8 +857,9 @@ int sched_cb_pktin_poll_old(int pktio_index, int num_queue, int index[])
return -1;
}
- q_int = entry->s.in_queue[index[idx]].queue_int;
- num_enq = queue_fn->enq_multi(q_int, hdr_tbl, num);
+ queue = entry->s.in_queue[index[idx]].queue;
+ num_enq = odp_queue_enq_multi(queue,
+ (odp_event_t *)hdr_tbl, num);
if (odp_unlikely(num_enq < num)) {
if (odp_unlikely(num_enq < 0))
@@ -1371,35 +1379,6 @@ int odp_pktio_stats_reset(odp_pktio_t pktio)
return ret;
}
-static int abort_pktin_enqueue(void *q_int ODP_UNUSED,
- odp_buffer_hdr_t *buf_hdr ODP_UNUSED)
-{
- ODP_ABORT("attempted enqueue to a pktin queue");
- return -1;
-}
-
-static int abort_pktin_enq_multi(void *q_int ODP_UNUSED,
- odp_buffer_hdr_t *buf_hdr[] ODP_UNUSED,
- int num ODP_UNUSED)
-{
- ODP_ABORT("attempted enqueue to a pktin queue");
- return 0;
-}
-
-static odp_buffer_hdr_t *abort_pktout_dequeue(void *q_int ODP_UNUSED)
-{
- ODP_ABORT("attempted dequeue from a pktout queue");
- return NULL;
-}
-
-static int abort_pktout_deq_multi(void *q_int ODP_UNUSED,
- odp_buffer_hdr_t *buf_hdr[] ODP_UNUSED,
- int num ODP_UNUSED)
-{
- ODP_ABORT("attempted dequeue from a pktout queue");
- return 0;
-}
-
int odp_pktin_queue_config(odp_pktio_t pktio,
const odp_pktin_queue_param_t *param)
{
@@ -1410,7 +1389,6 @@ int odp_pktin_queue_config(odp_pktio_t pktio,
unsigned i;
int rc;
odp_queue_t queue;
- void *q_int;
odp_pktin_queue_param_t default_param;
if (param == NULL) {
@@ -1490,23 +1468,19 @@ int odp_pktin_queue_config(odp_pktio_t pktio,
return -1;
}
- q_int = queue_fn->from_ext(queue);
-
if (mode == ODP_PKTIN_MODE_QUEUE) {
- queue_fn->set_pktin(q_int, pktio, i);
- queue_fn->set_enq_deq_fn(q_int,
- abort_pktin_enqueue,
- abort_pktin_enq_multi,
+ queue_fn->set_pktin(queue, pktio, i);
+ queue_fn->set_enq_deq_fn(queue,
+ NULL,
+ NULL,
pktin_dequeue,
pktin_deq_multi);
}
entry->s.in_queue[i].queue = queue;
- entry->s.in_queue[i].queue_int = q_int;
} else {
entry->s.in_queue[i].queue = ODP_QUEUE_INVALID;
- entry->s.in_queue[i].queue_int = NULL;
}
entry->s.in_queue[i].pktin.index = i;
@@ -1598,7 +1572,6 @@ int odp_pktout_queue_config(odp_pktio_t pktio,
for (i = 0; i < num_queues; i++) {
odp_queue_t queue;
odp_queue_param_t queue_param;
- void *q_int;
char name[ODP_QUEUE_NAME_LEN];
int pktio_id = odp_pktio_index(pktio);
@@ -1618,15 +1591,14 @@ int odp_pktout_queue_config(odp_pktio_t pktio,
return -1;
}
- q_int = queue_fn->from_ext(queue);
- queue_fn->set_pktout(q_int, pktio, i);
+ queue_fn->set_pktout(queue, pktio, i);
/* Override default enqueue / dequeue functions */
- queue_fn->set_enq_deq_fn(q_int,
+ queue_fn->set_enq_deq_fn(queue,
pktout_enqueue,
pktout_enq_multi,
- abort_pktout_dequeue,
- abort_pktout_deq_multi);
+ NULL,
+ NULL);
entry->s.out_queue[i].queue = queue;
}
diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c
index 01ee9234..7f5dfa8b 100644
--- a/platform/linux-generic/odp_pool.c
+++ b/platform/linux-generic/odp_pool.c
@@ -70,7 +70,7 @@ const _odp_pool_inline_offset_t ODP_ALIGNED_CACHE _odp_pool_inline = {
static inline odp_pool_t pool_index_to_handle(uint32_t pool_idx)
{
- return _odp_cast_scalar(odp_pool_t, pool_idx);
+ return _odp_cast_scalar(odp_pool_t, pool_idx + 1);
}
static inline pool_t *pool_from_buf(odp_buffer_t buf)
diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c
index 89eed3c0..7e8b7e34 100644
--- a/platform/linux-generic/odp_queue_basic.c
+++ b/platform/linux-generic/odp_queue_basic.c
@@ -47,11 +47,6 @@ static int queue_init(queue_entry_t *queue, const char *name,
queue_global_t *queue_glb;
extern _odp_queue_inline_offset_t _odp_queue_inline_offset;
-static inline queue_entry_t *qentry_from_handle(odp_queue_t handle)
-{
- return (queue_entry_t *)(uintptr_t)handle;
-}
-
static int queue_capa(odp_queue_capability_t *capa, int sched)
{
memset(capa, 0, sizeof(odp_queue_capability_t));
@@ -310,9 +305,9 @@ static odp_queue_t queue_create(const char *name,
if (!queue->s.spsc &&
param->nonblocking == ODP_NONBLOCKING_LF) {
- queue_lf_func_t *lf_func;
+ queue_lf_func_t *lf_fn;
- lf_func = &queue_glb->queue_lf_func;
+ lf_fn = &queue_glb->queue_lf_func;
queue_lf = queue_lf_create(queue);
@@ -322,10 +317,11 @@ static odp_queue_t queue_create(const char *name,
}
queue->s.queue_lf = queue_lf;
- queue->s.enqueue = lf_func->enq;
- queue->s.enqueue_multi = lf_func->enq_multi;
- queue->s.dequeue = lf_func->deq;
- queue->s.dequeue_multi = lf_func->deq_multi;
+ queue->s.enqueue = lf_fn->enq;
+ queue->s.enqueue_multi = lf_fn->enq_multi;
+ queue->s.dequeue = lf_fn->deq;
+ queue->s.dequeue_multi = lf_fn->deq_multi;
+ queue->s.orig_dequeue_multi = lf_fn->deq_multi;
}
type = queue->s.type;
@@ -404,8 +400,10 @@ static int queue_destroy(odp_queue_t handle)
if (queue->s.spsc)
empty = ring_spsc_is_empty(&queue->s.ring_spsc);
- else
+ else if (queue->s.type == ODP_QUEUE_TYPE_SCHED)
empty = ring_st_is_empty(&queue->s.ring_st);
+ else
+ empty = ring_mpmc_is_empty(&queue->s.ring_mpmc);
if (!empty) {
UNLOCK(queue);
@@ -489,137 +487,80 @@ static inline void buffer_index_to_buf(odp_buffer_hdr_t *buf_hdr[],
}
}
-static inline int enq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[],
- int num)
+static inline int _plain_queue_enq_multi(odp_queue_t handle,
+ odp_buffer_hdr_t *buf_hdr[], int num)
{
- int sched = 0;
- int ret;
queue_entry_t *queue;
- int num_enq;
- ring_st_t *ring_st;
+ int ret, num_enq;
+ ring_mpmc_t *ring_mpmc;
uint32_t buf_idx[num];
- queue = q_int;
- ring_st = &queue->s.ring_st;
+ queue = qentry_from_handle(handle);
+ ring_mpmc = &queue->s.ring_mpmc;
- if (sched_fn->ord_enq_multi(q_int, (void **)buf_hdr, num, &ret))
+ if (sched_fn->ord_enq_multi(handle, (void **)buf_hdr, num, &ret))
return ret;
buffer_index_from_buf(buf_idx, buf_hdr, num);
- LOCK(queue);
-
- if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) {
- UNLOCK(queue);
- ODP_ERR("Bad queue status\n");
- return -1;
- }
-
- num_enq = ring_st_enq_multi(ring_st, buf_idx, num);
-
- if (odp_unlikely(num_enq == 0)) {
- UNLOCK(queue);
- return 0;
- }
-
- if (queue->s.status == QUEUE_STATUS_NOTSCHED) {
- queue->s.status = QUEUE_STATUS_SCHED;
- sched = 1;
- }
-
- UNLOCK(queue);
-
- /* Add queue to scheduling */
- if (sched && sched_fn->sched_queue(queue->s.index))
- ODP_ABORT("schedule_queue failed\n");
+ num_enq = ring_mpmc_enq_multi(ring_mpmc, queue->s.ring_data,
+ queue->s.ring_mask, buf_idx, num);
return num_enq;
}
-static int queue_int_enq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[],
- int num)
-{
- return enq_multi(q_int, buf_hdr, num);
-}
-
-static int queue_int_enq(void *q_int, odp_buffer_hdr_t *buf_hdr)
+static inline int _plain_queue_deq_multi(odp_queue_t handle,
+ odp_buffer_hdr_t *buf_hdr[], int num)
{
- int ret;
-
- ret = enq_multi(q_int, &buf_hdr, 1);
+ int num_deq;
+ queue_entry_t *queue;
+ ring_mpmc_t *ring_mpmc;
+ uint32_t buf_idx[num];
- if (ret == 1)
- return 0;
- else
- return -1;
-}
+ queue = qentry_from_handle(handle);
+ ring_mpmc = &queue->s.ring_mpmc;
-static int queue_enq_multi(odp_queue_t handle, const odp_event_t ev[], int num)
-{
- queue_entry_t *queue = qentry_from_handle(handle);
+ num_deq = ring_mpmc_deq_multi(ring_mpmc, queue->s.ring_data,
+ queue->s.ring_mask, buf_idx, num);
- if (odp_unlikely(num == 0))
+ if (num_deq == 0)
return 0;
- if (num > QUEUE_MULTI_MAX)
- num = QUEUE_MULTI_MAX;
+ buffer_index_to_buf(buf_hdr, buf_idx, num_deq);
- return queue->s.enqueue_multi(queue,
- (odp_buffer_hdr_t **)(uintptr_t)ev, num);
+ return num_deq;
}
-static int queue_enq(odp_queue_t handle, odp_event_t ev)
+static int plain_queue_enq_multi(odp_queue_t handle,
+ odp_buffer_hdr_t *buf_hdr[], int num)
{
- queue_entry_t *queue = qentry_from_handle(handle);
-
- return queue->s.enqueue(queue,
- (odp_buffer_hdr_t *)(uintptr_t)ev);
+ return _plain_queue_enq_multi(handle, buf_hdr, num);
}
-static inline int plain_queue_deq(queue_entry_t *queue,
- odp_buffer_hdr_t *buf_hdr[], int num)
+static int plain_queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr)
{
- int num_deq;
- ring_st_t *ring_st;
- uint32_t buf_idx[num];
-
- ring_st = &queue->s.ring_st;
-
- LOCK(queue);
-
- if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) {
- /* Bad queue, or queue has been destroyed. */
- UNLOCK(queue);
- return -1;
- }
+ int ret;
- num_deq = ring_st_deq_multi(ring_st, buf_idx, num);
+ ret = _plain_queue_enq_multi(handle, &buf_hdr, 1);
- UNLOCK(queue);
-
- if (num_deq == 0)
+ if (ret == 1)
return 0;
-
- buffer_index_to_buf(buf_hdr, buf_idx, num_deq);
-
- return num_deq;
+ else
+ return -1;
}
-static int queue_int_deq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[],
- int num)
+static int plain_queue_deq_multi(odp_queue_t handle,
+ odp_buffer_hdr_t *buf_hdr[], int num)
{
- queue_entry_t *queue = q_int;
-
- return plain_queue_deq(queue, buf_hdr, num);
+ return _plain_queue_deq_multi(handle, buf_hdr, num);
}
-static odp_buffer_hdr_t *queue_int_deq(void *q_int)
+static odp_buffer_hdr_t *plain_queue_deq(odp_queue_t handle)
{
- queue_entry_t *queue = q_int;
odp_buffer_hdr_t *buf_hdr = NULL;
int ret;
- ret = plain_queue_deq(queue, &buf_hdr, 1);
+ ret = _plain_queue_deq_multi(handle, &buf_hdr, 1);
if (ret == 1)
return buf_hdr;
@@ -627,87 +568,46 @@ static odp_buffer_hdr_t *queue_int_deq(void *q_int)
return NULL;
}
-static int queue_deq_multi(odp_queue_t handle, odp_event_t ev[], int num)
+static int error_enqueue(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr)
{
- queue_entry_t *queue = qentry_from_handle(handle);
+ (void)buf_hdr;
- if (num > QUEUE_MULTI_MAX)
- num = QUEUE_MULTI_MAX;
+ ODP_ERR("Enqueue not supported (" PRIu64 ")\n",
+ odp_queue_to_u64(handle));
- return queue->s.dequeue_multi(queue,
- (odp_buffer_hdr_t **)ev, num);
+ return -1;
}
-static odp_event_t queue_deq(odp_queue_t handle)
+static int error_enqueue_multi(odp_queue_t handle,
+ odp_buffer_hdr_t *buf_hdr[], int num)
{
- queue_entry_t *queue = qentry_from_handle(handle);
+ (void)buf_hdr;
+ (void)num;
+
+ ODP_ERR("Enqueue multi not supported (" PRIu64 ")\n",
+ odp_queue_to_u64(handle));
- return (odp_event_t)queue->s.dequeue(queue);
+ return -1;
}
-static int queue_init(queue_entry_t *queue, const char *name,
- const odp_queue_param_t *param)
+static odp_buffer_hdr_t *error_dequeue(odp_queue_t handle)
{
- uint64_t offset;
- uint32_t queue_size;
- int spsc;
-
- if (name == NULL) {
- queue->s.name[0] = 0;
- } else {
- strncpy(queue->s.name, name, ODP_QUEUE_NAME_LEN - 1);
- queue->s.name[ODP_QUEUE_NAME_LEN - 1] = 0;
- }
- memcpy(&queue->s.param, param, sizeof(odp_queue_param_t));
- if (queue->s.param.sched.lock_count > sched_fn->max_ordered_locks())
- return -1;
+ ODP_ERR("Dequeue not supported (" PRIu64 ")\n",
+ odp_queue_to_u64(handle));
- if (param->type == ODP_QUEUE_TYPE_SCHED)
- queue->s.param.deq_mode = ODP_QUEUE_OP_DISABLED;
-
- queue->s.type = queue->s.param.type;
-
- queue->s.pktin = PKTIN_INVALID;
- queue->s.pktout = PKTOUT_INVALID;
-
- /* Use default size for all small queues to quarantee performance
- * level. */
- queue_size = queue_glb->config.default_queue_size;
- if (param->size > queue_glb->config.default_queue_size)
- queue_size = param->size;
-
- /* Round up if not already a power of two */
- queue_size = ROUNDUP_POWER2_U32(queue_size);
-
- if (queue_size > queue_glb->config.max_queue_size) {
- ODP_ERR("Too large queue size %u\n", queue_size);
- return -1;
- }
-
- offset = queue->s.index * (uint64_t)queue_glb->config.max_queue_size;
-
- /* Single-producer / single-consumer plain queue has simple and
- * lock-free implementation */
- spsc = (param->type == ODP_QUEUE_TYPE_PLAIN) &&
- (param->enq_mode == ODP_QUEUE_OP_MT_UNSAFE) &&
- (param->deq_mode == ODP_QUEUE_OP_MT_UNSAFE);
-
- queue->s.spsc = spsc;
- queue->s.queue_lf = NULL;
+ return NULL;
+}
- if (spsc) {
- queue_spsc_init(queue, queue_size);
- } else {
- queue->s.enqueue = queue_int_enq;
- queue->s.dequeue = queue_int_deq;
- queue->s.enqueue_multi = queue_int_enq_multi;
- queue->s.dequeue_multi = queue_int_deq_multi;
+static int error_dequeue_multi(odp_queue_t handle,
+ odp_buffer_hdr_t *buf_hdr[], int num)
+{
+ (void)buf_hdr;
+ (void)num;
- ring_st_init(&queue->s.ring_st, &queue_glb->ring_data[offset],
- queue_size);
- }
+ ODP_ERR("Dequeue multi not supported (" PRIu64 ")\n",
+ odp_queue_to_u64(handle));
- return 0;
+ return -1;
}
static void queue_param_init(odp_queue_param_t *params)
@@ -761,6 +661,54 @@ static int queue_info(odp_queue_t handle, odp_queue_info_t *info)
return 0;
}
+static inline int _sched_queue_enq_multi(odp_queue_t handle,
+ odp_buffer_hdr_t *buf_hdr[], int num)
+{
+ int sched = 0;
+ int ret;
+ queue_entry_t *queue;
+ int num_enq;
+ ring_st_t *ring_st;
+ uint32_t buf_idx[num];
+
+ queue = qentry_from_handle(handle);
+ ring_st = &queue->s.ring_st;
+
+ if (sched_fn->ord_enq_multi(handle, (void **)buf_hdr, num, &ret))
+ return ret;
+
+ buffer_index_from_buf(buf_idx, buf_hdr, num);
+
+ LOCK(queue);
+
+ if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) {
+ UNLOCK(queue);
+ ODP_ERR("Bad queue status\n");
+ return -1;
+ }
+
+ num_enq = ring_st_enq_multi(ring_st, queue->s.ring_data,
+ queue->s.ring_mask, buf_idx, num);
+
+ if (odp_unlikely(num_enq == 0)) {
+ UNLOCK(queue);
+ return 0;
+ }
+
+ if (queue->s.status == QUEUE_STATUS_NOTSCHED) {
+ queue->s.status = QUEUE_STATUS_SCHED;
+ sched = 1;
+ }
+
+ UNLOCK(queue);
+
+ /* Add queue to scheduling */
+ if (sched && sched_fn->sched_queue(queue->s.index))
+ ODP_ABORT("schedule_queue failed\n");
+
+ return num_enq;
+}
+
int sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num,
int update_status)
{
@@ -781,7 +729,8 @@ int sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num,
return -1;
}
- num_deq = ring_st_deq_multi(ring_st, buf_idx, max_num);
+ num_deq = ring_st_deq_multi(ring_st, queue->s.ring_data,
+ queue->s.ring_mask, buf_idx, max_num);
if (num_deq == 0) {
/* Already empty queue */
@@ -807,6 +756,24 @@ int sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num,
return num_deq;
}
+static int sched_queue_enq_multi(odp_queue_t handle,
+ odp_buffer_hdr_t *buf_hdr[], int num)
+{
+ return _sched_queue_enq_multi(handle, buf_hdr, num);
+}
+
+static int sched_queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr)
+{
+ int ret;
+
+ ret = _sched_queue_enq_multi(handle, &buf_hdr, 1);
+
+ if (ret == 1)
+ return 0;
+ else
+ return -1;
+}
+
int sched_queue_empty(uint32_t queue_index)
{
queue_entry_t *queue = qentry_from_index(queue_index);
@@ -833,48 +800,135 @@ int sched_queue_empty(uint32_t queue_index)
return ret;
}
+static int queue_init(queue_entry_t *queue, const char *name,
+ const odp_queue_param_t *param)
+{
+ uint64_t offset;
+ uint32_t queue_size;
+ odp_queue_type_t queue_type;
+ int spsc;
+
+ queue_type = param->type;
+
+ if (name == NULL) {
+ queue->s.name[0] = 0;
+ } else {
+ strncpy(queue->s.name, name, ODP_QUEUE_NAME_LEN - 1);
+ queue->s.name[ODP_QUEUE_NAME_LEN - 1] = 0;
+ }
+ memcpy(&queue->s.param, param, sizeof(odp_queue_param_t));
+ if (queue->s.param.sched.lock_count > sched_fn->max_ordered_locks())
+ return -1;
+
+ if (queue_type == ODP_QUEUE_TYPE_SCHED)
+ queue->s.param.deq_mode = ODP_QUEUE_OP_DISABLED;
+
+ queue->s.type = queue_type;
+
+ queue->s.pktin = PKTIN_INVALID;
+ queue->s.pktout = PKTOUT_INVALID;
+
+ /* Use default size for all small queues to quarantee performance
+ * level. */
+ queue_size = queue_glb->config.default_queue_size;
+ if (param->size > queue_glb->config.default_queue_size)
+ queue_size = param->size;
+
+ /* Round up if not already a power of two */
+ queue_size = ROUNDUP_POWER2_U32(queue_size);
+
+ if (queue_size > queue_glb->config.max_queue_size) {
+ ODP_ERR("Too large queue size %u\n", queue_size);
+ return -1;
+ }
+
+ offset = queue->s.index * (uint64_t)queue_glb->config.max_queue_size;
+
+ /* Single-producer / single-consumer plain queue has simple and
+ * lock-free implementation */
+ spsc = (queue_type == ODP_QUEUE_TYPE_PLAIN) &&
+ (param->enq_mode == ODP_QUEUE_OP_MT_UNSAFE) &&
+ (param->deq_mode == ODP_QUEUE_OP_MT_UNSAFE);
+
+ queue->s.spsc = spsc;
+ queue->s.queue_lf = NULL;
+
+ /* Default to error functions */
+ queue->s.enqueue = error_enqueue;
+ queue->s.enqueue_multi = error_enqueue_multi;
+ queue->s.dequeue = error_dequeue;
+ queue->s.dequeue_multi = error_dequeue_multi;
+ queue->s.orig_dequeue_multi = error_dequeue_multi;
+
+ if (spsc) {
+ queue_spsc_init(queue, queue_size);
+ } else {
+ if (queue_type == ODP_QUEUE_TYPE_PLAIN) {
+ queue->s.enqueue = plain_queue_enq;
+ queue->s.enqueue_multi = plain_queue_enq_multi;
+ queue->s.dequeue = plain_queue_deq;
+ queue->s.dequeue_multi = plain_queue_deq_multi;
+ queue->s.orig_dequeue_multi = plain_queue_deq_multi;
+
+ queue->s.ring_data = &queue_glb->ring_data[offset];
+ queue->s.ring_mask = queue_size - 1;
+ ring_mpmc_init(&queue->s.ring_mpmc);
+
+ } else {
+ queue->s.enqueue = sched_queue_enq;
+ queue->s.enqueue_multi = sched_queue_enq_multi;
+
+ queue->s.ring_data = &queue_glb->ring_data[offset];
+ queue->s.ring_mask = queue_size - 1;
+ ring_st_init(&queue->s.ring_st);
+ }
+ }
+
+ return 0;
+}
+
static uint64_t queue_to_u64(odp_queue_t hdl)
{
return _odp_pri(hdl);
}
-static odp_pktout_queue_t queue_get_pktout(void *q_int)
+static odp_pktout_queue_t queue_get_pktout(odp_queue_t handle)
{
- queue_entry_t *qentry = q_int;
+ queue_entry_t *qentry = qentry_from_handle(handle);
return qentry->s.pktout;
}
-static void queue_set_pktout(void *q_int, odp_pktio_t pktio, int index)
+static void queue_set_pktout(odp_queue_t handle, odp_pktio_t pktio, int index)
{
- queue_entry_t *qentry = q_int;
+ queue_entry_t *qentry = qentry_from_handle(handle);
qentry->s.pktout.pktio = pktio;
qentry->s.pktout.index = index;
}
-static odp_pktin_queue_t queue_get_pktin(void *q_int)
+static odp_pktin_queue_t queue_get_pktin(odp_queue_t handle)
{
- queue_entry_t *qentry = q_int;
+ queue_entry_t *qentry = qentry_from_handle(handle);
return qentry->s.pktin;
}
-static void queue_set_pktin(void *q_int, odp_pktio_t pktio, int index)
+static void queue_set_pktin(odp_queue_t handle, odp_pktio_t pktio, int index)
{
- queue_entry_t *qentry = q_int;
+ queue_entry_t *qentry = qentry_from_handle(handle);
qentry->s.pktin.pktio = pktio;
qentry->s.pktin.index = index;
}
-static void queue_set_enq_deq_func(void *q_int,
+static void queue_set_enq_deq_func(odp_queue_t handle,
queue_enq_fn_t enq,
queue_enq_multi_fn_t enq_multi,
queue_deq_fn_t deq,
queue_deq_multi_fn_t deq_multi)
{
- queue_entry_t *qentry = q_int;
+ queue_entry_t *qentry = qentry_from_handle(handle);
if (enq)
qentry->s.enqueue = enq;
@@ -889,29 +943,66 @@ static void queue_set_enq_deq_func(void *q_int,
qentry->s.dequeue_multi = deq_multi;
}
-static void *queue_from_ext(odp_queue_t handle)
+static int queue_orig_multi(odp_queue_t handle,
+ odp_buffer_hdr_t **buf_hdr, int num)
+{
+ queue_entry_t *queue = qentry_from_handle(handle);
+
+ return queue->s.orig_dequeue_multi(handle, buf_hdr, num);
+}
+
+static int queue_api_enq_multi(odp_queue_t handle,
+ const odp_event_t ev[], int num)
+{
+ queue_entry_t *queue = qentry_from_handle(handle);
+
+ if (odp_unlikely(num == 0))
+ return 0;
+
+ if (num > QUEUE_MULTI_MAX)
+ num = QUEUE_MULTI_MAX;
+
+ return queue->s.enqueue_multi(handle,
+ (odp_buffer_hdr_t **)(uintptr_t)ev, num);
+}
+
+static int queue_api_enq(odp_queue_t handle, odp_event_t ev)
+{
+ queue_entry_t *queue = qentry_from_handle(handle);
+
+ return queue->s.enqueue(handle,
+ (odp_buffer_hdr_t *)(uintptr_t)ev);
+}
+
+static int queue_api_deq_multi(odp_queue_t handle, odp_event_t ev[], int num)
{
- return qentry_from_handle(handle);
+ queue_entry_t *queue = qentry_from_handle(handle);
+
+ if (num > QUEUE_MULTI_MAX)
+ num = QUEUE_MULTI_MAX;
+
+ return queue->s.dequeue_multi(handle,
+ (odp_buffer_hdr_t **)ev, num);
}
-static odp_queue_t queue_to_ext(void *q_int)
+static odp_event_t queue_api_deq(odp_queue_t handle)
{
- queue_entry_t *qentry = q_int;
+ queue_entry_t *queue = qentry_from_handle(handle);
- return qentry->s.handle;
+ return (odp_event_t)queue->s.dequeue(handle);
}
/* API functions */
-queue_api_t queue_basic_api = {
+_odp_queue_api_fn_t queue_basic_api = {
.queue_create = queue_create,
.queue_destroy = queue_destroy,
.queue_lookup = queue_lookup,
.queue_capability = queue_capability,
.queue_context_set = queue_context_set,
- .queue_enq = queue_enq,
- .queue_enq_multi = queue_enq_multi,
- .queue_deq = queue_deq,
- .queue_deq_multi = queue_deq_multi,
+ .queue_enq = queue_api_enq,
+ .queue_enq_multi = queue_api_enq_multi,
+ .queue_deq = queue_api_deq,
+ .queue_deq_multi = queue_api_deq_multi,
.queue_type = queue_type,
.queue_sched_type = queue_sched_type,
.queue_sched_prio = queue_sched_prio,
@@ -928,15 +1019,10 @@ queue_fn_t queue_basic_fn = {
.term_global = queue_term_global,
.init_local = queue_init_local,
.term_local = queue_term_local,
- .from_ext = queue_from_ext,
- .to_ext = queue_to_ext,
- .enq = queue_int_enq,
- .enq_multi = queue_int_enq_multi,
- .deq = queue_int_deq,
- .deq_multi = queue_int_deq_multi,
.get_pktout = queue_get_pktout,
.set_pktout = queue_set_pktout,
.get_pktin = queue_get_pktin,
.set_pktin = queue_set_pktin,
- .set_enq_deq_fn = queue_set_enq_deq_func
+ .set_enq_deq_fn = queue_set_enq_deq_func,
+ .orig_deq_multi = queue_orig_multi
};
diff --git a/platform/linux-generic/odp_queue_if.c b/platform/linux-generic/odp_queue_if.c
index f3984542..65f65006 100644
--- a/platform/linux-generic/odp_queue_if.c
+++ b/platform/linux-generic/odp_queue_if.c
@@ -19,101 +19,81 @@
#include <odp/visibility_begin.h>
_odp_queue_inline_offset_t ODP_ALIGNED_CACHE _odp_queue_inline_offset;
+const _odp_queue_api_fn_t *_odp_queue_api;
#include <odp/visibility_end.h>
-extern const queue_api_t queue_scalable_api;
+extern const _odp_queue_api_fn_t queue_scalable_api;
extern const queue_fn_t queue_scalable_fn;
-extern const queue_api_t queue_basic_api;
+extern const _odp_queue_api_fn_t queue_basic_api;
extern const queue_fn_t queue_basic_fn;
-const queue_api_t *queue_api;
const queue_fn_t *queue_fn;
odp_queue_t odp_queue_create(const char *name, const odp_queue_param_t *param)
{
- return queue_api->queue_create(name, param);
+ return _odp_queue_api->queue_create(name, param);
}
int odp_queue_destroy(odp_queue_t queue)
{
- return queue_api->queue_destroy(queue);
+ return _odp_queue_api->queue_destroy(queue);
}
odp_queue_t odp_queue_lookup(const char *name)
{
- return queue_api->queue_lookup(name);
+ return _odp_queue_api->queue_lookup(name);
}
int odp_queue_capability(odp_queue_capability_t *capa)
{
- return queue_api->queue_capability(capa);
+ return _odp_queue_api->queue_capability(capa);
}
int odp_queue_context_set(odp_queue_t queue, void *context, uint32_t len)
{
- return queue_api->queue_context_set(queue, context, len);
-}
-
-int odp_queue_enq(odp_queue_t queue, odp_event_t ev)
-{
- return queue_api->queue_enq(queue, ev);
-}
-
-int odp_queue_enq_multi(odp_queue_t queue, const odp_event_t events[], int num)
-{
- return queue_api->queue_enq_multi(queue, events, num);
-}
-
-odp_event_t odp_queue_deq(odp_queue_t queue)
-{
- return queue_api->queue_deq(queue);
-}
-
-int odp_queue_deq_multi(odp_queue_t queue, odp_event_t events[], int num)
-{
- return queue_api->queue_deq_multi(queue, events, num);
+ return _odp_queue_api->queue_context_set(queue, context, len);
}
odp_queue_type_t odp_queue_type(odp_queue_t queue)
{
- return queue_api->queue_type(queue);
+ return _odp_queue_api->queue_type(queue);
}
odp_schedule_sync_t odp_queue_sched_type(odp_queue_t queue)
{
- return queue_api->queue_sched_type(queue);
+ return _odp_queue_api->queue_sched_type(queue);
}
odp_schedule_prio_t odp_queue_sched_prio(odp_queue_t queue)
{
- return queue_api->queue_sched_prio(queue);
+ return _odp_queue_api->queue_sched_prio(queue);
}
odp_schedule_group_t odp_queue_sched_group(odp_queue_t queue)
{
- return queue_api->queue_sched_group(queue);
+ return _odp_queue_api->queue_sched_group(queue);
}
uint32_t odp_queue_lock_count(odp_queue_t queue)
{
- return queue_api->queue_lock_count(queue);
+ return _odp_queue_api->queue_lock_count(queue);
}
uint64_t odp_queue_to_u64(odp_queue_t hdl)
{
- return queue_api->queue_to_u64(hdl);
+ return _odp_queue_api->queue_to_u64(hdl);
}
void odp_queue_param_init(odp_queue_param_t *param)
{
- return queue_api->queue_param_init(param);
+ return _odp_queue_api->queue_param_init(param);
}
int odp_queue_info(odp_queue_t queue, odp_queue_info_t *info)
{
- return queue_api->queue_info(queue, info);
+ return _odp_queue_api->queue_info(queue, info);
}
int _odp_queue_init_global(void)
@@ -127,10 +107,10 @@ int _odp_queue_init_global(void)
!strcmp(sched, "sp") ||
!strcmp(sched, "iquery")) {
queue_fn = &queue_basic_fn;
- queue_api = &queue_basic_api;
+ _odp_queue_api = &queue_basic_api;
} else if (!strcmp(sched, "scalable")) {
queue_fn = &queue_scalable_fn;
- queue_api = &queue_scalable_api;
+ _odp_queue_api = &queue_scalable_api;
} else {
ODP_ABORT("Unknown scheduler specified via ODP_SCHEDULER\n");
return -1;
diff --git a/platform/linux-generic/odp_queue_lf.c b/platform/linux-generic/odp_queue_lf.c
index d12a994b..a28da2c7 100644
--- a/platform/linux-generic/odp_queue_lf.c
+++ b/platform/linux-generic/odp_queue_lf.c
@@ -162,7 +162,7 @@ static inline int next_idx(int idx)
return next;
}
-static int queue_lf_enq(void *q_int, odp_buffer_hdr_t *buf_hdr)
+static int queue_lf_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr)
{
queue_entry_t *queue;
queue_lf_t *queue_lf;
@@ -172,7 +172,7 @@ static int queue_lf_enq(void *q_int, odp_buffer_hdr_t *buf_hdr)
ring_lf_node_t new_val;
ring_lf_node_t *node;
- queue = q_int;
+ queue = qentry_from_handle(handle);
queue_lf = queue->s.queue_lf;
new_val.s.ptr = (uintptr_t)buf_hdr;
@@ -209,18 +209,18 @@ static int queue_lf_enq(void *q_int, odp_buffer_hdr_t *buf_hdr)
return -1;
}
-static int queue_lf_enq_multi(void *q_int, odp_buffer_hdr_t **buf_hdr,
+static int queue_lf_enq_multi(odp_queue_t handle, odp_buffer_hdr_t **buf_hdr,
int num)
{
(void)num;
- if (queue_lf_enq(q_int, buf_hdr[0]) == 0)
+ if (queue_lf_enq(handle, buf_hdr[0]) == 0)
return 1;
return 0;
}
-static odp_buffer_hdr_t *queue_lf_deq(void *q_int)
+static odp_buffer_hdr_t *queue_lf_deq(odp_queue_t handle)
{
queue_entry_t *queue;
queue_lf_t *queue_lf;
@@ -231,7 +231,7 @@ static odp_buffer_hdr_t *queue_lf_deq(void *q_int)
uint64_t lowest, counter;
odp_buffer_hdr_t *buf_hdr;
- queue = q_int;
+ queue = qentry_from_handle(handle);
queue_lf = queue->s.queue_lf;
new_val.s.counter = 0;
new_val.s.ptr = 0;
@@ -287,14 +287,14 @@ static odp_buffer_hdr_t *queue_lf_deq(void *q_int)
return NULL;
}
-static int queue_lf_deq_multi(void *q_int, odp_buffer_hdr_t **buf_hdr,
+static int queue_lf_deq_multi(odp_queue_t handle, odp_buffer_hdr_t **buf_hdr,
int num)
{
odp_buffer_hdr_t *buf;
(void)num;
- buf = queue_lf_deq(q_int);
+ buf = queue_lf_deq(handle);
if (buf == NULL)
return 0;
diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c
index 13e93bc1..bbc57e44 100644
--- a/platform/linux-generic/odp_queue_scalable.c
+++ b/platform/linux-generic/odp_queue_scalable.c
@@ -54,12 +54,11 @@ typedef struct queue_table_t {
static queue_table_t *queue_tbl;
static _odp_ishm_pool_t *queue_shm_pool;
-static void *queue_from_ext(odp_queue_t handle);
-static int _queue_enq(void *handle, odp_buffer_hdr_t *buf_hdr);
-static odp_buffer_hdr_t *_queue_deq(void *handle);
-static int _queue_enq_multi(void *handle, odp_buffer_hdr_t *buf_hdr[],
+static int _queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr);
+static odp_buffer_hdr_t *_queue_deq(odp_queue_t handle);
+static int _queue_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[],
int num);
-static int _queue_deq_multi(void *handle, odp_buffer_hdr_t *buf_hdr[],
+static int _queue_deq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[],
int num);
static queue_entry_t *get_qentry(uint32_t queue_id)
@@ -129,6 +128,7 @@ static int queue_init(queue_entry_t *queue, const char *name,
queue->s.dequeue = _queue_deq;
queue->s.enqueue_multi = _queue_enq_multi;
queue->s.dequeue_multi = _queue_deq_multi;
+ queue->s.orig_dequeue_multi = _queue_deq_multi;
queue->s.pktin = PKTIN_INVALID;
sched_elem->node.next = NULL;
@@ -610,7 +610,7 @@ int _odp_queue_enq_sp(sched_elem_t *q,
return actual;
}
-static int _queue_enq_multi(void *handle, odp_buffer_hdr_t *buf_hdr[],
+static int _queue_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[],
int num)
{
int actual;
@@ -646,7 +646,7 @@ static int _queue_enq_multi(void *handle, odp_buffer_hdr_t *buf_hdr[],
return actual;
}
-static int _queue_enq(void *handle, odp_buffer_hdr_t *buf_hdr)
+static int _queue_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr)
{
return odp_likely(
_queue_enq_multi(handle, &buf_hdr, 1) == 1) ? 0 : -1;
@@ -666,7 +666,7 @@ static int queue_enq_multi(odp_queue_t handle, const odp_event_t ev[], int num)
for (i = 0; i < num; i++)
buf_hdr[i] = buf_hdl_to_hdr(odp_buffer_from_event(ev[i]));
- return queue->s.enqueue_multi(qentry_to_int(queue), buf_hdr, num);
+ return queue->s.enqueue_multi(handle, buf_hdr, num);
}
static int queue_enq(odp_queue_t handle, odp_event_t ev)
@@ -677,7 +677,7 @@ static int queue_enq(odp_queue_t handle, odp_event_t ev)
queue = qentry_from_ext(handle);
buf_hdr = buf_hdl_to_hdr(odp_buffer_from_event(ev));
- return queue->s.enqueue(qentry_to_int(queue), buf_hdr);
+ return queue->s.enqueue(handle, buf_hdr);
}
/* Single-consumer dequeue. */
@@ -809,7 +809,7 @@ inline int _odp_queue_deq_mc(sched_elem_t *q, odp_event_t *evp, int num)
return ret;
}
-static int _queue_deq_multi(void *handle, odp_buffer_hdr_t *buf_hdr[],
+static int _queue_deq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[],
int num)
{
sched_elem_t *q;
@@ -820,7 +820,7 @@ static int _queue_deq_multi(void *handle, odp_buffer_hdr_t *buf_hdr[],
return _odp_queue_deq(q, buf_hdr, num);
}
-static odp_buffer_hdr_t *_queue_deq(void *handle)
+static odp_buffer_hdr_t *_queue_deq(odp_queue_t handle)
{
sched_elem_t *q;
odp_buffer_hdr_t *buf_hdr;
@@ -842,7 +842,7 @@ static int queue_deq_multi(odp_queue_t handle, odp_event_t ev[], int num)
num = QUEUE_MULTI_MAX;
queue = qentry_from_ext(handle);
- return queue->s.dequeue_multi(qentry_to_int(queue), (odp_buffer_hdr_t **)ev, num);
+ return queue->s.dequeue_multi(handle, (odp_buffer_hdr_t **)ev, num);
}
static odp_event_t queue_deq(odp_queue_t handle)
@@ -850,7 +850,7 @@ static odp_event_t queue_deq(odp_queue_t handle)
queue_entry_t *queue;
queue = qentry_from_ext(handle);
- return (odp_event_t)queue->s.dequeue(qentry_to_int(queue));
+ return (odp_event_t)queue->s.dequeue(handle);
}
static void queue_param_init(odp_queue_param_t *params)
@@ -909,29 +909,29 @@ static uint64_t queue_to_u64(odp_queue_t hdl)
return _odp_pri(hdl);
}
-static odp_pktout_queue_t queue_get_pktout(void *handle)
+static odp_pktout_queue_t queue_get_pktout(odp_queue_t handle)
{
return qentry_from_int(handle)->s.pktout;
}
-static void queue_set_pktout(void *handle, odp_pktio_t pktio, int index)
+static void queue_set_pktout(odp_queue_t handle, odp_pktio_t pktio, int index)
{
qentry_from_int(handle)->s.pktout.pktio = pktio;
qentry_from_int(handle)->s.pktout.index = index;
}
-static odp_pktin_queue_t queue_get_pktin(void *handle)
+static odp_pktin_queue_t queue_get_pktin(odp_queue_t handle)
{
return qentry_from_int(handle)->s.pktin;
}
-static void queue_set_pktin(void *handle, odp_pktio_t pktio, int index)
+static void queue_set_pktin(odp_queue_t handle, odp_pktio_t pktio, int index)
{
qentry_from_int(handle)->s.pktin.pktio = pktio;
qentry_from_int(handle)->s.pktin.index = index;
}
-static void queue_set_enq_deq_func(void *handle,
+static void queue_set_enq_deq_func(odp_queue_t handle,
queue_enq_fn_t enq,
queue_enq_multi_fn_t enq_multi,
queue_deq_fn_t deq,
@@ -950,18 +950,15 @@ static void queue_set_enq_deq_func(void *handle,
qentry_from_int(handle)->s.dequeue_multi = deq_multi;
}
-static void *queue_from_ext(odp_queue_t handle)
+static int queue_orig_multi(odp_queue_t handle,
+ odp_buffer_hdr_t **buf_hdr, int num)
{
- return (void *)handle;
-}
-
-static odp_queue_t queue_to_ext(void *handle)
-{
- return (odp_queue_t)handle;
+ return qentry_from_int(handle)->s.orig_dequeue_multi(handle,
+ buf_hdr, num);
}
/* API functions */
-queue_api_t queue_scalable_api = {
+_odp_queue_api_fn_t queue_scalable_api = {
.queue_create = queue_create,
.queue_destroy = queue_destroy,
.queue_lookup = queue_lookup,
@@ -987,15 +984,10 @@ queue_fn_t queue_scalable_fn = {
.term_global = queue_term_global,
.init_local = queue_init_local,
.term_local = queue_term_local,
- .from_ext = queue_from_ext,
- .to_ext = queue_to_ext,
- .enq = _queue_enq,
- .enq_multi = _queue_enq_multi,
- .deq = _queue_deq,
- .deq_multi = _queue_deq_multi,
.get_pktout = queue_get_pktout,
.set_pktout = queue_set_pktout,
.get_pktin = queue_get_pktin,
.set_pktin = queue_set_pktin,
- .set_enq_deq_fn = queue_set_enq_deq_func
+ .set_enq_deq_fn = queue_set_enq_deq_func,
+ .orig_deq_multi = queue_orig_multi
};
diff --git a/platform/linux-generic/odp_queue_spsc.c b/platform/linux-generic/odp_queue_spsc.c
index 3e42b038..002561a4 100644
--- a/platform/linux-generic/odp_queue_spsc.c
+++ b/platform/linux-generic/odp_queue_spsc.c
@@ -32,14 +32,14 @@ static inline void buffer_index_to_buf(odp_buffer_hdr_t *buf_hdr[],
}
}
-static inline int spsc_enq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[],
- int num)
+static inline int spsc_enq_multi(odp_queue_t handle,
+ odp_buffer_hdr_t *buf_hdr[], int num)
{
queue_entry_t *queue;
ring_spsc_t *ring_spsc;
uint32_t buf_idx[num];
- queue = q_int;
+ queue = qentry_from_handle(handle);
ring_spsc = &queue->s.ring_spsc;
buffer_index_from_buf(buf_idx, buf_hdr, num);
@@ -49,18 +49,19 @@ static inline int spsc_enq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[],
return -1;
}
- return ring_spsc_enq_multi(ring_spsc, buf_idx, num);
+ return ring_spsc_enq_multi(ring_spsc, queue->s.ring_data,
+ queue->s.ring_mask, buf_idx, num);
}
-static inline int spsc_deq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[],
- int num)
+static inline int spsc_deq_multi(odp_queue_t handle,
+ odp_buffer_hdr_t *buf_hdr[], int num)
{
queue_entry_t *queue;
int num_deq;
ring_spsc_t *ring_spsc;
uint32_t buf_idx[num];
- queue = q_int;
+ queue = qentry_from_handle(handle);
ring_spsc = &queue->s.ring_spsc;
if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) {
@@ -68,7 +69,8 @@ static inline int spsc_deq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[],
return -1;
}
- num_deq = ring_spsc_deq_multi(ring_spsc, buf_idx, num);
+ num_deq = ring_spsc_deq_multi(ring_spsc, queue->s.ring_data,
+ queue->s.ring_mask, buf_idx, num);
if (num_deq == 0)
return 0;
@@ -78,17 +80,17 @@ static inline int spsc_deq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[],
return num_deq;
}
-static int queue_spsc_enq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[],
+static int queue_spsc_enq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[],
int num)
{
- return spsc_enq_multi(q_int, buf_hdr, num);
+ return spsc_enq_multi(handle, buf_hdr, num);
}
-static int queue_spsc_enq(void *q_int, odp_buffer_hdr_t *buf_hdr)
+static int queue_spsc_enq(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr)
{
int ret;
- ret = spsc_enq_multi(q_int, &buf_hdr, 1);
+ ret = spsc_enq_multi(handle, &buf_hdr, 1);
if (ret == 1)
return 0;
@@ -96,18 +98,18 @@ static int queue_spsc_enq(void *q_int, odp_buffer_hdr_t *buf_hdr)
return -1;
}
-static int queue_spsc_deq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[],
+static int queue_spsc_deq_multi(odp_queue_t handle, odp_buffer_hdr_t *buf_hdr[],
int num)
{
- return spsc_deq_multi(q_int, buf_hdr, num);
+ return spsc_deq_multi(handle, buf_hdr, num);
}
-static odp_buffer_hdr_t *queue_spsc_deq(void *q_int)
+static odp_buffer_hdr_t *queue_spsc_deq(odp_queue_t handle)
{
odp_buffer_hdr_t *buf_hdr = NULL;
int ret;
- ret = spsc_deq_multi(q_int, &buf_hdr, 1);
+ ret = spsc_deq_multi(handle, &buf_hdr, 1);
if (ret == 1)
return buf_hdr;
@@ -123,9 +125,11 @@ void queue_spsc_init(queue_entry_t *queue, uint32_t queue_size)
queue->s.dequeue = queue_spsc_deq;
queue->s.enqueue_multi = queue_spsc_enq_multi;
queue->s.dequeue_multi = queue_spsc_deq_multi;
+ queue->s.orig_dequeue_multi = queue_spsc_deq_multi;
offset = queue->s.index * (uint64_t)queue_glb->config.max_queue_size;
- ring_spsc_init(&queue->s.ring_spsc, &queue_glb->ring_data[offset],
- queue_size);
+ queue->s.ring_data = &queue_glb->ring_data[offset];
+ queue->s.ring_mask = queue_size - 1;
+ ring_spsc_init(&queue->s.ring_spsc);
}
diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c
index df63da72..e329a8e8 100644
--- a/platform/linux-generic/odp_schedule_basic.c
+++ b/platform/linux-generic/odp_schedule_basic.c
@@ -28,6 +28,7 @@
#include <odp_timer_internal.h>
#include <odp_queue_basic_internal.h>
#include <odp_libconfig_internal.h>
+#include <odp/api/plat/queue_inlines.h>
/* Number of priority levels */
#define NUM_PRIO 8
@@ -103,7 +104,7 @@ ODP_STATIC_ASSERT((8 * sizeof(pri_mask_t)) >= MAX_SPREAD,
/* Storage for stashed enqueue operation arguments */
typedef struct {
odp_buffer_hdr_t *buf_hdr[QUEUE_MULTI_MAX];
- queue_entry_t *queue_entry;
+ odp_queue_t queue;
int num;
} ordered_stash_t;
@@ -661,15 +662,16 @@ static inline void ordered_stash_release(void)
int i;
for (i = 0; i < sched_local.ordered.stash_num; i++) {
- queue_entry_t *queue_entry;
+ odp_queue_t queue;
odp_buffer_hdr_t **buf_hdr;
int num, num_enq;
- queue_entry = sched_local.ordered.stash[i].queue_entry;
+ queue = sched_local.ordered.stash[i].queue;
buf_hdr = sched_local.ordered.stash[i].buf_hdr;
num = sched_local.ordered.stash[i].num;
- num_enq = queue_fn->enq_multi(queue_entry, buf_hdr, num);
+ num_enq = odp_queue_enq_multi(queue,
+ (odp_event_t *)buf_hdr, num);
/* Drop packets that were not enqueued */
if (odp_unlikely(num_enq < num)) {
@@ -744,12 +746,12 @@ static inline int copy_from_stash(odp_event_t out_ev[], unsigned int max)
return i;
}
-static int schedule_ord_enq_multi(void *q_int, void *buf_hdr[],
+static int schedule_ord_enq_multi(odp_queue_t dst_queue, void *buf_hdr[],
int num, int *ret)
{
int i;
uint32_t stash_num = sched_local.ordered.stash_num;
- queue_entry_t *dst_queue = q_int;
+ queue_entry_t *dst_qentry = qentry_from_handle(dst_queue);
uint32_t src_queue = sched_local.ordered.src_queue;
if ((src_queue == NULL_INDEX) || sched_local.ordered.in_order)
@@ -763,7 +765,7 @@ static int schedule_ord_enq_multi(void *q_int, void *buf_hdr[],
}
/* Pktout may drop packets, so the operation cannot be stashed. */
- if (dst_queue->s.pktout.pktio != ODP_PKTIO_INVALID ||
+ if (dst_qentry->s.pktout.pktio != ODP_PKTIO_INVALID ||
odp_unlikely(stash_num >= MAX_ORDERED_STASH)) {
/* If the local stash is full, wait until it is our turn and
* then release the stash and do enqueue directly. */
@@ -775,7 +777,7 @@ static int schedule_ord_enq_multi(void *q_int, void *buf_hdr[],
return 0;
}
- sched_local.ordered.stash[stash_num].queue_entry = dst_queue;
+ sched_local.ordered.stash[stash_num].queue = dst_queue;
sched_local.ordered.stash[stash_num].num = num;
for (i = 0; i < num; i++)
sched_local.ordered.stash[stash_num].buf_hdr[i] = buf_hdr[i];
@@ -839,7 +841,7 @@ static inline int poll_pktin(uint32_t qi, int direct_recv,
q_int = qentry_from_index(qi);
- ret = queue_fn->enq_multi(q_int, b_hdr, num);
+ ret = odp_queue_enq_multi(q_int, (odp_event_t *)b_hdr, num);
/* Drop packets that were not enqueued */
if (odp_unlikely(ret < num)) {
diff --git a/platform/linux-generic/odp_schedule_iquery.c b/platform/linux-generic/odp_schedule_iquery.c
index 2501a3f6..7dde7784 100644
--- a/platform/linux-generic/odp_schedule_iquery.c
+++ b/platform/linux-generic/odp_schedule_iquery.c
@@ -26,6 +26,7 @@
#include <odp_config_internal.h>
#include <odp_timer_internal.h>
#include <odp_queue_basic_internal.h>
+#include <odp/api/plat/queue_inlines.h>
/* Number of priority levels */
#define NUM_SCHED_PRIO 8
@@ -171,7 +172,7 @@ typedef struct {
/* Storage for stashed enqueue operation arguments */
typedef struct {
odp_buffer_hdr_t *buf_hdr[QUEUE_MULTI_MAX];
- queue_entry_t *queue_entry;
+ odp_queue_t queue;
int num;
} ordered_stash_t;
@@ -1132,15 +1133,16 @@ static inline void ordered_stash_release(void)
int i;
for (i = 0; i < thread_local.ordered.stash_num; i++) {
- queue_entry_t *queue_entry;
+ odp_queue_t queue;
odp_buffer_hdr_t **buf_hdr;
int num, num_enq;
- queue_entry = thread_local.ordered.stash[i].queue_entry;
+ queue = thread_local.ordered.stash[i].queue;
buf_hdr = thread_local.ordered.stash[i].buf_hdr;
num = thread_local.ordered.stash[i].num;
- num_enq = queue_fn->enq_multi(queue_entry, buf_hdr, num);
+ num_enq = odp_queue_enq_multi(queue,
+ (odp_event_t *)buf_hdr, num);
if (odp_unlikely(num_enq < num)) {
if (odp_unlikely(num_enq < 0))
@@ -1200,12 +1202,12 @@ static inline void schedule_release_context(void)
schedule_release_atomic();
}
-static int schedule_ord_enq_multi(void *q_int, void *buf_hdr[],
+static int schedule_ord_enq_multi(odp_queue_t dst_queue, void *buf_hdr[],
int num, int *ret)
{
int i;
uint32_t stash_num = thread_local.ordered.stash_num;
- queue_entry_t *dst_queue = q_int;
+ queue_entry_t *dst_qentry = qentry_from_handle(dst_queue);
uint32_t src_queue = thread_local.ordered.src_queue;
if ((src_queue == NULL_INDEX) || thread_local.ordered.in_order)
@@ -1219,7 +1221,7 @@ static int schedule_ord_enq_multi(void *q_int, void *buf_hdr[],
}
/* Pktout may drop packets, so the operation cannot be stashed. */
- if (dst_queue->s.pktout.pktio != ODP_PKTIO_INVALID ||
+ if (dst_qentry->s.pktout.pktio != ODP_PKTIO_INVALID ||
odp_unlikely(stash_num >= MAX_ORDERED_STASH)) {
/* If the local stash is full, wait until it is our turn and
* then release the stash and do enqueue directly. */
@@ -1231,7 +1233,7 @@ static int schedule_ord_enq_multi(void *q_int, void *buf_hdr[],
return 0;
}
- thread_local.ordered.stash[stash_num].queue_entry = dst_queue;
+ thread_local.ordered.stash[stash_num].queue = dst_queue;
thread_local.ordered.stash[stash_num].num = num;
for (i = 0; i < num; i++)
thread_local.ordered.stash[stash_num].buf_hdr[i] = buf_hdr[i];
diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c
index 826747ee..9acb997a 100644
--- a/platform/linux-generic/odp_schedule_scalable.c
+++ b/platform/linux-generic/odp_schedule_scalable.c
@@ -2007,7 +2007,7 @@ static int sched_queue(uint32_t queue_index)
return 0;
}
-static int ord_enq_multi(void *handle, void *buf_hdr[], int num,
+static int ord_enq_multi(odp_queue_t handle, void *buf_hdr[], int num,
int *ret)
{
diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c
index ae292051..7932e186 100644
--- a/platform/linux-generic/odp_schedule_sp.c
+++ b/platform/linux-generic/odp_schedule_sp.c
@@ -415,10 +415,10 @@ static int sched_queue(uint32_t qi)
return 0;
}
-static int ord_enq_multi(void *q_int, void *buf_hdr[], int num,
+static int ord_enq_multi(odp_queue_t queue, void *buf_hdr[], int num,
int *ret)
{
- (void)q_int;
+ (void)queue;
(void)buf_hdr;
(void)num;
(void)ret;
diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c
index 5fef5af2..0d2ecd27 100644
--- a/platform/linux-generic/odp_timer.c
+++ b/platform/linux-generic/odp_timer.c
@@ -56,6 +56,7 @@
#include <odp/api/plat/time_inlines.h>
#include <odp/api/timer.h>
#include <odp_timer_internal.h>
+#include <odp/api/plat/queue_inlines.h>
/* Inlined API functions */
#include <odp/api/plat/event_inlines.h>
@@ -231,7 +232,7 @@ static inline timer_pool_t *handle_to_tp(odp_timer_t hdl)
static inline uint32_t handle_to_idx(odp_timer_t hdl,
timer_pool_t *tp)
{
- uint32_t idx = _odp_typeval(hdl) & ((1U << INDEX_BITS) - 1U);
+ uint32_t idx = (_odp_typeval(hdl) & ((1U << INDEX_BITS) - 1U)) - 1;
__builtin_prefetch(&tp->tick_buf[idx], 0, 0);
if (odp_likely(idx < odp_atomic_load_u32(&tp->high_wm)))
return idx;
@@ -241,8 +242,9 @@ static inline uint32_t handle_to_idx(odp_timer_t hdl,
static inline odp_timer_t tp_idx_to_handle(timer_pool_t *tp,
uint32_t idx)
{
- ODP_ASSERT(idx < (1U << INDEX_BITS));
- return _odp_cast_scalar(odp_timer_t, (tp->tp_idx << INDEX_BITS) | idx);
+ ODP_ASSERT((idx + 1) < (1U << INDEX_BITS));
+ return _odp_cast_scalar(odp_timer_t, (tp->tp_idx << INDEX_BITS) |
+ (idx + 1));
}
/* Forward declarations */
@@ -259,6 +261,7 @@ static odp_timer_pool_t timer_pool_new(const char *name,
if (timer_global.num_timer_pools >= MAX_TIMER_POOLS) {
odp_ticketlock_unlock(&timer_global.lock);
+ ODP_DBG("No more free timer pools\n");
return ODP_TIMER_POOL_INVALID;
}
diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c
index af403b4b..7ca1a4ce 100644
--- a/platform/linux-generic/odp_traffic_mngr.c
+++ b/platform/linux-generic/odp_traffic_mngr.c
@@ -108,20 +108,20 @@ static odp_bool_t tm_demote_pkt_desc(tm_system_t *tm_system,
tm_shaper_obj_t *timer_shaper,
pkt_desc_t *demoted_pkt_desc);
-static int queue_tm_reenq(void *queue, odp_buffer_hdr_t *buf_hdr)
+static int queue_tm_reenq(odp_queue_t queue, odp_buffer_hdr_t *buf_hdr)
{
- odp_tm_queue_t tm_queue = MAKE_ODP_TM_QUEUE((uint8_t *)queue -
- offsetof(tm_queue_obj_t,
- tm_qentry));
+ odp_tm_queue_t tm_queue = MAKE_ODP_TM_QUEUE(odp_queue_context(queue));
odp_packet_t pkt = packet_from_buf_hdr(buf_hdr);
return odp_tm_enq(tm_queue, pkt);
}
-static int queue_tm_reenq_multi(void *queue ODP_UNUSED,
- odp_buffer_hdr_t *buf[] ODP_UNUSED,
- int num ODP_UNUSED)
+static int queue_tm_reenq_multi(odp_queue_t queue, odp_buffer_hdr_t *buf[],
+ int num)
{
+ (void)queue;
+ (void)buf;
+ (void)num;
ODP_ABORT("Invalid call to queue_tm_reenq_multi()\n");
return 0;
}
@@ -3936,8 +3936,10 @@ odp_tm_queue_t odp_tm_queue_create(odp_tm_t odp_tm,
free(tm_queue_obj);
return ODP_TM_INVALID;
}
- tm_queue_obj->tm_qentry = queue_fn->from_ext(queue);
- queue_fn->set_enq_deq_fn(tm_queue_obj->tm_qentry,
+
+ tm_queue_obj->queue = queue;
+ odp_queue_context_set(queue, tm_queue_obj, sizeof(tm_queue_obj_t));
+ queue_fn->set_enq_deq_fn(queue,
queue_tm_reenq, queue_tm_reenq_multi,
NULL, NULL);
@@ -3967,12 +3969,9 @@ odp_tm_queue_t odp_tm_queue_create(odp_tm_t odp_tm,
int odp_tm_queue_destroy(odp_tm_queue_t tm_queue)
{
- tm_wred_params_t *wred_params;
tm_shaper_obj_t *shaper_obj;
tm_queue_obj_t *tm_queue_obj;
- tm_wred_node_t *tm_wred_node;
tm_system_t *tm_system;
- uint32_t color;
/* First lookup tm_queue. */
tm_queue_obj = GET_TM_QUEUE_OBJ(tm_queue);
@@ -3990,28 +3989,11 @@ int odp_tm_queue_destroy(odp_tm_queue_t tm_queue)
(tm_queue_obj->pkt != ODP_PACKET_INVALID))
return -1;
- /* Check that there is no shaper profile, threshold profile or wred
- * profile currently associated with this tm_queue. */
- if (shaper_obj->shaper_params != NULL)
- return -1;
-
- tm_wred_node = tm_queue_obj->tm_wred_node;
- if (tm_wred_node != NULL) {
- if (tm_wred_node->threshold_params != NULL)
- return -1;
-
- for (color = 0; color < ODP_NUM_PACKET_COLORS; color++) {
- wred_params = tm_wred_node->wred_params[color];
- if (wred_params != NULL)
- return -1;
- }
- }
-
/* Now that all of the checks are done, time to so some freeing. */
odp_ticketlock_lock(&tm_system->tm_system_lock);
tm_system->queue_num_tbl[tm_queue_obj->queue_num - 1] = NULL;
- odp_queue_destroy(queue_fn->to_ext(tm_queue_obj->tm_qentry));
+ odp_queue_destroy(tm_queue_obj->queue);
/* First delete any associated tm_wred_node and then the tm_queue_obj
* itself */
diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c
index 4e320aa0..caebdfe8 100644
--- a/platform/linux-generic/pktio/dpdk.c
+++ b/platform/linux-generic/pktio/dpdk.c
@@ -168,6 +168,8 @@ void refer_constructors(void)
}
#endif
+static int pool_alloc(struct rte_mempool *mp);
+
static int lookup_opt(const char *opt_name, const char *drv_name, int *val)
{
const char *base = "pktio_dpdk";
@@ -341,7 +343,7 @@ static struct rte_mempool *mbuf_pool_create(const char *name,
rte_pktmbuf_pool_init(mp, &mbp_priv);
- if (rte_mempool_ops_alloc(mp)) {
+ if (pool_alloc(mp)) {
ODP_ERR("Failed allocating mempool\n");
return NULL;
}
diff --git a/platform/linux-generic/pktio/loop.c b/platform/linux-generic/pktio/loop.c
index faaef3ab..e01dd458 100644
--- a/platform/linux-generic/pktio/loop.c
+++ b/platform/linux-generic/pktio/loop.c
@@ -18,6 +18,7 @@
#include <odp/api/hints.h>
#include <odp/api/plat/byteorder_inlines.h>
#include <odp_queue_if.h>
+#include <odp/api/plat/queue_inlines.h>
#include <protocols/eth.h>
#include <protocols/ip.h>
@@ -93,7 +94,7 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
{
int nbr, i;
odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX];
- void *queue;
+ odp_queue_t queue;
odp_packet_hdr_t *pkt_hdr;
odp_packet_t pkt;
odp_time_t ts_val;
@@ -106,8 +107,8 @@ static int loopback_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
odp_ticketlock_lock(&pktio_entry->s.rxl);
- queue = queue_fn->from_ext(pkt_priv(pktio_entry)->loopq);
- nbr = queue_fn->deq_multi(queue, hdr_tbl, num);
+ queue = pkt_priv(pktio_entry)->loopq;
+ nbr = odp_queue_deq_multi(queue, (odp_event_t *)hdr_tbl, num);
if (pktio_entry->s.config.pktin.bit.ts_all ||
pktio_entry->s.config.pktin.bit.ts_ptp) {
@@ -235,7 +236,8 @@ static inline void loopback_fix_checksums(odp_packet_t pkt,
uint8_t l4_proto;
void *l3_hdr;
uint32_t l3_len;
- odp_bool_t ipv4_chksum_pkt, udp_chksum_pkt, tcp_chksum_pkt;
+ odp_bool_t ipv4_chksum_pkt, udp_chksum_pkt, tcp_chksum_pkt,
+ sctp_chksum_pkt;
odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
l3_hdr = odp_packet_l3_ptr(pkt, &l3_len);
@@ -259,6 +261,11 @@ static inline void loopback_fix_checksums(odp_packet_t pkt,
l4_proto == _ODP_IPPROTO_TCP,
pkt_hdr->p.flags.l4_chksum_set,
pkt_hdr->p.flags.l4_chksum);
+ sctp_chksum_pkt = OL_TX_CHKSUM_PKT(pktout_cfg->bit.sctp_chksum,
+ pktout_capa->bit.sctp_chksum,
+ l4_proto == _ODP_IPPROTO_SCTP,
+ pkt_hdr->p.flags.l4_chksum_set,
+ pkt_hdr->p.flags.l4_chksum);
if (ipv4_chksum_pkt)
_odp_packet_ipv4_chksum_insert(pkt);
@@ -268,13 +275,16 @@ static inline void loopback_fix_checksums(odp_packet_t pkt,
if (udp_chksum_pkt)
_odp_packet_udp_chksum_insert(pkt);
+
+ if (sctp_chksum_pkt)
+ _odp_packet_sctp_chksum_insert(pkt);
}
static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
const odp_packet_t pkt_tbl[], int num)
{
odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX];
- void *queue;
+ odp_queue_t queue;
int i;
int ret;
int nb_tx = 0;
@@ -324,8 +334,8 @@ static int loopback_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED,
odp_ticketlock_lock(&pktio_entry->s.txl);
- queue = queue_fn->from_ext(pkt_priv(pktio_entry)->loopq);
- ret = queue_fn->enq_multi(queue, hdr_tbl, nb_tx);
+ queue = pkt_priv(pktio_entry)->loopq;
+ ret = odp_queue_enq_multi(queue, (odp_event_t *)hdr_tbl, nb_tx);
if (ret > 0) {
pktio_entry->s.stats.out_ucast_pkts += ret;
@@ -375,9 +385,11 @@ static int loopback_init_capability(pktio_entry_t *pktio_entry)
capa->config.pktin.bit.ipv4_chksum = 1;
capa->config.pktin.bit.tcp_chksum = 1;
capa->config.pktin.bit.udp_chksum = 1;
+ capa->config.pktin.bit.sctp_chksum = 1;
capa->config.pktout.bit.ipv4_chksum = 1;
capa->config.pktout.bit.tcp_chksum = 1;
capa->config.pktout.bit.udp_chksum = 1;
+ capa->config.pktout.bit.sctp_chksum = 1;
capa->config.inbound_ipsec = 1;
capa->config.outbound_ipsec = 1;
@@ -387,6 +399,8 @@ static int loopback_init_capability(pktio_entry_t *pktio_entry)
capa->config.pktout.bit.udp_chksum;
capa->config.pktout.bit.tcp_chksum_ena =
capa->config.pktout.bit.tcp_chksum;
+ capa->config.pktout.bit.sctp_chksum_ena =
+ capa->config.pktout.bit.sctp_chksum;
return 0;
}
diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c
index 0da2b7af..53d3e58c 100644
--- a/platform/linux-generic/pktio/netmap.c
+++ b/platform/linux-generic/pktio/netmap.c
@@ -836,8 +836,10 @@ static inline int netmap_recv_desc(pktio_entry_t *pktio_entry,
if (odp_likely(ring->slot[slot_id].len <= mtu)) {
slot_tbl[num_rx].buf = buf;
slot_tbl[num_rx].len = ring->slot[slot_id].len;
- ODP_DBG("dropped oversized packet\n");
num_rx++;
+ } else {
+ ODP_DBG("Dropped oversized packet: %" PRIu16 " "
+ "B\n", ring->slot[slot_id].len);
}
ring->cur = nm_ring_next(ring, slot_id);
}
diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh
new file mode 100755
index 00000000..995c797b
--- /dev/null
+++ b/scripts/ci/build.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+set -e
+
+cd "$(dirname "$0")"/../..
+./bootstrap
+./configure \
+ --host=${TARGET_ARCH} --build=x86_64-linux-gnu \
+ --enable-dpdk \
+ --prefix=/opt/odp \
+ ${CONF}
+
+make -j 8
+
+make install
+
+pushd ${HOME}
+${CC} ${CFLAGS} ${OLDPWD}/example/hello/odp_hello.c -o odp_hello_inst_dynamic `PKG_CONFIG_PATH=/opt/odp/lib/pkgconfig:${PKG_CONFIG_PATH} pkg-config --cflags --libs libodp-linux`
+if [ -z "$TARGET_ARCH" ]
+then
+ LD_LIBRARY_PATH="/opt/odp/lib:$LD_LIBRARY_PATH" ./odp_hello_inst_dynamic
+fi
+popd
diff --git a/scripts/ci/build_arm64.sh b/scripts/ci/build_arm64.sh
new file mode 100755
index 00000000..647dd29c
--- /dev/null
+++ b/scripts/ci/build_arm64.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+export TARGET_ARCH=aarch64-linux-gnu
+if [ "${CC#clang}" != "${CC}" ] ; then
+ export CC="clang --target=${TARGET_ARCH}"
+ export CXX="clang++ --target=${TARGET_ARCH}"
+else
+ export CC="${TARGET_ARCH}-gcc"
+ export CXX="${TARGET_ARCH}-g++"
+fi
+export CPPFLAGS="-I/usr/include/${TARGET_ARCH}/dpdk"
+
+exec "$(dirname "$0")"/build.sh
diff --git a/scripts/ci/build_armhf.sh b/scripts/ci/build_armhf.sh
new file mode 100755
index 00000000..837561f8
--- /dev/null
+++ b/scripts/ci/build_armhf.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+set -e
+
+export TARGET_ARCH=arm-linux-gnueabihf
+if [ "${CC#clang}" != "${CC}" ] ; then
+ export CC="clang --target=${TARGET_ARCH}"
+ export CXX="clang++ --target=${TARGET_ARCH}"
+else
+ export CC="${TARGET_ARCH}-gcc"
+ export CXX="${TARGET_ARCH}-g++"
+fi
+export CPPFLAGS="-I/usr/include/${TARGET_ARCH}/dpdk"
+export CFLAGS="-march=armv7-a"
+export CXXFLAGS="-march=armv7-a"
+
+exec "$(dirname "$0")"/build.sh
diff --git a/scripts/ci/build_i386.sh b/scripts/ci/build_i386.sh
new file mode 100755
index 00000000..17b6bf66
--- /dev/null
+++ b/scripts/ci/build_i386.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+
+export TARGET_ARCH=i686-linux-gnu
+if [ "${CC#clang}" != "${CC}" ] ; then
+ export CC="clang --target=${TARGET_ARCH}"
+ export CXX="clang++ --target=${TARGET_ARCH}"
+else
+ export CFLAGS="-m32"
+ export CXXFLAGS="-m32"
+ export LDFLAGS="-m32"
+fi
+export CPPFLAGS="-I/usr/include/i386-linux-gnu/dpdk"
+
+exec "$(dirname "$0")"/build.sh
diff --git a/scripts/ci/build_powerpc.sh b/scripts/ci/build_powerpc.sh
new file mode 100755
index 00000000..a213ee1d
--- /dev/null
+++ b/scripts/ci/build_powerpc.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+
+export TARGET_ARCH=powerpc-linux-gnu
+if [ "${CC#clang}" != "${CC}" ] ; then
+ export CC="clang --target=${TARGET_ARCH}"
+ export CXX="clang++ --target=${TARGET_ARCH}"
+else
+ export CC="${TARGET_ARCH}-gcc"
+ export CXX="${TARGET_ARCH}-g++"
+fi
+# No DPDK on PowerPC
+export CONF="${CONF} --disable-dpdk"
+
+exec "$(dirname "$0")"/build.sh
diff --git a/scripts/ci/build_x86_64.sh b/scripts/ci/build_x86_64.sh
new file mode 100755
index 00000000..01182fd9
--- /dev/null
+++ b/scripts/ci/build_x86_64.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+if [ "${CC#clang}" != "${CC}" ] ; then
+ export CXX="clang++"
+fi
+
+exec "$(dirname "$0")"/build.sh
diff --git a/scripts/ci/check.sh b/scripts/ci/check.sh
new file mode 100755
index 00000000..43168680
--- /dev/null
+++ b/scripts/ci/check.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+"`dirname "$0"`"/build_x86_64.sh
+
+cd "$(dirname "$0")"/../..
+
+echo 1000 | tee /proc/sys/vm/nr_hugepages
+mkdir -p /mnt/huge
+mount -t hugetlbfs nodev /mnt/huge
+
+# Ignore possible failures there because these tests depends on measurements
+# and systems might differ in performance.
+export CI="true"
+make check
+
+umount /mnt/huge
diff --git a/scripts/ci/coverage.sh b/scripts/ci/coverage.sh
new file mode 100755
index 00000000..18e2e980
--- /dev/null
+++ b/scripts/ci/coverage.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+set -e
+
+if [ "${CC#clang}" != "${CC}" ] ; then
+ export CXX="clang++"
+fi
+
+cd "$(dirname "$0")"/../..
+./bootstrap
+./configure \
+ CFLAGS="-O0 -coverage $CLFAGS" CXXFLAGS="-O0 -coverage $CXXFLAGS" LDFLAGS="--coverage $LDFLAGS" \
+ --enable-debug=full --enable-helper-linux --enable-dpdk --disable-test-perf --disable-test-perf-proc
+export CCACHE_DISABLE=1
+make -j $(nproc)
+
+echo 1000 | tee /proc/sys/vm/nr_hugepages
+mkdir -p /mnt/huge
+mount -t hugetlbfs nodev /mnt/huge
+
+# Ignore possible failures there because these tests depends on measurements
+# and systems might differ in performance.
+export CI="true"
+
+ODP_SCHEDULER=basic make check
+ODP_SCHEDULER=sp make check
+ODP_SCHEDULER=iquery make check
+ODP_SCHEDULER=scalable make check
+
+bash <(curl -s https://codecov.io/bash) -X coveragepy
+
+umount /mnt/huge
diff --git a/scripts/ci/distcheck.sh b/scripts/ci/distcheck.sh
new file mode 100755
index 00000000..c6e0f4ea
--- /dev/null
+++ b/scripts/ci/distcheck.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+
+if [ "${CC#clang}" != "${CC}" ] ; then
+ export CXX="clang++"
+fi
+
+cd "$(dirname "$0")"/../..
+./bootstrap
+./configure \
+ --enable-user-guides
+
+make distcheck
+
+make clean
+
+# Ignore possible failures there because these tests depends on measurements
+# and systems might differ in performance.
+export CI="true"
+
+make distcheck DISTCHECK__CONFIGURE_FLAGS=--disable-abi-compat
diff --git a/test/common/test_packet_parser.h b/test/common/test_packet_parser.h
index 2d02b98c..0f17e856 100644
--- a/test/common/test_packet_parser.h
+++ b/test/common/test_packet_parser.h
@@ -168,6 +168,32 @@ static const uint8_t test_packet_ipv4_sctp[] = {
0x79, 0x74, 0x65, 0x73, 0x2E
};
+/* IPv6 SCTP
+ * - chunk type: payload data
+ */
+static const uint8_t test_packet_ipv6_sctp[] = {
+ 0x00, 0x00, 0x09, 0x00, 0x05, 0x00, 0x00, 0x00,
+ 0x09, 0x00, 0x04, 0x00, 0x86, 0xDD, 0x60, 0x30,
+ 0x00, 0x00, 0x00, 0x63, 0x84, 0xFF, 0xFE, 0x80,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00,
+ 0x09, 0xFF, 0xFE, 0x00, 0x04, 0x00, 0x35, 0x55,
+ 0x55, 0x55, 0x66, 0x66, 0x66, 0x66, 0x77, 0x77,
+ 0x77, 0x77, 0x88, 0x88, 0x88, 0x88, 0x04, 0xD2,
+ 0x16, 0x2E, 0xDE, 0xAD, 0xBE, 0xEF, 0x31, 0x44,
+ 0xE3, 0xFE, 0x00, 0x00, 0x00, 0x57, 0x00, 0x00,
+ 0x00, 0x0A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x54, 0x68, 0x69, 0x73, 0x20, 0x69,
+ 0x73, 0x20, 0x6D, 0x79, 0x20, 0x64, 0x75, 0x6D,
+ 0x6D, 0x79, 0x20, 0x70, 0x61, 0x79, 0x6C, 0x6F,
+ 0x61, 0x64, 0x20, 0x73, 0x74, 0x72, 0x69, 0x6E,
+ 0x67, 0x2E, 0x20, 0x54, 0x68, 0x65, 0x20, 0x6C,
+ 0x65, 0x6E, 0x67, 0x74, 0x68, 0x20, 0x6F, 0x66,
+ 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x73, 0x74,
+ 0x72, 0x69, 0x6E, 0x67, 0x20, 0x69, 0x73, 0x20,
+ 0x37, 0x31, 0x20, 0x62, 0x79, 0x74, 0x65, 0x73,
+ 0x2E
+};
+
static const uint8_t test_packet_ipv4_ipsec_ah[] = {
/* ETH */
0xf1, 0xf1, 0xf1, 0xf1, 0xf1, 0xf1,
diff --git a/test/performance/odp_pool_perf.c b/test/performance/odp_pool_perf.c
index 4a77f327..d771e53e 100644
--- a/test/performance/odp_pool_perf.c
+++ b/test/performance/odp_pool_perf.c
@@ -128,7 +128,7 @@ static int set_num_cpu(test_global_t *global)
ret = odp_cpumask_default_worker(&global->cpumask, num_cpu);
if (num_cpu && ret != num_cpu) {
- printf("Error: Too many workers. Max supported %i\n.", ret);
+ printf("Error: Too many workers. Max supported %i.\n", ret);
return -1;
}
diff --git a/test/performance/odp_queue_perf.c b/test/performance/odp_queue_perf.c
index d5ff254d..09c8592b 100644
--- a/test/performance/odp_queue_perf.c
+++ b/test/performance/odp_queue_perf.c
@@ -12,16 +12,44 @@
#include <getopt.h>
#include <odp_api.h>
+#include <odp/helper/odph_api.h>
+
+#define MAX_QUEUES (32 * 1024)
typedef struct test_options_t {
uint32_t num_queue;
uint32_t num_event;
uint32_t num_round;
+ uint32_t max_burst;
odp_nonblocking_t nonblock;
int single;
+ int num_cpu;
} test_options_t;
+typedef struct test_stat_t {
+ uint64_t rounds;
+ uint64_t events;
+ uint64_t nsec;
+ uint64_t cycles;
+ uint64_t deq_retry;
+
+} test_stat_t;
+
+typedef struct test_global_t {
+ odp_barrier_t barrier;
+ test_options_t options;
+ odp_instance_t instance;
+ odp_shm_t shm;
+ odp_pool_t pool;
+ odp_queue_t queue[MAX_QUEUES];
+ odph_odpthread_t thread_tbl[ODP_THREAD_COUNT_MAX];
+ test_stat_t stat[ODP_THREAD_COUNT_MAX];
+
+} test_global_t;
+
+static test_global_t test_global;
+
static void print_usage(void)
{
printf("\n"
@@ -29,8 +57,10 @@ static void print_usage(void)
"\n"
"Usage: odp_queue_perf [options]\n"
"\n"
- " -q, --num_queue Number of queues\n"
- " -e, --num_event Number of events per queue\n"
+ " -c, --num_cpu Number of worker threads. Default: 1\n"
+ " -q, --num_queue Number of queues. Default: 1\n"
+ " -e, --num_event Number of events per queue. Default: 1\n"
+ " -b, --burst_size Maximum number of events per operation. Default: 1\n"
" -r, --num_round Number of rounds\n"
" -l, --lockfree Lockfree queues\n"
" -w, --waitfree Waitfree queues\n"
@@ -46,20 +76,24 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options)
int ret = 0;
static const struct option longopts[] = {
- {"num_queue", required_argument, NULL, 'q'},
- {"num_event", required_argument, NULL, 'e'},
- {"num_round", required_argument, NULL, 'r'},
- {"lockfree", no_argument, NULL, 'l'},
- {"waitfree", no_argument, NULL, 'w'},
- {"single", no_argument, NULL, 's'},
- {"help", no_argument, NULL, 'h'},
+ {"num_cpu", required_argument, NULL, 'c'},
+ {"num_queue", required_argument, NULL, 'q'},
+ {"num_event", required_argument, NULL, 'e'},
+ {"burst_size", required_argument, NULL, 'b'},
+ {"num_round", required_argument, NULL, 'r'},
+ {"lockfree", no_argument, NULL, 'l'},
+ {"waitfree", no_argument, NULL, 'w'},
+ {"single", no_argument, NULL, 's'},
+ {"help", no_argument, NULL, 'h'},
{NULL, 0, NULL, 0}
};
- static const char *shortopts = "+q:e:r:lwsh";
+ static const char *shortopts = "+c:q:e:b:r:lwsh";
+ test_options->num_cpu = 1;
test_options->num_queue = 1;
test_options->num_event = 1;
+ test_options->max_burst = 1;
test_options->num_round = 1000;
test_options->nonblock = ODP_BLOCKING;
test_options->single = 0;
@@ -71,12 +105,18 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options)
break;
switch (opt) {
+ case 'c':
+ test_options->num_cpu = atoi(optarg);
+ break;
case 'q':
test_options->num_queue = atoi(optarg);
break;
case 'e':
test_options->num_event = atoi(optarg);
break;
+ case 'b':
+ test_options->max_burst = atoi(optarg);
+ break;
case 'r':
test_options->num_round = atoi(optarg);
break;
@@ -98,27 +138,31 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options)
}
}
+ if (test_options->num_queue > MAX_QUEUES) {
+ printf("Too many queues %u. Test maximum %u.\n",
+ test_options->num_queue, MAX_QUEUES);
+ return -1;
+ }
+
return ret;
}
-static int test_queue(test_options_t *test_options)
+static int create_queues(test_global_t *global)
{
odp_pool_capability_t pool_capa;
odp_queue_capability_t queue_capa;
odp_pool_param_t pool_param;
odp_queue_param_t queue_param;
odp_pool_t pool;
- odp_event_t ev;
- uint32_t i, j, rounds;
- uint64_t c1, c2, diff, ops, nsec;
- odp_time_t t1, t2;
- uint64_t num_retry = 0;
+ uint32_t i, j, max_size;
+ test_options_t *test_options = &global->options;
odp_nonblocking_t nonblock = test_options->nonblock;
uint32_t num_queue = test_options->num_queue;
uint32_t num_event = test_options->num_event;
uint32_t num_round = test_options->num_round;
uint32_t tot_event = num_queue * num_event;
- odp_queue_t queue[num_queue];
+ int ret = 0;
+ odp_queue_t *queue = global->queue;
odp_event_t event[tot_event];
printf("\nTesting %s queues\n",
@@ -127,7 +171,8 @@ static int test_queue(test_options_t *test_options)
(nonblock == ODP_NONBLOCKING_WF ? "WAITFREE" : "???")));
printf(" num rounds %u\n", num_round);
printf(" num queues %u\n", num_queue);
- printf(" num events per queue %u\n\n", num_event);
+ printf(" num events per queue %u\n", num_event);
+ printf(" max burst size %u\n", test_options->max_burst);
for (i = 0; i < num_queue; i++)
queue[i] = ODP_QUEUE_INVALID;
@@ -152,9 +197,9 @@ static int test_queue(test_options_t *test_options)
return -1;
}
- if (num_event > queue_capa.plain.max_size) {
- printf("Max queue size supported %u\n",
- queue_capa.plain.max_size);
+ max_size = queue_capa.plain.max_size;
+ if (max_size && num_event > max_size) {
+ printf("Max queue size supported %u\n", max_size);
return -1;
}
} else if (nonblock == ODP_NONBLOCKING_LF) {
@@ -169,9 +214,10 @@ static int test_queue(test_options_t *test_options)
return -1;
}
- if (num_event > queue_capa.plain.lockfree.max_size) {
+ max_size = queue_capa.plain.lockfree.max_size;
+ if (max_size && num_event > max_size) {
printf("Max lockfree queue size supported %u\n",
- queue_capa.plain.lockfree.max_size);
+ max_size);
return -1;
}
} else if (nonblock == ODP_NONBLOCKING_WF) {
@@ -186,9 +232,10 @@ static int test_queue(test_options_t *test_options)
return -1;
}
- if (num_event > queue_capa.plain.waitfree.max_size) {
+ max_size = queue_capa.plain.waitfree.max_size;
+ if (max_size && num_event > max_size) {
printf("Max waitfree queue size supported %u\n",
- queue_capa.plain.waitfree.max_size);
+ max_size);
return -1;
}
} else {
@@ -212,6 +259,8 @@ static int test_queue(test_options_t *test_options)
return -1;
}
+ global->pool = pool;
+
odp_queue_param_init(&queue_param);
queue_param.type = ODP_QUEUE_TYPE_PLAIN;
queue_param.nonblocking = nonblock;
@@ -227,7 +276,7 @@ static int test_queue(test_options_t *test_options)
if (queue[i] == ODP_QUEUE_INVALID) {
printf("Error: Queue create failed %u.\n", i);
- goto error;
+ return -1;
}
}
@@ -236,7 +285,8 @@ static int test_queue(test_options_t *test_options)
if (event[i] == ODP_EVENT_INVALID) {
printf("Error: Event alloc failed %u.\n", i);
- goto error;
+ ret = -1;
+ goto free_events;
}
}
@@ -246,98 +296,240 @@ static int test_queue(test_options_t *test_options)
if (odp_queue_enq(queue[i], event[id])) {
printf("Error: Queue enq failed %u/%u\n", i, j);
- goto error;
+ ret = -1;
+ goto free_events;
}
event[id] = ODP_EVENT_INVALID;
}
}
+free_events:
+ /* Free events that were not stored into queues */
+ for (i = 0; i < tot_event; i++) {
+ if (event[i] != ODP_EVENT_INVALID)
+ odp_event_free(event[i]);
+ }
+
+ return ret;
+}
+
+static int destroy_queues(test_global_t *global)
+{
+ odp_event_t ev;
+ uint32_t i, j;
+ int ret = 0;
+ test_options_t *test_options = &global->options;
+ uint32_t num_queue = test_options->num_queue;
+ uint32_t num_event = test_options->num_event;
+ odp_queue_t *queue = global->queue;
+ odp_pool_t pool = global->pool;
+
+ for (i = 0; i < num_queue; i++) {
+ if (queue[i] == ODP_QUEUE_INVALID) {
+ printf("Error: Invalid queue handle %u.\n", i);
+ ret = -1;
+ break;
+ }
+
+ for (j = 0; j < num_event; j++) {
+ ev = odp_queue_deq(queue[i]);
+
+ if (ev != ODP_EVENT_INVALID)
+ odp_event_free(ev);
+ }
+
+ if (odp_queue_destroy(queue[i])) {
+ printf("Error: Queue destroy failed %u.\n", i);
+ ret = -1;
+ break;
+ }
+ }
+
+ if (odp_pool_destroy(pool)) {
+ printf("Error: Pool destroy failed.\n");
+ ret = -1;
+ }
+
+ return ret;
+}
+
+static int run_test(void *arg)
+{
+ uint64_t c1, c2, cycles, nsec;
+ odp_time_t t1, t2;
+ uint32_t rounds;
+ int num_ev;
+ test_stat_t *stat;
+ test_global_t *global = arg;
+ test_options_t *test_options = &global->options;
+ odp_queue_t queue;
+ uint64_t num_retry = 0;
+ uint64_t events = 0;
+ uint32_t num_queue = test_options->num_queue;
+ uint32_t num_round = test_options->num_round;
+ int thr = odp_thread_id();
+ int ret = 0;
+ uint32_t i = 0;
+ uint32_t max_burst = test_options->max_burst;
+ odp_event_t ev[max_burst];
+
+ stat = &global->stat[thr];
+
+ /* Start all workers at the same time */
+ odp_barrier_wait(&global->barrier);
+
t1 = odp_time_local();
c1 = odp_cpu_cycles();
for (rounds = 0; rounds < num_round; rounds++) {
- int retry = 0;
+ do {
+ queue = global->queue[i++];
- for (i = 0; i < num_queue; i++) {
- ev = odp_queue_deq(queue[i]);
+ if (i == num_queue)
+ i = 0;
- if (ev == ODP_EVENT_INVALID) {
- if (retry < 5) {
- retry++;
- num_retry++;
- continue;
- }
+ num_ev = odp_queue_deq_multi(queue, ev, max_burst);
- printf("Error: Queue deq failed %u\n", i);
- goto error;
- }
+ if (odp_unlikely(num_ev <= 0))
+ num_retry++;
- retry = 0;
+ } while (num_ev <= 0);
- if (odp_queue_enq(queue[i], ev)) {
- printf("Error: Queue enq failed %u\n", i);
- goto error;
- }
+ if (odp_queue_enq_multi(queue, ev, num_ev) != num_ev) {
+ printf("Error: Queue enq failed %u\n", i);
+ ret = -1;
+ goto error;
}
+
+ events += num_ev;
}
c2 = odp_cpu_cycles();
t2 = odp_time_local();
- nsec = odp_time_diff_ns(t2, t1);
- diff = odp_cpu_cycles_diff(c2, c1);
- ops = num_round * num_queue;
+ nsec = odp_time_diff_ns(t2, t1);
+ cycles = odp_cpu_cycles_diff(c2, c1);
- printf("RESULT:\n");
- printf(" num deq + enq operations: %" PRIu64 "\n", ops);
- printf(" duration (nsec): %" PRIu64 "\n", nsec);
- printf(" num cycles: %" PRIu64 "\n", diff);
- printf(" cycles per deq + enq: %.3f\n", (double)diff / ops);
- printf(" num retries: %" PRIu64 "\n\n", num_retry);
+ stat->rounds = rounds;
+ stat->events = events;
+ stat->nsec = nsec;
+ stat->cycles = cycles;
+ stat->deq_retry = num_retry;
error:
+ return ret;
+}
- for (i = 0; i < num_queue; i++) {
- for (j = 0; j < num_event; j++) {
- ev = odp_queue_deq(queue[i]);
-
- if (ev != ODP_EVENT_INVALID)
- odp_event_free(ev);
- }
+static int start_workers(test_global_t *global)
+{
+ odph_odpthread_params_t thr_params;
+ odp_cpumask_t cpumask;
+ int ret;
+ test_options_t *test_options = &global->options;
+ int num_cpu = test_options->num_cpu;
+
+ memset(&thr_params, 0, sizeof(thr_params));
+ thr_params.thr_type = ODP_THREAD_WORKER;
+ thr_params.instance = global->instance;
+ thr_params.start = run_test;
+ thr_params.arg = global;
+
+ ret = odp_cpumask_default_worker(&cpumask, num_cpu);
+
+ if (num_cpu && ret != num_cpu) {
+ printf("Error: Too many workers. Max supported %i\n.", ret);
+ return -1;
}
- for (i = 0; i < tot_event; i++) {
- if (event[i] != ODP_EVENT_INVALID)
- odp_event_free(event[i]);
+ /* Zero: all available workers */
+ if (num_cpu == 0) {
+ num_cpu = ret;
+ test_options->num_cpu = num_cpu;
}
- for (i = 0; i < num_queue; i++) {
- if (queue[i] == ODP_QUEUE_INVALID)
- break;
+ printf(" num workers %u\n\n", num_cpu);
- if (odp_queue_destroy(queue[i])) {
- printf("Error: Queue destroy failed %u.\n", i);
- break;
- }
- }
+ odp_barrier_init(&global->barrier, num_cpu);
- if (odp_pool_destroy(pool)) {
- printf("Error: Pool destroy failed.\n");
+ if (odph_odpthreads_create(global->thread_tbl, &cpumask, &thr_params)
+ != num_cpu)
return -1;
- }
return 0;
}
+static void print_stat(test_global_t *global)
+{
+ int i, num;
+ double rounds_ave, events_ave, nsec_ave, cycles_ave, retry_ave;
+ test_options_t *test_options = &global->options;
+ int num_cpu = test_options->num_cpu;
+ uint64_t rounds_sum = 0;
+ uint64_t events_sum = 0;
+ uint64_t nsec_sum = 0;
+ uint64_t cycles_sum = 0;
+ uint64_t retry_sum = 0;
+
+ /* Averages */
+ for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) {
+ rounds_sum += global->stat[i].rounds;
+ events_sum += global->stat[i].events;
+ nsec_sum += global->stat[i].nsec;
+ cycles_sum += global->stat[i].cycles;
+ retry_sum += global->stat[i].deq_retry;
+ }
+
+ if (rounds_sum == 0) {
+ printf("No results.\n");
+ return;
+ }
+
+ rounds_ave = rounds_sum / num_cpu;
+ events_ave = events_sum / num_cpu;
+ nsec_ave = nsec_sum / num_cpu;
+ cycles_ave = cycles_sum / num_cpu;
+ retry_ave = retry_sum / num_cpu;
+ num = 0;
+
+ printf("RESULTS - per thread (Million events per sec):\n");
+ printf("----------------------------------------------\n");
+ printf(" 1 2 3 4 5 6 7 8 9 10");
+
+ for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) {
+ if (global->stat[i].rounds) {
+ if ((num % 10) == 0)
+ printf("\n ");
+
+ printf("%6.1f ", (1000.0 * global->stat[i].events) /
+ global->stat[i].nsec);
+ num++;
+ }
+ }
+ printf("\n\n");
+
+ printf("RESULTS - per thread average (%i threads):\n", num_cpu);
+ printf("------------------------------------------\n");
+ printf(" duration: %.3f msec\n", nsec_ave / 1000000);
+ printf(" num cycles: %.3f M\n", cycles_ave / 1000000);
+ printf(" evenst per dequeue: %.3f\n",
+ events_ave / rounds_ave);
+ printf(" cycles per event: %.3f\n",
+ cycles_ave / events_ave);
+ printf(" deq retries per sec: %.3f k\n",
+ (1000000.0 * retry_ave) / nsec_ave);
+ printf(" events per sec: %.3f M\n\n",
+ (1000.0 * events_ave) / nsec_ave);
+
+ printf("TOTAL events per sec: %.3f M\n\n",
+ (1000.0 * events_sum) / nsec_ave);
+}
+
int main(int argc, char **argv)
{
odp_instance_t instance;
odp_init_t init;
- test_options_t test_options;
-
- if (parse_options(argc, argv, &test_options))
- return -1;
+ test_global_t *global;
/* List features not to be used */
odp_init_param_init(&init);
@@ -360,8 +552,34 @@ int main(int argc, char **argv)
return -1;
}
- if (test_queue(&test_options))
- printf("Error: Queue test failed.\n");
+ global = &test_global;
+ memset(global, 0, sizeof(test_global_t));
+
+ if (parse_options(argc, argv, &global->options))
+ return -1;
+
+ global->instance = instance;
+
+ if (create_queues(global)) {
+ printf("Error: Create queues failed.\n");
+ goto destroy;
+ }
+
+ if (start_workers(global)) {
+ printf("Error: Test start failed.\n");
+ return -1;
+ }
+
+ /* Wait workers to exit */
+ odph_odpthreads_join(global->thread_tbl);
+
+ print_stat(global);
+
+destroy:
+ if (destroy_queues(global)) {
+ printf("Error: Destroy queues failed.\n");
+ return -1;
+ }
if (odp_term_local()) {
printf("Error: term local failed.\n");
diff --git a/test/performance/odp_sched_perf.c b/test/performance/odp_sched_perf.c
index e76725cc..ac2b9005 100644
--- a/test/performance/odp_sched_perf.c
+++ b/test/performance/odp_sched_perf.c
@@ -14,12 +14,18 @@
#include <odp_api.h>
#include <odp/helper/odph_api.h>
+#define MAX_QUEUES_PER_CPU 1024
+#define MAX_QUEUES (ODP_THREAD_COUNT_MAX * MAX_QUEUES_PER_CPU)
+
typedef struct test_options_t {
uint32_t num_cpu;
+ uint32_t num_queue;
uint32_t num_event;
uint32_t num_round;
uint32_t max_burst;
int queue_type;
+ uint32_t tot_queue;
+ uint32_t tot_event;
} test_options_t;
@@ -38,7 +44,7 @@ typedef struct test_global_t {
odp_barrier_t barrier;
odp_pool_t pool;
odp_cpumask_t cpumask;
- odp_queue_t queue[ODP_THREAD_COUNT_MAX];
+ odp_queue_t queue[MAX_QUEUES];
odph_odpthread_t thread_tbl[ODP_THREAD_COUNT_MAX];
test_stat_t stat[ODP_THREAD_COUNT_MAX];
@@ -53,11 +59,12 @@ static void print_usage(void)
"\n"
"Usage: odp_sched_perf [options]\n"
"\n"
- " -c, --num_cpu Number of CPUs (worker threads). 0: all available CPUs. Default 1.\n"
+ " -c, --num_cpu Number of CPUs (worker threads). 0: all available CPUs. Default: 1.\n"
+ " -q, --num_queue Number of queues per CPU. Default: 1.\n"
" -e, --num_event Number of events per queue\n"
" -r, --num_round Number of rounds\n"
" -b, --burst Maximum number of events per operation\n"
- " -t, --type Queue type. 0: parallel, 1: atomic, 2: ordered. Default 0.\n"
+ " -t, --type Queue type. 0: parallel, 1: atomic, 2: ordered. Default: 0.\n"
" -h, --help This help\n"
"\n");
}
@@ -70,6 +77,7 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options)
static const struct option longopts[] = {
{"num_cpu", required_argument, NULL, 'c'},
+ {"num_queue", required_argument, NULL, 'q'},
{"num_event", required_argument, NULL, 'e'},
{"num_round", required_argument, NULL, 'r'},
{"burst", required_argument, NULL, 'b'},
@@ -78,9 +86,10 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options)
{NULL, 0, NULL, 0}
};
- static const char *shortopts = "+c:e:r:b:t:h";
+ static const char *shortopts = "+c:q:e:r:b:t:h";
test_options->num_cpu = 1;
+ test_options->num_queue = 1;
test_options->num_event = 100;
test_options->num_round = 100000;
test_options->max_burst = 100;
@@ -96,6 +105,9 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options)
case 'c':
test_options->num_cpu = atoi(optarg);
break;
+ case 'q':
+ test_options->num_queue = atoi(optarg);
+ break;
case 'e':
test_options->num_event = atoi(optarg);
break;
@@ -117,6 +129,17 @@ static int parse_options(int argc, char *argv[], test_options_t *test_options)
}
}
+ if (test_options->num_queue > MAX_QUEUES_PER_CPU) {
+ printf("Error: Too many queues per worker. Max supported %i\n.",
+ MAX_QUEUES_PER_CPU);
+ ret = -1;
+ }
+
+ test_options->tot_queue = test_options->num_queue *
+ test_options->num_cpu;
+ test_options->tot_event = test_options->tot_queue *
+ test_options->num_event;
+
return ret;
}
@@ -157,18 +180,22 @@ static int create_pool(test_global_t *global)
odp_pool_param_t pool_param;
odp_pool_t pool;
test_options_t *test_options = &global->test_options;
+ uint32_t num_cpu = test_options->num_cpu;
+ uint32_t num_queue = test_options->num_queue;
uint32_t num_event = test_options->num_event;
uint32_t num_round = test_options->num_round;
uint32_t max_burst = test_options->max_burst;
- int num_cpu = test_options->num_cpu;
- uint32_t tot_event = num_event * num_cpu;
+ uint32_t tot_queue = test_options->tot_queue;
+ uint32_t tot_event = test_options->tot_event;
printf("\nScheduler performance test\n");
- printf(" num cpu %i\n", num_cpu);
- printf(" num rounds %u\n", num_round);
- printf(" num events %u\n", tot_event);
+ printf(" num cpu %u\n", num_cpu);
+ printf(" queues per cpu %u\n", num_queue);
printf(" events per queue %u\n", num_event);
- printf(" max burst %u\n", max_burst);
+ printf(" max burst size %u\n", max_burst);
+ printf(" num queues %u\n", tot_queue);
+ printf(" num events %u\n", tot_event);
+ printf(" num rounds %u\n", num_round);
if (odp_pool_capability(&pool_capa)) {
printf("Error: Pool capa failed.\n");
@@ -207,7 +234,7 @@ static int create_queues(test_global_t *global)
uint32_t i, j;
test_options_t *test_options = &global->test_options;
uint32_t num_event = test_options->num_event;
- uint32_t num_queue = test_options->num_cpu;
+ uint32_t tot_queue = test_options->tot_queue;
int type = test_options->queue_type;
odp_pool_t pool = global->pool;
@@ -222,7 +249,6 @@ static int create_queues(test_global_t *global)
sync = ODP_SCHED_SYNC_ORDERED;
}
- printf(" num queues %u\n", num_queue);
printf(" queue type %s\n\n", type_str);
if (odp_queue_capability(&queue_capa)) {
@@ -230,7 +256,7 @@ static int create_queues(test_global_t *global)
return -1;
}
- if (num_queue > queue_capa.sched.max_num) {
+ if (tot_queue > queue_capa.sched.max_num) {
printf("Max queues supported %u\n", queue_capa.sched.max_num);
return -1;
}
@@ -241,9 +267,6 @@ static int create_queues(test_global_t *global)
return -1;
}
- for (i = 0; i < ODP_THREAD_COUNT_MAX; i++)
- global->queue[i] = ODP_QUEUE_INVALID;
-
odp_queue_param_init(&queue_param);
queue_param.type = ODP_QUEUE_TYPE_SCHED;
queue_param.sched.prio = ODP_SCHED_PRIO_DEFAULT;
@@ -251,18 +274,18 @@ static int create_queues(test_global_t *global)
queue_param.sched.group = ODP_SCHED_GROUP_ALL;
queue_param.size = num_event;
- for (i = 0; i < num_queue; i++) {
+ for (i = 0; i < tot_queue; i++) {
queue = odp_queue_create(NULL, &queue_param);
+ global->queue[i] = queue;
+
if (queue == ODP_QUEUE_INVALID) {
printf("Error: Queue create failed %u\n", i);
return -1;
}
-
- global->queue[i] = queue;
}
- for (i = 0; i < num_queue; i++) {
+ for (i = 0; i < tot_queue; i++) {
queue = global->queue[i];
for (j = 0; j < num_event; j++) {
@@ -288,13 +311,15 @@ static int destroy_queues(test_global_t *global)
uint32_t i;
odp_event_t ev;
uint64_t wait;
+ test_options_t *test_options = &global->test_options;
+ uint32_t tot_queue = test_options->tot_queue;
wait = odp_schedule_wait_time(200 * ODP_TIME_MSEC_IN_NS);
while ((ev = odp_schedule(NULL, wait)) != ODP_EVENT_INVALID)
odp_event_free(ev);
- for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) {
+ for (i = 0; i < tot_queue; i++) {
if (global->queue[i] != ODP_QUEUE_INVALID) {
if (odp_queue_destroy(global->queue[i])) {
printf("Error: Queue destroy failed %u\n", i);
diff --git a/test/validation/api/pktio/parser.c b/test/validation/api/pktio/parser.c
index 76df47b1..db37c1c5 100644
--- a/test/validation/api/pktio/parser.c
+++ b/test/validation/api/pktio/parser.c
@@ -301,6 +301,7 @@ static void parser_test_ipv4_icmp(void)
CU_ASSERT(!odp_packet_has_ipv6(pkt));
CU_ASSERT(!odp_packet_has_tcp(pkt));
CU_ASSERT(!odp_packet_has_udp(pkt));
+ CU_ASSERT(!odp_packet_has_sctp(pkt));
odp_packet_free(pkt);
}
@@ -318,6 +319,7 @@ static void parser_test_ipv4_tcp(void)
CU_ASSERT(!odp_packet_has_ipv6(pkt));
CU_ASSERT(!odp_packet_has_udp(pkt));
+ CU_ASSERT(!odp_packet_has_sctp(pkt));
odp_packet_free(pkt);
}
@@ -335,6 +337,7 @@ static void parser_test_ipv4_udp(void)
CU_ASSERT(!odp_packet_has_ipv6(pkt));
CU_ASSERT(!odp_packet_has_tcp(pkt));
+ CU_ASSERT(!odp_packet_has_sctp(pkt));
odp_packet_free(pkt);
}
@@ -353,6 +356,7 @@ static void parser_test_vlan_ipv4_udp(void)
CU_ASSERT(!odp_packet_has_ipv6(pkt));
CU_ASSERT(!odp_packet_has_tcp(pkt));
+ CU_ASSERT(!odp_packet_has_sctp(pkt));
odp_packet_free(pkt);
}
@@ -372,6 +376,25 @@ static void parser_test_vlan_qinq_ipv4_udp(void)
CU_ASSERT(!odp_packet_has_ipv6(pkt));
CU_ASSERT(!odp_packet_has_tcp(pkt));
+ CU_ASSERT(!odp_packet_has_sctp(pkt));
+
+ odp_packet_free(pkt);
+}
+
+static void parser_test_ipv4_sctp(void)
+{
+ odp_packet_t pkt;
+
+ pkt = loopback_packet(pktio_a, pktio_b, test_packet_ipv4_sctp,
+ sizeof(test_packet_ipv4_sctp));
+ CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID);
+ CU_ASSERT(odp_packet_has_eth(pkt));
+ CU_ASSERT(odp_packet_has_ipv4(pkt));
+ CU_ASSERT(odp_packet_has_sctp(pkt));
+
+ CU_ASSERT(!odp_packet_has_ipv6(pkt));
+ CU_ASSERT(!odp_packet_has_tcp(pkt));
+ CU_ASSERT(!odp_packet_has_udp(pkt));
odp_packet_free(pkt);
}
@@ -390,6 +413,7 @@ static void parser_test_ipv6_icmp(void)
CU_ASSERT(!odp_packet_has_ipv4(pkt));
CU_ASSERT(!odp_packet_has_tcp(pkt));
CU_ASSERT(!odp_packet_has_udp(pkt));
+ CU_ASSERT(!odp_packet_has_sctp(pkt));
odp_packet_free(pkt);
}
@@ -407,6 +431,7 @@ static void parser_test_ipv6_tcp(void)
CU_ASSERT(!odp_packet_has_ipv4(pkt));
CU_ASSERT(!odp_packet_has_udp(pkt));
+ CU_ASSERT(!odp_packet_has_sctp(pkt));
odp_packet_free(pkt);
}
@@ -424,6 +449,7 @@ static void parser_test_ipv6_udp(void)
CU_ASSERT(!odp_packet_has_ipv4(pkt));
CU_ASSERT(!odp_packet_has_tcp(pkt));
+ CU_ASSERT(!odp_packet_has_sctp(pkt));
odp_packet_free(pkt);
}
@@ -442,6 +468,25 @@ static void parser_test_vlan_ipv6_udp(void)
CU_ASSERT(!odp_packet_has_ipv4(pkt));
CU_ASSERT(!odp_packet_has_tcp(pkt));
+ CU_ASSERT(!odp_packet_has_sctp(pkt));
+
+ odp_packet_free(pkt);
+}
+
+static void parser_test_ipv6_sctp(void)
+{
+ odp_packet_t pkt;
+
+ pkt = loopback_packet(pktio_a, pktio_b, test_packet_ipv6_sctp,
+ sizeof(test_packet_ipv6_sctp));
+ CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID);
+ CU_ASSERT(odp_packet_has_eth(pkt));
+ CU_ASSERT(odp_packet_has_ipv6(pkt));
+ CU_ASSERT(odp_packet_has_sctp(pkt));
+
+ CU_ASSERT(!odp_packet_has_ipv4(pkt));
+ CU_ASSERT(!odp_packet_has_tcp(pkt));
+ CU_ASSERT(!odp_packet_has_udp(pkt));
odp_packet_free(pkt);
}
@@ -555,9 +600,11 @@ odp_testinfo_t parser_suite[] = {
ODP_TEST_INFO(parser_test_ipv4_udp),
ODP_TEST_INFO_CONDITIONAL(parser_test_vlan_ipv4_udp, loop_pktio),
ODP_TEST_INFO_CONDITIONAL(parser_test_vlan_qinq_ipv4_udp, loop_pktio),
+ ODP_TEST_INFO(parser_test_ipv4_sctp),
ODP_TEST_INFO(parser_test_ipv6_icmp),
ODP_TEST_INFO(parser_test_ipv6_tcp),
ODP_TEST_INFO(parser_test_ipv6_udp),
ODP_TEST_INFO_CONDITIONAL(parser_test_vlan_ipv6_udp, loop_pktio),
+ ODP_TEST_INFO(parser_test_ipv6_sctp),
ODP_TEST_INFO_NULL
};
diff --git a/test/validation/api/pktio/pktio.c b/test/validation/api/pktio/pktio.c
index 09367d4e..3f082674 100644
--- a/test/validation/api/pktio/pktio.c
+++ b/test/validation/api/pktio/pktio.c
@@ -164,7 +164,7 @@ static void pktio_pkt_set_macs(odp_packet_t pkt, odp_pktio_t src, odp_pktio_t ds
CU_ASSERT(ret <= ODP_PKTIO_MACADDR_MAXSIZE);
}
-static uint32_t pktio_pkt_set_seq(odp_packet_t pkt)
+static uint32_t pktio_pkt_set_seq(odp_packet_t pkt, size_t l4_hdr_len)
{
static uint32_t tstseq;
size_t off;
@@ -180,7 +180,7 @@ static uint32_t pktio_pkt_set_seq(odp_packet_t pkt)
head.magic = TEST_SEQ_MAGIC;
head.seq = tstseq;
- off += ODPH_UDPHDR_LEN;
+ off += l4_hdr_len;
if (odp_packet_copy_from_mem(pkt, off, sizeof(head), &head) != 0)
return TEST_SEQ_INVALID;
@@ -194,7 +194,7 @@ static uint32_t pktio_pkt_set_seq(odp_packet_t pkt)
return head.seq;
}
-static uint32_t pktio_pkt_seq(odp_packet_t pkt)
+static uint32_t pktio_pkt_seq_hdr(odp_packet_t pkt, size_t l4_hdr_len)
{
size_t off;
uint32_t seq = TEST_SEQ_INVALID;
@@ -212,7 +212,7 @@ static uint32_t pktio_pkt_seq(odp_packet_t pkt)
return TEST_SEQ_INVALID;
}
- off += ODPH_UDPHDR_LEN;
+ off += l4_hdr_len;
if (odp_packet_copy_to_mem(pkt, off, sizeof(head), &head) != 0) {
fprintf(stderr, "error: header copy failed\n");
return TEST_SEQ_INVALID;
@@ -250,11 +250,15 @@ static uint32_t pktio_pkt_seq(odp_packet_t pkt)
return seq;
}
-static uint32_t pktio_init_packet(odp_packet_t pkt)
+static uint32_t pktio_pkt_seq(odp_packet_t pkt)
+{
+ return pktio_pkt_seq_hdr(pkt, ODPH_UDPHDR_LEN);
+}
+
+static void pktio_init_packet_eth_ipv4(odp_packet_t pkt, uint8_t proto)
{
odph_ethhdr_t *eth;
odph_ipv4hdr_t *ip;
- odph_udphdr_t *udp;
char *buf;
uint16_t seq;
uint8_t src_mac[ODP_PKTIO_MACADDR_MAXSIZE] = PKTIO_SRC_MAC;
@@ -278,11 +282,22 @@ static uint32_t pktio_init_packet(odp_packet_t pkt)
ip->ver_ihl = ODPH_IPV4 << 4 | ODPH_IPV4HDR_IHL_MIN;
ip->tot_len = odp_cpu_to_be_16(pkt_len - ODPH_ETHHDR_LEN);
ip->ttl = 128;
- ip->proto = ODPH_IPPROTO_UDP;
+ ip->proto = proto;
seq = odp_atomic_fetch_inc_u32(&ip_seq);
ip->id = odp_cpu_to_be_16(seq);
ip->chksum = 0;
odph_ipv4_csum_update(pkt);
+}
+
+static uint32_t pktio_init_packet_udp(odp_packet_t pkt)
+{
+ odph_udphdr_t *udp;
+ char *buf;
+ int pkt_len = odp_packet_len(pkt);
+
+ buf = odp_packet_data(pkt);
+
+ pktio_init_packet_eth_ipv4(pkt, ODPH_IPPROTO_UDP);
/* UDP */
odp_packet_l4_offset_set(pkt, ODPH_ETHHDR_LEN + ODPH_IPV4HDR_LEN);
@@ -293,40 +308,76 @@ static uint32_t pktio_init_packet(odp_packet_t pkt)
ODPH_ETHHDR_LEN - ODPH_IPV4HDR_LEN);
udp->chksum = 0;
- return pktio_pkt_set_seq(pkt);
+ return pktio_pkt_set_seq(pkt, ODPH_UDPHDR_LEN);
+}
+
+static uint32_t pktio_init_packet_sctp(odp_packet_t pkt)
+{
+ odph_sctphdr_t *sctp;
+ char *buf;
+
+ buf = odp_packet_data(pkt);
+
+ pktio_init_packet_eth_ipv4(pkt, ODPH_IPPROTO_SCTP);
+
+ /* SCTP */
+ odp_packet_l4_offset_set(pkt, ODPH_ETHHDR_LEN + ODPH_IPV4HDR_LEN);
+ sctp = (odph_sctphdr_t *)(buf + ODPH_ETHHDR_LEN + ODPH_IPV4HDR_LEN);
+ sctp->src_port = odp_cpu_to_be_16(12049);
+ sctp->dst_port = odp_cpu_to_be_16(12050);
+ sctp->tag = 0;
+ sctp->chksum = 0;
+
+ return pktio_pkt_set_seq(pkt, ODPH_SCTPHDR_LEN);
}
static int pktio_zero_checksums(odp_packet_t pkt)
{
odph_ipv4hdr_t *ip;
- odph_udphdr_t *udp;
uint32_t len;
ip = (odph_ipv4hdr_t *)odp_packet_l3_ptr(pkt, &len);
- if (ip->proto != ODPH_IPPROTO_UDP) {
+ ip->chksum = 0;
+
+ if (ip->proto == ODPH_IPPROTO_UDP) {
+ odph_udphdr_t *udp;
+
+ udp = (odph_udphdr_t *)odp_packet_l4_ptr(pkt, &len);
+ udp->chksum = 0;
+ } else if (ip->proto == ODPH_IPPROTO_SCTP) {
+ odph_sctphdr_t *sctp;
+
+ sctp = (odph_sctphdr_t *)odp_packet_l4_ptr(pkt, &len);
+ sctp->chksum = 0;
+ } else {
CU_FAIL("unexpected L4 protocol");
return -1;
}
- udp = (odph_udphdr_t *)odp_packet_l4_ptr(pkt, &len);
-
- ip->chksum = 0;
- udp->chksum = 0;
-
return 0;
}
static int pktio_fixup_checksums(odp_packet_t pkt)
{
- odph_udphdr_t *udp;
+ odph_ipv4hdr_t *ip;
pktio_zero_checksums(pkt);
- udp = (odph_udphdr_t *)odp_packet_l4_ptr(pkt, NULL);
-
odph_ipv4_csum_update(pkt);
- udp->chksum = odph_ipv4_udp_chksum(pkt);
+
+ ip = (odph_ipv4hdr_t *)odp_packet_l3_ptr(pkt, NULL);
+ if (ip->proto == ODPH_IPPROTO_UDP) {
+ odph_udphdr_t *udp;
+
+ udp = (odph_udphdr_t *)odp_packet_l4_ptr(pkt, NULL);
+ udp->chksum = odph_ipv4_udp_chksum(pkt);
+ } else if (ip->proto == ODPH_IPPROTO_SCTP) {
+ odph_sctp_chksum_set(pkt);
+ } else {
+ CU_FAIL("unexpected L4 protocol");
+ return -1;
+ }
return 0;
}
@@ -419,12 +470,12 @@ static int flush_input_queue(odp_pktio_t pktio, odp_pktin_mode_t imode)
return 0;
}
-static int create_packets_cs(odp_packet_t pkt_tbl[],
- uint32_t pkt_seq[],
- int num,
- odp_pktio_t pktio_src,
- odp_pktio_t pktio_dst,
- odp_bool_t fix_cs)
+static int create_packets_udp(odp_packet_t pkt_tbl[],
+ uint32_t pkt_seq[],
+ int num,
+ odp_pktio_t pktio_src,
+ odp_pktio_t pktio_dst,
+ odp_bool_t fix_cs)
{
int i, ret;
@@ -433,7 +484,7 @@ static int create_packets_cs(odp_packet_t pkt_tbl[],
if (pkt_tbl[i] == ODP_PACKET_INVALID)
break;
- pkt_seq[i] = pktio_init_packet(pkt_tbl[i]);
+ pkt_seq[i] = pktio_init_packet_udp(pkt_tbl[i]);
if (pkt_seq[i] == TEST_SEQ_INVALID) {
odp_packet_free(pkt_tbl[i]);
break;
@@ -454,11 +505,42 @@ static int create_packets_cs(odp_packet_t pkt_tbl[],
return i;
}
+static int create_packets_sctp(odp_packet_t pkt_tbl[],
+ uint32_t pkt_seq[],
+ int num,
+ odp_pktio_t pktio_src,
+ odp_pktio_t pktio_dst)
+{
+ int i, ret;
+
+ for (i = 0; i < num; i++) {
+ pkt_tbl[i] = odp_packet_alloc(default_pkt_pool, packet_len);
+ if (pkt_tbl[i] == ODP_PACKET_INVALID)
+ break;
+
+ pkt_seq[i] = pktio_init_packet_sctp(pkt_tbl[i]);
+ if (pkt_seq[i] == TEST_SEQ_INVALID) {
+ odp_packet_free(pkt_tbl[i]);
+ break;
+ }
+
+ pktio_pkt_set_macs(pkt_tbl[i], pktio_src, pktio_dst);
+
+ ret = pktio_zero_checksums(pkt_tbl[i]);
+ if (ret != 0) {
+ odp_packet_free(pkt_tbl[i]);
+ break;
+ }
+ }
+
+ return i;
+}
+
static int create_packets(odp_packet_t pkt_tbl[], uint32_t pkt_seq[], int num,
odp_pktio_t pktio_src, odp_pktio_t pktio_dst)
{
- return create_packets_cs(pkt_tbl, pkt_seq, num, pktio_src, pktio_dst,
- true);
+ return create_packets_udp(pkt_tbl, pkt_seq, num, pktio_src, pktio_dst,
+ true);
}
static int get_packets(pktio_info_t *pktio_rx, odp_packet_t pkt_tbl[],
@@ -506,9 +588,9 @@ static int get_packets(pktio_info_t *pktio_rx, odp_packet_t pkt_tbl[],
return num_pkts;
}
-static int wait_for_packets(pktio_info_t *pktio_rx, odp_packet_t pkt_tbl[],
- uint32_t seq_tbl[], int num, txrx_mode_e mode,
- uint64_t ns)
+static int wait_for_packets_hdr(pktio_info_t *pktio_rx, odp_packet_t pkt_tbl[],
+ uint32_t seq_tbl[], int num, txrx_mode_e mode,
+ uint64_t ns, size_t l4_hdr_len)
{
odp_time_t wait_time, end;
int num_rx = 0;
@@ -525,7 +607,8 @@ static int wait_for_packets(pktio_info_t *pktio_rx, odp_packet_t pkt_tbl[],
break;
for (i = 0; i < n; ++i) {
- if (pktio_pkt_seq(pkt_tmp[i]) == seq_tbl[num_rx])
+ if (pktio_pkt_seq_hdr(pkt_tmp[i], l4_hdr_len) ==
+ seq_tbl[num_rx])
pkt_tbl[num_rx++] = pkt_tmp[i];
else
odp_packet_free(pkt_tmp[i]);
@@ -535,6 +618,14 @@ static int wait_for_packets(pktio_info_t *pktio_rx, odp_packet_t pkt_tbl[],
return num_rx;
}
+static int wait_for_packets(pktio_info_t *pktio_rx, odp_packet_t pkt_tbl[],
+ uint32_t seq_tbl[], int num, txrx_mode_e mode,
+ uint64_t ns)
+{
+ return wait_for_packets_hdr(pktio_rx, pkt_tbl, seq_tbl, num, mode, ns,
+ ODPH_UDPHDR_LEN);
+}
+
static int recv_packets_tmo(odp_pktio_t pktio, odp_packet_t pkt_tbl[],
uint32_t seq_tbl[], int num, recv_tmo_mode_e mode,
uint64_t tmo, uint64_t ns, int no_pkt)
@@ -1956,9 +2047,16 @@ static void pktio_test_chksum(void (*config_fn)(odp_pktio_t, odp_pktio_t),
_pktio_wait_linkup(pktio[i]);
}
- ret = create_packets_cs(pkt_tbl, pkt_seq, TX_BATCH_LEN, pktio_tx,
- pktio_rx, false);
- CU_ASSERT_FATAL(ret == TX_BATCH_LEN);
+ ret = create_packets_udp(pkt_tbl, pkt_seq, TX_BATCH_LEN, pktio_tx,
+ pktio_rx, false);
+ CU_ASSERT(ret == TX_BATCH_LEN);
+ if (ret != TX_BATCH_LEN) {
+ for (i = 0; i < num_ifaces; i++) {
+ CU_ASSERT_FATAL(odp_pktio_stop(pktio[i]) == 0);
+ CU_ASSERT_FATAL(odp_pktio_close(pktio[i]) == 0);
+ }
+ return;
+ }
ret = odp_pktout_queue(pktio_tx, &pktout_queue, 1);
CU_ASSERT_FATAL(ret > 0);
@@ -1983,6 +2081,75 @@ static void pktio_test_chksum(void (*config_fn)(odp_pktio_t, odp_pktio_t),
}
}
+static void pktio_test_chksum_sctp(void (*config_fn)(odp_pktio_t, odp_pktio_t),
+ void (*prep_fn)(odp_packet_t pkt),
+ void (*test_fn)(odp_packet_t pkt))
+{
+ odp_pktio_t pktio_tx, pktio_rx;
+ odp_pktio_t pktio[MAX_NUM_IFACES] = {ODP_PKTIO_INVALID};
+ pktio_info_t pktio_rx_info;
+ odp_pktout_queue_t pktout_queue;
+ odp_packet_t pkt_tbl[TX_BATCH_LEN];
+ uint32_t pkt_seq[TX_BATCH_LEN];
+ int ret;
+ int i, num_rx;
+
+ CU_ASSERT_FATAL(num_ifaces >= 1);
+
+ /* Open and configure interfaces */
+ for (i = 0; i < num_ifaces; ++i) {
+ pktio[i] = create_pktio(i, ODP_PKTIN_MODE_DIRECT,
+ ODP_PKTOUT_MODE_DIRECT);
+ CU_ASSERT_FATAL(pktio[i] != ODP_PKTIO_INVALID);
+ }
+
+ pktio_tx = pktio[0];
+ pktio_rx = (num_ifaces > 1) ? pktio[1] : pktio_tx;
+ pktio_rx_info.id = pktio_rx;
+ pktio_rx_info.inq = ODP_QUEUE_INVALID;
+ pktio_rx_info.in_mode = ODP_PKTIN_MODE_DIRECT;
+
+ config_fn(pktio_tx, pktio_rx);
+
+ for (i = 0; i < num_ifaces; ++i) {
+ CU_ASSERT_FATAL(odp_pktio_start(pktio[i]) == 0);
+ _pktio_wait_linkup(pktio[i]);
+ }
+
+ ret = create_packets_sctp(pkt_tbl, pkt_seq, TX_BATCH_LEN, pktio_tx,
+ pktio_rx);
+ CU_ASSERT(ret == TX_BATCH_LEN);
+ if (ret != TX_BATCH_LEN) {
+ for (i = 0; i < num_ifaces; i++) {
+ CU_ASSERT_FATAL(odp_pktio_stop(pktio[i]) == 0);
+ CU_ASSERT_FATAL(odp_pktio_close(pktio[i]) == 0);
+ }
+ return;
+ }
+
+ ret = odp_pktout_queue(pktio_tx, &pktout_queue, 1);
+ CU_ASSERT_FATAL(ret > 0);
+
+ for (i = 0; i < TX_BATCH_LEN; i++)
+ if (prep_fn)
+ prep_fn(pkt_tbl[i]);
+
+ send_packets(pktout_queue, pkt_tbl, TX_BATCH_LEN);
+ num_rx = wait_for_packets_hdr(&pktio_rx_info, pkt_tbl, pkt_seq,
+ TX_BATCH_LEN, TXRX_MODE_MULTI,
+ ODP_TIME_SEC_IN_NS, ODPH_SCTPHDR_LEN);
+ CU_ASSERT(num_rx == TX_BATCH_LEN);
+ for (i = 0; i < num_rx; i++) {
+ test_fn(pkt_tbl[i]);
+ odp_packet_free(pkt_tbl[i]);
+ }
+
+ for (i = 0; i < num_ifaces; i++) {
+ CU_ASSERT_FATAL(odp_pktio_stop(pktio[i]) == 0);
+ CU_ASSERT_FATAL(odp_pktio_close(pktio[i]) == 0);
+ }
+}
+
static int pktio_check_chksum_in_ipv4(void)
{
odp_pktio_t pktio;
@@ -2098,6 +2265,65 @@ static void pktio_test_chksum_in_udp(void)
pktio_test_chksum_in_udp_test);
}
+static int pktio_check_chksum_in_sctp(void)
+{
+ odp_pktio_t pktio;
+ odp_pktio_capability_t capa;
+ odp_pktio_param_t pktio_param;
+ int idx = (num_ifaces == 1) ? 0 : 1;
+ int ret;
+
+ odp_pktio_param_init(&pktio_param);
+ pktio_param.in_mode = ODP_PKTIN_MODE_DIRECT;
+
+ pktio = odp_pktio_open(iface_name[idx], pool[idx], &pktio_param);
+ if (pktio == ODP_PKTIO_INVALID)
+ return ODP_TEST_INACTIVE;
+
+ ret = odp_pktio_capability(pktio, &capa);
+ (void)odp_pktio_close(pktio);
+
+ if (ret < 0 ||
+ !capa.config.pktin.bit.sctp_chksum)
+ return ODP_TEST_INACTIVE;
+
+ return ODP_TEST_ACTIVE;
+}
+
+static void pktio_test_chksum_in_sctp_config(odp_pktio_t pktio_tx ODP_UNUSED,
+ odp_pktio_t pktio_rx)
+{
+ odp_pktio_capability_t capa;
+ odp_pktio_config_t config;
+
+ CU_ASSERT_FATAL(odp_pktio_capability(pktio_rx, &capa) == 0);
+ CU_ASSERT_FATAL(capa.config.pktin.bit.sctp_chksum);
+
+ odp_pktio_config_init(&config);
+ config.pktin.bit.sctp_chksum = 1;
+ CU_ASSERT_FATAL(odp_pktio_config(pktio_rx, &config) == 0);
+}
+
+static void pktio_test_chksum_in_sctp_prep(odp_packet_t pkt)
+{
+ odp_packet_has_ipv4_set(pkt, 1);
+ odp_packet_has_sctp_set(pkt, 1);
+ odph_ipv4_csum_update(pkt);
+ odph_sctp_chksum_set(pkt);
+}
+
+static void pktio_test_chksum_in_sctp_test(odp_packet_t pkt)
+{
+ CU_ASSERT(odp_packet_l4_chksum_status(pkt) == ODP_PACKET_CHKSUM_OK);
+}
+
+static void pktio_test_chksum_in_sctp(void)
+{
+ pktio_test_chksum_sctp(pktio_test_chksum_in_sctp_config,
+ pktio_test_chksum_in_sctp_prep,
+ pktio_test_chksum_in_sctp_test);
+}
+
static int pktio_check_chksum_out_ipv4(void)
{
odp_pktio_t pktio;
@@ -2296,8 +2522,10 @@ static void pktio_test_chksum_out_udp_ovr_test(odp_packet_t pkt)
odph_udphdr_t *udp = odp_packet_l4_ptr(pkt, NULL);
CU_ASSERT(udp != NULL);
- if (udp != NULL)
+ if (udp != NULL) {
CU_ASSERT(udp->chksum != 0);
+ CU_ASSERT(!odph_udp_chksum_verify(pkt));
+ }
}
static void pktio_test_chksum_out_udp_ovr(void)
@@ -2331,6 +2559,126 @@ static void pktio_test_chksum_out_udp_pktio(void)
pktio_test_chksum_out_udp_test);
}
+static int pktio_check_chksum_out_sctp(void)
+{
+ odp_pktio_t pktio;
+ odp_pktio_capability_t capa;
+ odp_pktio_param_t pktio_param;
+ int ret;
+
+ odp_pktio_param_init(&pktio_param);
+ pktio_param.in_mode = ODP_PKTIN_MODE_DIRECT;
+
+ pktio = odp_pktio_open(iface_name[0], pool[0], &pktio_param);
+ if (pktio == ODP_PKTIO_INVALID)
+ return ODP_TEST_INACTIVE;
+
+ ret = odp_pktio_capability(pktio, &capa);
+ (void)odp_pktio_close(pktio);
+
+ if (ret < 0 ||
+ !capa.config.pktout.bit.sctp_chksum_ena ||
+ !capa.config.pktout.bit.sctp_chksum)
+ return ODP_TEST_INACTIVE;
+
+ return ODP_TEST_ACTIVE;
+}
+
+static void pktio_test_chksum_out_sctp_config(odp_pktio_t pktio_tx,
+ odp_pktio_t pktio_rx ODP_UNUSED)
+{
+ odp_pktio_capability_t capa;
+ odp_pktio_config_t config;
+
+ CU_ASSERT_FATAL(odp_pktio_capability(pktio_tx, &capa) == 0);
+ CU_ASSERT_FATAL(capa.config.pktout.bit.sctp_chksum_ena);
+ CU_ASSERT_FATAL(capa.config.pktout.bit.sctp_chksum);
+
+ odp_pktio_config_init(&config);
+ config.pktout.bit.sctp_chksum_ena = 1;
+ CU_ASSERT_FATAL(odp_pktio_config(pktio_tx, &config) == 0);
+}
+
+static void pktio_test_chksum_out_sctp_test(odp_packet_t pkt)
+{
+ odph_sctphdr_t *sctp = odp_packet_l4_ptr(pkt, NULL);
+
+ CU_ASSERT(sctp != NULL);
+ if (sctp != NULL) {
+ CU_ASSERT(sctp->chksum != 0);
+ CU_ASSERT(!odph_sctp_chksum_verify(pkt));
+ }
+}
+
+static void pktio_test_chksum_out_sctp_no_ovr_prep(odp_packet_t pkt)
+{
+ odph_ipv4_csum_update(pkt);
+ odp_packet_l4_chksum_insert(pkt, false);
+}
+
+static void pktio_test_chksum_out_sctp_no_ovr_test(odp_packet_t pkt)
+{
+ odph_sctphdr_t *sctp = odp_packet_l4_ptr(pkt, NULL);
+
+ CU_ASSERT(sctp != NULL);
+ if (sctp != NULL)
+ CU_ASSERT(sctp->chksum == 0);
+}
+
+static void pktio_test_chksum_out_sctp_no_ovr(void)
+{
+ pktio_test_chksum_sctp(pktio_test_chksum_out_sctp_config,
+ pktio_test_chksum_out_sctp_no_ovr_prep,
+ pktio_test_chksum_out_sctp_no_ovr_test);
+}
+
+static void pktio_test_chksum_out_sctp_ovr_prep(odp_packet_t pkt)
+{
+ odp_packet_l4_chksum_insert(pkt, true);
+}
+
+static void pktio_test_chksum_out_sctp_ovr_test(odp_packet_t pkt)
+{
+ odph_sctphdr_t *sctp = odp_packet_l4_ptr(pkt, NULL);
+
+ CU_ASSERT(sctp != NULL);
+ if (sctp != NULL) {
+ CU_ASSERT(sctp->chksum != 0);
+ CU_ASSERT(!odph_sctp_chksum_verify(pkt));
+ }
+}
+
+static void pktio_test_chksum_out_sctp_ovr(void)
+{
+ pktio_test_chksum_sctp(pktio_test_chksum_out_sctp_config,
+ pktio_test_chksum_out_sctp_ovr_prep,
+ pktio_test_chksum_out_sctp_ovr_test);
+}
+
+static void pktio_test_chksum_out_sctp_pktio_config(odp_pktio_t pktio_tx,
+ odp_pktio_t pktio_rx
+ ODP_UNUSED)
+{
+ odp_pktio_capability_t capa;
+ odp_pktio_config_t config;
+
+ CU_ASSERT_FATAL(odp_pktio_capability(pktio_tx, &capa) == 0);
+ CU_ASSERT_FATAL(capa.config.pktout.bit.sctp_chksum_ena);
+ CU_ASSERT_FATAL(capa.config.pktout.bit.sctp_chksum);
+
+ odp_pktio_config_init(&config);
+ config.pktout.bit.sctp_chksum_ena = 1;
+ config.pktout.bit.sctp_chksum = 1;
+ CU_ASSERT_FATAL(odp_pktio_config(pktio_tx, &config) == 0);
+}
+
+static void pktio_test_chksum_out_sctp_pktio(void)
+{
+ pktio_test_chksum_sctp(pktio_test_chksum_out_sctp_pktio_config,
+ NULL,
+ pktio_test_chksum_out_sctp_test);
+}
+
static int create_pool(const char *iface, int num)
{
char pool_name[ODP_POOL_NAME_LEN];
@@ -2475,6 +2823,8 @@ odp_testinfo_t pktio_suite_unsegmented[] = {
pktio_check_chksum_in_ipv4),
ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_in_udp,
pktio_check_chksum_in_udp),
+ ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_in_sctp,
+ pktio_check_chksum_in_sctp),
ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_out_ipv4_no_ovr,
pktio_check_chksum_out_ipv4),
ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_out_ipv4_pktio,
@@ -2487,6 +2837,12 @@ odp_testinfo_t pktio_suite_unsegmented[] = {
pktio_check_chksum_out_udp),
ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_out_udp_ovr,
pktio_check_chksum_out_udp),
+ ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_out_sctp_no_ovr,
+ pktio_check_chksum_out_sctp),
+ ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_out_sctp_pktio,
+ pktio_check_chksum_out_sctp),
+ ODP_TEST_INFO_CONDITIONAL(pktio_test_chksum_out_sctp_ovr,
+ pktio_check_chksum_out_sctp),
ODP_TEST_INFO_NULL
};