aboutsummaryrefslogtreecommitdiff
path: root/platform/linux-dpdk
diff options
context:
space:
mode:
Diffstat (limited to 'platform/linux-dpdk')
-rw-r--r--platform/linux-dpdk/Makefile.am17
l---------platform/linux-dpdk/include/odp/api/deprecated.h1
-rw-r--r--platform/linux-dpdk/include/odp/api/plat/packet_types.h2
l---------[-rw-r--r--]platform/linux-dpdk/include/odp/api/plat/time_types.h44
-rw-r--r--platform/linux-dpdk/include/odp_packet_internal.h79
-rw-r--r--platform/linux-dpdk/include/odp_time_internal.h44
-rw-r--r--platform/linux-dpdk/m4/configure.m445
-rw-r--r--platform/linux-dpdk/odp_crypto.c230
-rw-r--r--platform/linux-dpdk/odp_packet.c341
-rw-r--r--platform/linux-dpdk/odp_packet_dpdk.c22
-rw-r--r--platform/linux-dpdk/odp_packet_flags.c114
-rw-r--r--platform/linux-dpdk/odp_pool.c8
-rw-r--r--platform/linux-dpdk/odp_time.c359
13 files changed, 634 insertions, 672 deletions
diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am
index dfbc57619..c1497f0f6 100644
--- a/platform/linux-dpdk/Makefile.am
+++ b/platform/linux-dpdk/Makefile.am
@@ -42,6 +42,7 @@ odpapiinclude_HEADERS = \
$(srcdir)/include/odp/api/cpumask.h \
$(srcdir)/include/odp/api/crypto.h \
$(srcdir)/include/odp/api/debug.h \
+ $(srcdir)/include/odp/api/deprecated.h \
$(srcdir)/include/odp/api/errno.h \
$(srcdir)/include/odp/api/event.h \
$(srcdir)/include/odp/api/hash.h \
@@ -154,7 +155,7 @@ noinst_HEADERS = \
${top_srcdir}/platform/linux-generic/include/odp_schedule_if.h \
${top_srcdir}/platform/linux-generic/include/odp_sorted_list_internal.h \
${top_srcdir}/platform/linux-generic/include/odp_shm_internal.h \
- ${srcdir}/include/odp_time_internal.h \
+ ${top_srcdir}/platform/linux-generic/include/odp_time_internal.h \
${top_srcdir}/platform/linux-generic/include/odp_timer_internal.h \
${top_srcdir}/platform/linux-generic/include/odp_timer_wheel_internal.h \
${top_srcdir}/platform/linux-generic/include/odp_traffic_mngr_internal.h \
@@ -217,12 +218,18 @@ __LIB__libodp_dpdk_la_SOURCES = \
arch/@ARCH_DIR@/odp_cpu_arch.c \
arch/@ARCH_DIR@/odp_sysinfo_parse.c
+__LIB__libodp_dpdk_la_LIBADD = $(ATOMIC_LIBS)
+
+if ARCH_IS_X86
+__LIB__libodp_dpdk_la_SOURCES += arch/@ARCH_DIR@/cpu_flags.c
+endif
+
# Create symlink for ABI header files. Application does not need to use the arch
# specific include path for installed files.
install-data-hook:
- if [ -h $(prefix)/include/odp/api/abi ]; then \
- : \
+ if [ -h $(DESTDIR)$(prefix)/include/odp/api/abi ]; then \
+ : ; \
else \
- $(LN_S) -rf $(prefix)/include/odp/arch/@ARCH_ABI@/odp/api/abi \
- $(prefix)/include/odp/api/abi; \
+ $(LN_S) -rf $(DESTDIR)$(prefix)/include/odp/arch/@ARCH_ABI@/odp/api/abi \
+ $(DESTDIR)$(prefix)/include/odp/api/abi; \
fi
diff --git a/platform/linux-dpdk/include/odp/api/deprecated.h b/platform/linux-dpdk/include/odp/api/deprecated.h
new file mode 120000
index 000000000..42bc7ef16
--- /dev/null
+++ b/platform/linux-dpdk/include/odp/api/deprecated.h
@@ -0,0 +1 @@
+../../../../linux-generic/include/odp/api/deprecated.h \ No newline at end of file
diff --git a/platform/linux-dpdk/include/odp/api/plat/packet_types.h b/platform/linux-dpdk/include/odp/api/plat/packet_types.h
index c6ee3966f..5df0a41ec 100644
--- a/platform/linux-dpdk/include/odp/api/plat/packet_types.h
+++ b/platform/linux-dpdk/include/odp/api/plat/packet_types.h
@@ -99,8 +99,8 @@ typedef union {
/** All input flags */
uint64_t all;
+ /** Individual input flags */
struct {
- uint64_t parsed_l2:1; /**< L2 parsed */
uint64_t dst_queue:1; /**< Dst queue present */
uint64_t timestamp:1; /**< Timestamp present */
diff --git a/platform/linux-dpdk/include/odp/api/plat/time_types.h b/platform/linux-dpdk/include/odp/api/plat/time_types.h
index e53ad2f97..b42bc83c2 100644..120000
--- a/platform/linux-dpdk/include/odp/api/plat/time_types.h
+++ b/platform/linux-dpdk/include/odp/api/plat/time_types.h
@@ -1,43 +1 @@
-/* Copyright (c) 2016, Linaro Limited
- * All rights reserved.
- *
- * SPDX-License-Identifier: BSD-3-Clause
- */
-
-/**
- * @file
- *
- * ODP time service
- */
-
-#ifndef ODP_TIME_TYPES_H_
-#define ODP_TIME_TYPES_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/** @addtogroup odp_time
- * @{
- **/
-
-/**
- * @internal Time structure used to isolate odp-linux implementation from
- * the linux timespec structure, which is dependent on POSIX extension level.
- */
-typedef struct odp_time_t {
- int64_t tv_sec; /**< @internal Seconds or DPDK ticks */
- int64_t tv_nsec; /**< @internal Nanoseconds */
-} odp_time_t;
-
-#define ODP_TIME_NULL ((odp_time_t){0, 0})
-
-/**
- * @}
- */
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
+../../../../../linux-generic/include/odp/api/plat/time_types.h \ No newline at end of file
diff --git a/platform/linux-dpdk/include/odp_packet_internal.h b/platform/linux-dpdk/include/odp_packet_internal.h
index d3f00847e..481e4afb1 100644
--- a/platform/linux-dpdk/include/odp_packet_internal.h
+++ b/platform/linux-dpdk/include/odp_packet_internal.h
@@ -84,18 +84,6 @@ ODP_STATIC_ASSERT(sizeof(output_flags_t) == sizeof(uint32_t),
"OUTPUT_FLAGS_SIZE_ERROR");
/**
- * Protocol stack layers
- */
-typedef enum {
- LAYER_NONE = 0,
- LAYER_L1,
- LAYER_L2,
- LAYER_L3,
- LAYER_L4,
- LAYER_ALL
-} layer_t;
-
-/**
* Packet parser metadata
*/
typedef struct {
@@ -106,14 +94,6 @@ typedef struct {
uint32_t l2_offset; /**< offset to L2 hdr, e.g. Eth */
uint32_t l3_offset; /**< offset to L3 hdr, e.g. IPv4, IPv6 */
uint32_t l4_offset; /**< offset to L4 hdr (TCP, UDP, SCTP, also ICMP) */
-
- uint32_t l3_len; /**< Layer 3 length */
- uint32_t l4_len; /**< Layer 4 length */
-
- uint16_t ethtype; /**< EtherType */
- uint8_t ip_proto; /**< IP protocol */
- uint8_t parsed_layers; /**< Highest parsed protocol stack layer */
-
} packet_parser_t;
/**
@@ -190,58 +170,25 @@ static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len)
rte_pktmbuf_pkt_len(&pkt_hdr->buf_hdr.mb) = len;
}
-static inline int packet_parse_l2_not_done(packet_parser_t *prs)
-{
- return !prs->input_flags.parsed_l2;
-}
-
-static inline int packet_parse_not_complete(odp_packet_hdr_t *pkt_hdr)
-{
- return pkt_hdr->p.parsed_layers != LAYER_ALL;
-}
-
/* Forward declarations */
int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt);
-/* Fill in parser metadata for L2 */
-static inline void packet_parse_l2(packet_parser_t *prs, uint32_t frame_len)
-{
- /* Packet alloc or reset have already init other offsets and flags */
-
- /* We only support Ethernet for now */
- prs->input_flags.eth = 1;
-
- /* Detect jumbo frames */
- if (frame_len > _ODP_ETH_LEN_MAX)
- prs->input_flags.jumbo = 1;
-
- /* Assume valid L2 header, no CRC/FCS check in SW */
- prs->input_flags.l2 = 1;
-
- prs->input_flags.parsed_l2 = 1;
-}
+/* Perform packet parse up to a given protocol layer */
+int packet_parse_layer(odp_packet_hdr_t *pkt_hdr,
+ odp_pktio_parser_layer_t layer);
-static inline void _odp_packet_reset_parse(odp_packet_t pkt)
+/* Reset parser metadata for a new parse */
+static inline void packet_parse_reset(odp_packet_hdr_t *pkt_hdr)
{
- odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
-
- uint32_t frame_len = rte_pktmbuf_pkt_len(&pkt_hdr->buf_hdr.mb);
-
- pkt_hdr->p.parsed_layers = LAYER_NONE;
- pkt_hdr->p.input_flags.all = 0;
+ /* Reset parser metadata before new parse */
+ pkt_hdr->p.error_flags.all = 0;
+ pkt_hdr->p.input_flags.all = 0;
pkt_hdr->p.output_flags.all = 0;
- pkt_hdr->p.error_flags.all = 0;
- pkt_hdr->p.l2_offset = 0;
-
- packet_parse_l2(&pkt_hdr->p, frame_len);
+ pkt_hdr->p.l2_offset = 0;
+ pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID;
+ pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID;
}
-/* Perform packet parse up to a given protocol layer */
-int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, layer_t layer);
-
-/* Reset parser metadata for a new parse */
-void packet_parse_reset(odp_packet_hdr_t *pkt_hdr);
-
/* Convert a packet handle to a buffer handle */
odp_buffer_t _odp_packet_to_buffer(odp_packet_t pkt);
@@ -277,7 +224,9 @@ static inline void packet_set_ts(odp_packet_hdr_t *pkt_hdr, odp_time_t *ts)
}
int packet_parse_common(packet_parser_t *pkt_hdr, const uint8_t *ptr,
- uint32_t pkt_len, uint32_t seg_len, layer_t layer);
+ uint32_t pkt_len, uint32_t seg_len,
+ odp_pktio_parser_layer_t layer);
+
/* We can't enforce tailroom reservation for received packets */
ODP_STATIC_ASSERT(CONFIG_PACKET_TAILROOM == 0,
diff --git a/platform/linux-dpdk/include/odp_time_internal.h b/platform/linux-dpdk/include/odp_time_internal.h
deleted file mode 100644
index 8cbf81419..000000000
--- a/platform/linux-dpdk/include/odp_time_internal.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/* Copyright (c) 2016, Linaro Limited
- * All rights reserved.
- *
- * SPDX-License-Identifier: BSD-3-Clause
- */
-
-/**
- * @file
- *
- * ODP time service
- */
-
-#ifndef ODP_TIME_INTERNAL_H_
-#define ODP_TIME_INTERNAL_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef uint64_t (*time_to_ns_fn) (odp_time_t time);
-typedef odp_time_t (*time_diff_fn) (odp_time_t t2, odp_time_t t1);
-typedef odp_time_t (*time_curr_fn)(void);
-typedef int (*time_cmp_fn) (odp_time_t t2, odp_time_t t1);
-typedef odp_time_t (*time_sum_fn) (odp_time_t t1, odp_time_t t2);
-typedef odp_time_t (*time_local_from_ns_fn) (uint64_t ns);
-typedef uint64_t (*time_local_res_fn)(void);
-typedef uint64_t (*time_to_u64_fn) (odp_time_t time);
-
-typedef struct time_handler_ {
- time_to_ns_fn time_to_ns;
- time_diff_fn time_diff;
- time_curr_fn time_curr;
- time_cmp_fn time_cmp;
- time_sum_fn time_sum;
- time_local_from_ns_fn time_local_from_ns;
- time_local_res_fn time_local_res;
- time_to_u64_fn time_to_u64;
-} time_handler_t;
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
diff --git a/platform/linux-dpdk/m4/configure.m4 b/platform/linux-dpdk/m4/configure.m4
index fb9a91399..230ef6eff 100644
--- a/platform/linux-dpdk/m4/configure.m4
+++ b/platform/linux-dpdk/m4/configure.m4
@@ -28,6 +28,51 @@ AC_LINK_IFELSE(
echo "Use newer version. For gcc > 4.7.0"
exit -1)
+dnl Check whether -latomic is needed
+use_libatomic=no
+
+AC_MSG_CHECKING(whether -latomic is needed for 64-bit atomic built-ins)
+AC_LINK_IFELSE(
+ [AC_LANG_SOURCE([[
+ static int loc;
+ int main(void)
+ {
+ int prev = __atomic_exchange_n(&loc, 7, __ATOMIC_RELAXED);
+ return 0;
+ }
+ ]])],
+ [AC_MSG_RESULT(no)],
+ [AC_MSG_RESULT(yes)
+ AC_CHECK_LIB(
+ [atomic], [__atomic_exchange_8],
+ [use_libatomic=yes],
+ [AC_MSG_CHECKING([__atomic_exchange_8 is not available])])
+ ])
+
+AC_MSG_CHECKING(whether -latomic is needed for 128-bit atomic built-ins)
+AC_LINK_IFELSE(
+ [AC_LANG_SOURCE([[
+ static __int128 loc;
+ int main(void)
+ {
+ __int128 prev;
+ prev = __atomic_exchange_n(&loc, 7, __ATOMIC_RELAXED);
+ return 0;
+ }
+ ]])],
+ [AC_MSG_RESULT(no)],
+ [AC_MSG_RESULT(yes)
+ AC_CHECK_LIB(
+ [atomic], [__atomic_exchange_16],
+ [use_libatomic=yes],
+ [AC_MSG_CHECKING([cannot detect support for 128-bit atomics])])
+ ])
+
+if test "x$use_libatomic" = "xyes"; then
+ ATOMIC_LIBS="-latomic"
+fi
+AC_SUBST([ATOMIC_LIBS])
+
# linux-generic PCAP support is not relevant as the code doesn't use
# linux-generic pktio at all. And DPDK has its own PCAP support anyway
AM_CONDITIONAL([HAVE_PCAP], [false])
diff --git a/platform/linux-dpdk/odp_crypto.c b/platform/linux-dpdk/odp_crypto.c
index e1f7026ed..d95b15780 100644
--- a/platform/linux-dpdk/odp_crypto.c
+++ b/platform/linux-dpdk/odp_crypto.c
@@ -94,13 +94,15 @@ static int cipher_alg_odp_to_rte(odp_cipher_alg_t cipher_alg,
cipher_xform->cipher.algo = RTE_CRYPTO_CIPHER_3DES_CBC;
break;
case ODP_CIPHER_ALG_AES_CBC:
- /* deprecated */
+#if ODP_DEPRECATED_API
case ODP_CIPHER_ALG_AES128_CBC:
+#endif
cipher_xform->cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC;
break;
case ODP_CIPHER_ALG_AES_GCM:
- /* deprecated */
+#if ODP_DEPRECATED_API
case ODP_CIPHER_ALG_AES128_GCM:
+#endif
cipher_xform->cipher.algo = RTE_CRYPTO_CIPHER_AES_GCM;
break;
default:
@@ -121,20 +123,31 @@ static int auth_alg_odp_to_rte(odp_auth_alg_t auth_alg,
auth_xform->auth.algo = RTE_CRYPTO_AUTH_NULL;
break;
case ODP_AUTH_ALG_MD5_HMAC:
- /* deprecated */
+#if ODP_DEPRECATED_API
case ODP_AUTH_ALG_MD5_96:
+#endif
auth_xform->auth.algo = RTE_CRYPTO_AUTH_MD5_HMAC;
auth_xform->auth.digest_length = 12;
break;
case ODP_AUTH_ALG_SHA256_HMAC:
- /* deprecated */
+#if ODP_DEPRECATED_API
case ODP_AUTH_ALG_SHA256_128:
+#endif
auth_xform->auth.algo = RTE_CRYPTO_AUTH_SHA256_HMAC;
auth_xform->auth.digest_length = 16;
break;
+ case ODP_AUTH_ALG_SHA1_HMAC:
+ auth_xform->auth.algo = RTE_CRYPTO_AUTH_SHA1_HMAC;
+ auth_xform->auth.digest_length = 20;
+ break;
+ case ODP_AUTH_ALG_SHA512_HMAC:
+ auth_xform->auth.algo = RTE_CRYPTO_AUTH_SHA512_HMAC;
+ auth_xform->auth.digest_length = 64;
+ break;
case ODP_AUTH_ALG_AES_GCM:
- /* deprecated */
+#if ODP_DEPRECATED_API
case ODP_AUTH_ALG_AES128_GCM:
+#endif
auth_xform->auth.algo = RTE_CRYPTO_AUTH_AES_GCM;
auth_xform->auth.digest_length = 16;
break;
@@ -151,8 +164,10 @@ static crypto_session_entry_t *alloc_session(void)
odp_spinlock_lock(&global->lock);
session = global->free;
- if (session)
+ if (session) {
global->free = session->next;
+ session->next = NULL;
+ }
odp_spinlock_unlock(&global->lock);
return session;
@@ -349,14 +364,16 @@ int odp_crypto_capability(odp_crypto_capability_t *capability)
if (cap_cipher_algo ==
RTE_CRYPTO_CIPHER_AES_CBC) {
hw_ciphers->bit.aes_cbc = 1;
- /* Deprecated */
+#if ODP_DEPRECATED_API
hw_ciphers->bit.aes128_cbc = 1;
+#endif
}
if (cap_cipher_algo ==
RTE_CRYPTO_CIPHER_AES_GCM) {
hw_ciphers->bit.aes_gcm = 1;
- /* Deprecated */
+#if ODP_DEPRECATED_API
hw_ciphers->bit.aes128_gcm = 1;
+#endif
}
}
@@ -373,20 +390,31 @@ int odp_crypto_capability(odp_crypto_capability_t *capability)
if (cap_auth_algo ==
RTE_CRYPTO_AUTH_AES_GCM) {
hw_auths->bit.aes_gcm = 1;
- /* Deprecated */
+#if ODP_DEPRECATED_API
hw_auths->bit.aes128_gcm = 1;
+#endif
}
if (cap_auth_algo ==
RTE_CRYPTO_AUTH_MD5_HMAC) {
hw_auths->bit.md5_hmac = 1;
- /* Deprecated */
+#if ODP_DEPRECATED_API
hw_auths->bit.md5_96 = 1;
+#endif
}
if (cap_auth_algo ==
RTE_CRYPTO_AUTH_SHA256_HMAC) {
hw_auths->bit.sha256_hmac = 1;
- /* Deprecated */
+#if ODP_DEPRECATED_API
hw_auths->bit.sha256_128 = 1;
+#endif
+ }
+ if (cap_auth_algo ==
+ RTE_CRYPTO_AUTH_SHA1_HMAC) {
+ hw_auths->bit.sha1_hmac = 1;
+ }
+ if (cap_auth_algo ==
+ RTE_CRYPTO_AUTH_SHA512_HMAC) {
+ hw_auths->bit.sha512_hmac = 1;
}
}
cap = &dev_info.capabilities[++i];
@@ -411,14 +439,16 @@ int odp_crypto_capability(odp_crypto_capability_t *capability)
if (cap_cipher_algo ==
RTE_CRYPTO_CIPHER_AES_CBC) {
ciphers->bit.aes_cbc = 1;
- /* Deprecated */
+#if ODP_DEPRECATED_API
ciphers->bit.aes128_cbc = 1;
+#endif
}
if (cap_cipher_algo ==
RTE_CRYPTO_CIPHER_AES_GCM) {
ciphers->bit.aes_gcm = 1;
- /* Deprecated */
+#if ODP_DEPRECATED_API
ciphers->bit.aes128_gcm = 1;
+#endif
}
}
@@ -435,20 +465,31 @@ int odp_crypto_capability(odp_crypto_capability_t *capability)
if (cap_auth_algo ==
RTE_CRYPTO_AUTH_AES_GCM) {
auths->bit.aes_gcm = 1;
- /* Deprecated */
+#if ODP_DEPRECATED_API
auths->bit.aes128_gcm = 1;
+#endif
}
if (cap_auth_algo ==
RTE_CRYPTO_AUTH_MD5_HMAC) {
auths->bit.md5_hmac = 1;
- /* Deprecated */
+#if ODP_DEPRECATED_API
auths->bit.md5_96 = 1;
+#endif
}
if (cap_auth_algo ==
RTE_CRYPTO_AUTH_SHA256_HMAC) {
auths->bit.sha256_hmac = 1;
- /* Deprecated */
+#if ODP_DEPRECATED_API
auths->bit.sha256_128 = 1;
+#endif
+ }
+ if (cap_auth_algo ==
+ RTE_CRYPTO_AUTH_SHA1_HMAC) {
+ auths->bit.sha1_hmac = 1;
+ }
+ if (cap_auth_algo ==
+ RTE_CRYPTO_AUTH_SHA512_HMAC) {
+ auths->bit.sha512_hmac = 1;
}
}
cap = &dev_info.capabilities[++i];
@@ -547,7 +588,7 @@ int odp_crypto_cipher_capability(odp_cipher_alg_t cipher,
int odp_crypto_auth_capability(odp_auth_alg_t auth,
odp_crypto_auth_capability_t dst[],
- int num_copy)
+ int num_copy)
{
odp_crypto_auth_capability_t src[num_copy];
int idx = 0, rc = 0;
@@ -740,7 +781,7 @@ static int get_crypto_dev(struct rte_crypto_sym_xform *cipher_xform,
return -1;
}
-int odp_crypto_session_create(odp_crypto_session_params_t *params,
+int odp_crypto_session_create(odp_crypto_session_param_t *param,
odp_crypto_session_t *session_out,
odp_crypto_ses_create_err_t *status)
{
@@ -771,7 +812,7 @@ int odp_crypto_session_create(odp_crypto_session_params_t *params,
/* Cipher Data */
cipher_xform.cipher.key.data = rte_malloc("crypto key",
- params->cipher_key.length, 0);
+ param->cipher_key.length, 0);
if (cipher_xform.cipher.key.data == NULL) {
ODP_ERR("Failed to allocate memory for cipher key\n");
/* remove the crypto_session_entry_t */
@@ -782,14 +823,14 @@ int odp_crypto_session_create(odp_crypto_session_params_t *params,
cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
cipher_xform.next = NULL;
- cipher_xform.cipher.key.length = params->cipher_key.length;
+ cipher_xform.cipher.key.length = param->cipher_key.length;
memcpy(cipher_xform.cipher.key.data,
- params->cipher_key.data,
- params->cipher_key.length);
+ param->cipher_key.data,
+ param->cipher_key.length);
/* Authentication Data */
auth_xform.auth.key.data = rte_malloc("auth key",
- params->auth_key.length, 0);
+ param->auth_key.length, 0);
if (auth_xform.auth.key.data == NULL) {
ODP_ERR("Failed to allocate memory for auth key\n");
/* remove the crypto_session_entry_t */
@@ -799,16 +840,16 @@ int odp_crypto_session_create(odp_crypto_session_params_t *params,
}
auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH;
auth_xform.next = NULL;
- auth_xform.auth.key.length = params->auth_key.length;
+ auth_xform.auth.key.length = param->auth_key.length;
memcpy(auth_xform.auth.key.data,
- params->auth_key.data,
- params->auth_key.length);
+ param->auth_key.data,
+ param->auth_key.length);
/* Derive order */
- if (ODP_CRYPTO_OP_ENCODE == params->op)
- entry->do_cipher_first = params->auth_cipher_text;
+ if (ODP_CRYPTO_OP_ENCODE == param->op)
+ entry->do_cipher_first = param->auth_cipher_text;
else
- entry->do_cipher_first = !params->auth_cipher_text;
+ entry->do_cipher_first = !param->auth_cipher_text;
/* Process based on cipher */
/* Derive order */
@@ -824,7 +865,7 @@ int odp_crypto_session_create(odp_crypto_session_params_t *params,
first_xform->next = &cipher_xform;
}
- rc = cipher_alg_odp_to_rte(params->cipher_alg, &cipher_xform);
+ rc = cipher_alg_odp_to_rte(param->cipher_alg, &cipher_xform);
/* Check result */
if (rc) {
@@ -832,7 +873,7 @@ int odp_crypto_session_create(odp_crypto_session_params_t *params,
return -1;
}
- rc = auth_alg_odp_to_rte(params->auth_alg, &auth_xform);
+ rc = auth_alg_odp_to_rte(param->auth_alg, &auth_xform);
/* Check result */
if (rc) {
@@ -845,7 +886,7 @@ int odp_crypto_session_create(odp_crypto_session_params_t *params,
rc = get_crypto_dev(&cipher_xform,
&auth_xform,
- params->iv.length,
+ param->iv.length,
&cdev_id);
if (rc) {
@@ -868,10 +909,10 @@ int odp_crypto_session_create(odp_crypto_session_params_t *params,
entry->rte_session = (intptr_t)session;
entry->cipher_xform = cipher_xform;
entry->auth_xform = auth_xform;
- entry->iv.length = params->iv.length;
- entry->iv.data = params->iv.data;
- entry->output_pool = params->output_pool;
- entry->compl_queue = params->compl_queue;
+ entry->iv.length = param->iv.length;
+ entry->iv.data = param->iv.data;
+ entry->output_pool = param->output_pool;
+ entry->compl_queue = param->compl_queue;
/* We're happy */
*session_out = (intptr_t)entry;
@@ -903,7 +944,7 @@ int odp_crypto_session_destroy(odp_crypto_session_t session)
return 0;
}
-int odp_crypto_operation(odp_crypto_op_params_t *params,
+int odp_crypto_operation(odp_crypto_op_param_t *param,
odp_bool_t *posted,
odp_crypto_op_result_t *result)
{
@@ -917,10 +958,10 @@ int odp_crypto_operation(odp_crypto_op_params_t *params,
uint8_t *data_addr, *aad_head;
struct rte_crypto_op *op;
uint16_t rc;
- uint32_t plain_len, aad_len;
- odp_bool_t pkt_allocated = 0;
+ uint32_t aad_len;
+ odp_bool_t allocated = false;
- entry = (crypto_session_entry_t *)(intptr_t)params->session;
+ entry = (crypto_session_entry_t *)(intptr_t)param->session;
if (entry == NULL)
return -1;
@@ -935,71 +976,71 @@ int odp_crypto_operation(odp_crypto_op_params_t *params,
auth_xform = entry->auth_xform;
/* Resolve output buffer */
- if (ODP_PACKET_INVALID == params->out_pkt &&
+ if (ODP_PACKET_INVALID == param->out_pkt &&
ODP_POOL_INVALID != entry->output_pool) {
- params->out_pkt = odp_packet_alloc(entry->output_pool,
- odp_packet_len(params->pkt));
- pkt_allocated = 1;
+ param->out_pkt = odp_packet_alloc(entry->output_pool,
+ odp_packet_len(param->pkt));
+ allocated = true;
}
- if (params->pkt != params->out_pkt) {
- if (odp_unlikely(ODP_PACKET_INVALID == params->out_pkt))
+ if (param->pkt != param->out_pkt) {
+ if (odp_unlikely(ODP_PACKET_INVALID == param->out_pkt))
ODP_ABORT();
- (void)odp_packet_copy_from_pkt(params->out_pkt,
+ int ret;
+
+ ret = odp_packet_copy_from_pkt(param->out_pkt,
0,
- params->pkt,
+ param->pkt,
0,
- odp_packet_len(params->pkt));
- _odp_packet_copy_md_to_packet(params->pkt, params->out_pkt);
- odp_packet_free(params->pkt);
- params->pkt = ODP_PACKET_INVALID;
+ odp_packet_len(param->pkt));
+ if (odp_unlikely(ret < 0))
+ goto err;
+
+ _odp_packet_copy_md_to_packet(param->pkt, param->out_pkt);
+ odp_packet_free(param->pkt);
+ param->pkt = ODP_PACKET_INVALID;
}
- data_addr = odp_packet_data(params->out_pkt);
+ data_addr = odp_packet_data(param->out_pkt);
odp_spinlock_init(&global->lock);
odp_spinlock_lock(&global->lock);
op = rte_crypto_op_alloc(global->crypto_op_pool,
RTE_CRYPTO_OP_TYPE_SYMMETRIC);
if (op == NULL) {
- if (pkt_allocated)
- odp_packet_free(params->out_pkt);
ODP_ERR("Failed to allocate crypto operation");
- return -1;
+ goto err;
}
odp_spinlock_unlock(&global->lock);
/* Set crypto operation data parameters */
rte_crypto_op_attach_sym_session(op, rte_session);
- op->sym->auth.digest.data = data_addr + params->hash_result_offset;
+ op->sym->auth.digest.data = data_addr + param->hash_result_offset;
op->sym->auth.digest.phys_addr =
- rte_pktmbuf_mtophys_offset((struct rte_mbuf *)params->out_pkt,
- odp_packet_len(params->out_pkt) -
+ rte_pktmbuf_mtophys_offset((struct rte_mbuf *)param->out_pkt,
+ odp_packet_len(param->out_pkt) -
auth_xform.auth.digest_length);
op->sym->auth.digest.length = auth_xform.auth.digest_length;
/* For SNOW3G algorithms, offset/length must be in bits */
if (auth_xform.auth.algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
- op->sym->auth.data.offset = params->auth_range.offset << 3;
- op->sym->auth.data.length = params->auth_range.length << 3;
+ op->sym->auth.data.offset = param->auth_range.offset << 3;
+ op->sym->auth.data.length = param->auth_range.length << 3;
} else {
- op->sym->auth.data.offset = params->auth_range.offset;
- op->sym->auth.data.length = params->auth_range.length;
+ op->sym->auth.data.offset = param->auth_range.offset;
+ op->sym->auth.data.length = param->auth_range.length;
}
- aad_head = data_addr + params->auth_range.offset;
- plain_len = params->cipher_range.length;
- aad_len = params->auth_range.length - plain_len;
+ aad_head = param->aad.ptr;
+ aad_len = param->aad.length;
if (aad_len > 0) {
op->sym->auth.aad.data = rte_malloc("aad", aad_len, 0);
if (op->sym->auth.aad.data == NULL) {
rte_crypto_op_free(op);
- if (pkt_allocated)
- odp_packet_free(params->out_pkt);
ODP_ERR("Failed to allocate memory for AAD");
- return -1;
+ goto err;
}
memcpy(op->sym->auth.aad.data, aad_head, aad_len);
@@ -1010,24 +1051,20 @@ int odp_crypto_operation(odp_crypto_op_params_t *params,
if (entry->iv.length == 0) {
rte_crypto_op_free(op);
- if (pkt_allocated)
- odp_packet_free(params->out_pkt);
ODP_ERR("Wrong IV length");
- return -1;
+ goto err;
}
op->sym->cipher.iv.data = rte_malloc("iv", entry->iv.length, 0);
if (op->sym->cipher.iv.data == NULL) {
rte_crypto_op_free(op);
- if (pkt_allocated)
- odp_packet_free(params->out_pkt);
ODP_ERR("Failed to allocate memory for IV");
- return -1;
+ goto err;
}
- if (params->override_iv_ptr) {
+ if (param->override_iv_ptr) {
memcpy(op->sym->cipher.iv.data,
- params->override_iv_ptr,
+ param->override_iv_ptr,
entry->iv.length);
} else if (entry->iv.data) {
memcpy(op->sym->cipher.iv.data,
@@ -1043,27 +1080,25 @@ int odp_crypto_operation(odp_crypto_op_params_t *params,
/* For SNOW3G algorithms, offset/length must be in bits */
if (cipher_xform.cipher.algo == RTE_CRYPTO_CIPHER_SNOW3G_UEA2) {
- op->sym->cipher.data.offset = params->cipher_range.offset << 3;
- op->sym->cipher.data.length = params->cipher_range.length << 3;
+ op->sym->cipher.data.offset = param->cipher_range.offset << 3;
+ op->sym->cipher.data.length = param->cipher_range.length << 3;
} else {
- op->sym->cipher.data.offset = params->cipher_range.offset;
- op->sym->cipher.data.length = params->cipher_range.length;
+ op->sym->cipher.data.offset = param->cipher_range.offset;
+ op->sym->cipher.data.length = param->cipher_range.length;
}
if (rc_cipher == ODP_CRYPTO_ALG_ERR_NONE &&
rc_auth == ODP_CRYPTO_ALG_ERR_NONE) {
int queue_pair = odp_cpu_id();
- op->sym->m_src = (struct rte_mbuf *)params->out_pkt;
+ op->sym->m_src = (struct rte_mbuf *)param->out_pkt;
rc = rte_cryptodev_enqueue_burst(rte_session->dev_id,
queue_pair, &op, 1);
if (rc == 0) {
rte_crypto_op_free(op);
- if (pkt_allocated)
- odp_packet_free(params->out_pkt);
ODP_ERR("Failed to enqueue packet");
- return -1;
+ goto err;
}
rc = rte_cryptodev_dequeue_burst(rte_session->dev_id,
@@ -1071,18 +1106,16 @@ int odp_crypto_operation(odp_crypto_op_params_t *params,
if (rc == 0) {
rte_crypto_op_free(op);
- if (pkt_allocated)
- odp_packet_free(params->out_pkt);
ODP_ERR("Failed to dequeue packet");
- return -1;
+ goto err;
}
- params->out_pkt = (odp_packet_t)op->sym->m_src;
+ param->out_pkt = (odp_packet_t)op->sym->m_src;
}
/* Fill in result */
- local_result.ctx = params->ctx;
- local_result.pkt = params->out_pkt;
+ local_result.ctx = param->ctx;
+ local_result.pkt = param->out_pkt;
local_result.cipher_status.alg_err = rc_cipher;
local_result.cipher_status.hw_err = ODP_CRYPTO_HW_ERR_NONE;
local_result.auth_status.alg_err = rc_auth;
@@ -1098,7 +1131,7 @@ int odp_crypto_operation(odp_crypto_op_params_t *params,
odp_event_t completion_event;
odp_crypto_generic_op_result_t *op_result;
- completion_event = odp_packet_to_event(params->out_pkt);
+ completion_event = odp_packet_to_event(param->out_pkt);
_odp_buffer_event_type_set(
odp_buffer_from_event(completion_event),
ODP_EVENT_CRYPTO_COMPL);
@@ -1108,7 +1141,7 @@ int odp_crypto_operation(odp_crypto_op_params_t *params,
op_result->result = local_result;
if (odp_queue_enq(entry->compl_queue, completion_event)) {
odp_event_free(completion_event);
- return -1;
+ goto err;
}
/* Indicate to caller operation was async */
@@ -1116,14 +1149,21 @@ int odp_crypto_operation(odp_crypto_op_params_t *params,
} else {
/* Synchronous, simply return results */
if (!result)
- return -1;
+ goto err;
*result = local_result;
/* Indicate to caller operation was sync */
*posted = 0;
}
-
return 0;
+
+err:
+ if (allocated) {
+ odp_packet_free(param->out_pkt);
+ param->out_pkt = ODP_PACKET_INVALID;
+ }
+
+ return -1;
}
int odp_crypto_term_global(void)
diff --git a/platform/linux-dpdk/odp_packet.c b/platform/linux-dpdk/odp_packet.c
index 68ad26b29..0d05e23f2 100644
--- a/platform/linux-dpdk/odp_packet.c
+++ b/platform/linux-dpdk/odp_packet.c
@@ -21,6 +21,8 @@
#include <stddef.h>
#include <inttypes.h>
+#include <odp/visibility_begin.h>
+
/* Fill in packet header field offsets for inline functions */
const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = {
@@ -49,6 +51,8 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = {
.rss_flag = PKT_RX_RSS_HASH
};
+#include <odp/visibility_end.h>
+
struct rte_mbuf dummy;
ODP_STATIC_ASSERT(sizeof(dummy.data_off) == sizeof(uint16_t),
"data_off should be uint16_t");
@@ -77,24 +81,6 @@ static inline odp_packet_hdr_t *buf_to_packet_hdr(odp_buffer_t buf)
return (odp_packet_hdr_t *)buf_hdl_to_hdr(buf);
}
-static inline void packet_parse_disable(odp_packet_hdr_t *pkt_hdr)
-{
- pkt_hdr->p.input_flags.parsed_l2 = 1;
- pkt_hdr->p.parsed_layers = LAYER_ALL;
-}
-
-void packet_parse_reset(odp_packet_hdr_t *pkt_hdr)
-{
- /* Reset parser metadata before new parse */
- pkt_hdr->p.parsed_layers = LAYER_NONE;
- pkt_hdr->p.error_flags.all = 0;
- pkt_hdr->p.input_flags.all = 0;
- pkt_hdr->p.output_flags.all = 0;
- pkt_hdr->p.l2_offset = 0;
- pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID;
- pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID;
-}
-
static odp_packet_t packet_alloc(pool_entry_t* pool, uint32_t len)
{
odp_packet_t pkt;
@@ -194,7 +180,6 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len)
return -1;
}
- pkt_hdr->p.parsed_layers = LAYER_NONE;
pkt_hdr->p.input_flags.all = 0;
pkt_hdr->p.output_flags.all = 0;
pkt_hdr->p.error_flags.all = 0;
@@ -207,9 +192,6 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len)
pkt_hdr->input = ODP_PKTIO_INVALID;
- /* Disable lazy parsing on user allocated packets */
- packet_parse_disable(pkt_hdr);
-
mb->port = 0xff;
mb->pkt_len = len;
mb->data_off = RTE_PKTMBUF_HEADROOM;
@@ -623,8 +605,6 @@ void *odp_packet_l3_ptr(odp_packet_t pkt, uint32_t *len)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- if (pkt_hdr->p.parsed_layers < LAYER_L3)
- packet_parse_layer(pkt_hdr, LAYER_L3);
return packet_offset_to_ptr(pkt, len, pkt_hdr->p.l3_offset);
}
@@ -632,8 +612,6 @@ uint32_t odp_packet_l3_offset(odp_packet_t pkt)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- if (pkt_hdr->p.parsed_layers < LAYER_L3)
- packet_parse_layer(pkt_hdr, LAYER_L3);
return pkt_hdr->p.l3_offset;
}
@@ -644,8 +622,6 @@ int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset)
if (odp_unlikely(offset >= (odp_packet_len(pkt) - 1)))
return -1;
- if (pkt_hdr->p.parsed_layers < LAYER_L3)
- packet_parse_layer(pkt_hdr, LAYER_L3);
pkt_hdr->p.l3_offset = offset;
return 0;
}
@@ -654,8 +630,6 @@ void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- if (pkt_hdr->p.parsed_layers < LAYER_L4)
- packet_parse_layer(pkt_hdr, LAYER_L4);
return packet_offset_to_ptr(pkt, len, pkt_hdr->p.l4_offset);
}
@@ -663,8 +637,6 @@ uint32_t odp_packet_l4_offset(odp_packet_t pkt)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- if (pkt_hdr->p.parsed_layers < LAYER_L4)
- packet_parse_layer(pkt_hdr, LAYER_L4);
return pkt_hdr->p.l4_offset;
}
@@ -675,8 +647,6 @@ int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset)
if (odp_unlikely(offset >= (odp_packet_len(pkt) - 1)))
return -1;
- if (pkt_hdr->p.parsed_layers < LAYER_L4)
- packet_parse_layer(pkt_hdr, LAYER_L4);
pkt_hdr->p.l4_offset = offset;
return 0;
}
@@ -1148,12 +1118,11 @@ static inline uint8_t parse_ipv4(packet_parser_t *prs, const uint8_t **parseptr,
uint8_t ihl = _ODP_IPV4HDR_IHL(ipv4->ver_ihl);
uint16_t frag_offset;
uint32_t dstaddr = odp_be_to_cpu_32(ipv4->dst_addr);
-
- prs->l3_len = odp_be_to_cpu_16(ipv4->tot_len);
+ uint32_t l3_len = odp_be_to_cpu_16(ipv4->tot_len);
if (odp_unlikely(ihl < _ODP_IPV4HDR_IHL_MIN) ||
odp_unlikely(ver != 4) ||
- (prs->l3_len > frame_len - *offset)) {
+ (l3_len > frame_len - *offset)) {
prs->error_flags.ip_err = 1;
return 0;
}
@@ -1190,13 +1159,12 @@ static inline uint8_t parse_ipv6(packet_parser_t *prs, const uint8_t **parseptr,
const _odp_ipv6hdr_t *ipv6 = (const _odp_ipv6hdr_t *)*parseptr;
const _odp_ipv6hdr_ext_t *ipv6ext;
uint32_t dstaddr0 = odp_be_to_cpu_32(ipv6->dst_addr.u8[0]);
-
- prs->l3_len = odp_be_to_cpu_16(ipv6->payload_len) +
- _ODP_IPV6HDR_LEN;
+ uint32_t l3_len = odp_be_to_cpu_16(ipv6->payload_len) +
+ _ODP_IPV6HDR_LEN;
/* Basic sanity checks on IPv6 header */
if ((odp_be_to_cpu_32(ipv6->ver_tc_flow) >> 28) != 6 ||
- prs->l3_len > frame_len - *offset) {
+ l3_len > frame_len - *offset) {
prs->error_flags.ip_err = 1;
return 0;
}
@@ -1257,9 +1225,6 @@ static inline void parse_tcp(packet_parser_t *prs,
else if ((uint32_t)tcp->hl * 4 > sizeof(_odp_tcphdr_t))
prs->input_flags.tcpopt = 1;
- prs->l4_len = prs->l3_len +
- prs->l3_offset - prs->l4_offset;
-
if (offset)
*offset += (uint32_t)tcp->hl * 4;
*parseptr += (uint32_t)tcp->hl * 4;
@@ -1274,13 +1239,8 @@ static inline void parse_udp(packet_parser_t *prs,
const _odp_udphdr_t *udp = (const _odp_udphdr_t *)*parseptr;
uint32_t udplen = odp_be_to_cpu_16(udp->length);
- if (udplen < sizeof(_odp_udphdr_t) ||
- udplen > (prs->l3_len +
- prs->l4_offset - prs->l3_offset)) {
+ if (odp_unlikely(udplen < sizeof(_odp_udphdr_t)))
prs->error_flags.udp_err = 1;
- }
-
- prs->l4_len = udplen;
if (offset)
*offset += sizeof(_odp_udphdr_t);
@@ -1294,197 +1254,172 @@ static inline void parse_udp(packet_parser_t *prs,
* available from the ptr.
*/
int packet_parse_common(packet_parser_t *prs, const uint8_t *ptr,
- uint32_t frame_len, uint32_t seg_len, layer_t layer)
+ uint32_t frame_len, uint32_t seg_len,
+ odp_pktio_parser_layer_t layer)
{
uint32_t offset;
+ uint16_t ethtype;
const uint8_t *parseptr;
+ uint8_t ip_proto;
+ const _odp_ethhdr_t *eth;
+ uint16_t macaddr0, macaddr2, macaddr4;
+ const _odp_vlanhdr_t *vlan;
- switch (prs->parsed_layers) {
- case LAYER_NONE:
- /* Fall through */
+ if (layer == ODP_PKTIO_PARSER_LAYER_NONE)
+ return 0;
- case LAYER_L2:
- {
- const _odp_ethhdr_t *eth;
- uint16_t macaddr0, macaddr2, macaddr4;
- const _odp_vlanhdr_t *vlan;
-
- offset = sizeof(_odp_ethhdr_t);
- if (packet_parse_l2_not_done(prs))
- packet_parse_l2(prs, frame_len);
-
- eth = (const _odp_ethhdr_t *)ptr;
-
- /* Handle Ethernet broadcast/multicast addresses */
- macaddr0 = odp_be_to_cpu_16(*((const uint16_t *)
- (const void *)eth));
- prs->input_flags.eth_mcast = (macaddr0 & 0x0100) == 0x0100;
-
- if (macaddr0 == 0xffff) {
- macaddr2 =
- odp_be_to_cpu_16(*((const uint16_t *)
- (const void *)eth + 1));
- macaddr4 =
- odp_be_to_cpu_16(*((const uint16_t *)
- (const void *)eth + 2));
- prs->input_flags.eth_bcast =
- (macaddr2 == 0xffff) && (macaddr4 == 0xffff);
- } else {
- prs->input_flags.eth_bcast = 0;
- }
+ /* We only support Ethernet for now */
+ prs->input_flags.eth = 1;
+ /* Assume valid L2 header, no CRC/FCS check in SW */
+ prs->input_flags.l2 = 1;
+ /* Detect jumbo frames */
+ if (frame_len > _ODP_ETH_LEN_MAX)
+ prs->input_flags.jumbo = 1;
+
+ offset = sizeof(_odp_ethhdr_t);
+ eth = (const _odp_ethhdr_t *)ptr;
+
+ /* Handle Ethernet broadcast/multicast addresses */
+ macaddr0 = odp_be_to_cpu_16(*((const uint16_t *)(const void *)eth));
+ prs->input_flags.eth_mcast = (macaddr0 & 0x0100) == 0x0100;
+
+ if (macaddr0 == 0xffff) {
+ macaddr2 =
+ odp_be_to_cpu_16(*((const uint16_t *)
+ (const void *)eth + 1));
+ macaddr4 =
+ odp_be_to_cpu_16(*((const uint16_t *)
+ (const void *)eth + 2));
+ prs->input_flags.eth_bcast =
+ (macaddr2 == 0xffff) && (macaddr4 == 0xffff);
+ } else {
+ prs->input_flags.eth_bcast = 0;
+ }
- /* Get Ethertype */
- prs->ethtype = odp_be_to_cpu_16(eth->type);
- parseptr = (const uint8_t *)(eth + 1);
+ /* Get Ethertype */
+ ethtype = odp_be_to_cpu_16(eth->type);
+ parseptr = (const uint8_t *)(eth + 1);
- /* Check for SNAP vs. DIX */
- if (prs->ethtype < _ODP_ETH_LEN_MAX) {
- prs->input_flags.snap = 1;
- if (prs->ethtype > frame_len - offset) {
- prs->error_flags.snap_len = 1;
- goto parse_exit;
- }
- prs->ethtype = odp_be_to_cpu_16(*((const uint16_t *)
- (uintptr_t)
- (parseptr + 6)));
- offset += 8;
- parseptr += 8;
+ /* Check for SNAP vs. DIX */
+ if (ethtype < _ODP_ETH_LEN_MAX) {
+ prs->input_flags.snap = 1;
+ if (ethtype > frame_len - offset) {
+ prs->error_flags.snap_len = 1;
+ goto parse_exit;
}
+ ethtype = odp_be_to_cpu_16(*((const uint16_t *)(uintptr_t)
+ (parseptr + 6)));
+ offset += 8;
+ parseptr += 8;
+ }
- /* Parse the VLAN header(s), if present */
- if (prs->ethtype == _ODP_ETHTYPE_VLAN_OUTER) {
- prs->input_flags.vlan_qinq = 1;
- prs->input_flags.vlan = 1;
-
- vlan = (const _odp_vlanhdr_t *)parseptr;
- prs->ethtype = odp_be_to_cpu_16(vlan->type);
- offset += sizeof(_odp_vlanhdr_t);
- parseptr += sizeof(_odp_vlanhdr_t);
- }
+ /* Parse the VLAN header(s), if present */
+ if (ethtype == _ODP_ETHTYPE_VLAN_OUTER) {
+ prs->input_flags.vlan_qinq = 1;
+ prs->input_flags.vlan = 1;
- if (prs->ethtype == _ODP_ETHTYPE_VLAN) {
- prs->input_flags.vlan = 1;
- vlan = (const _odp_vlanhdr_t *)parseptr;
- prs->ethtype = odp_be_to_cpu_16(vlan->type);
- offset += sizeof(_odp_vlanhdr_t);
- parseptr += sizeof(_odp_vlanhdr_t);
- }
+ vlan = (const _odp_vlanhdr_t *)parseptr;
+ ethtype = odp_be_to_cpu_16(vlan->type);
+ offset += sizeof(_odp_vlanhdr_t);
+ parseptr += sizeof(_odp_vlanhdr_t);
+ }
- prs->l3_offset = offset;
- prs->parsed_layers = LAYER_L2;
- if (layer == LAYER_L2)
- return prs->error_flags.all != 0;
+ if (ethtype == _ODP_ETHTYPE_VLAN) {
+ prs->input_flags.vlan = 1;
+ vlan = (const _odp_vlanhdr_t *)parseptr;
+ ethtype = odp_be_to_cpu_16(vlan->type);
+ offset += sizeof(_odp_vlanhdr_t);
+ parseptr += sizeof(_odp_vlanhdr_t);
}
- /* Fall through */
- case LAYER_L3:
- {
- offset = prs->l3_offset;
- parseptr = (const uint8_t *)(ptr + offset);
- /* Set l3_offset+flag only for known ethtypes */
- prs->input_flags.l3 = 1;
-
- /* Parse Layer 3 headers */
- switch (prs->ethtype) {
- case _ODP_ETHTYPE_IPV4:
- prs->input_flags.ipv4 = 1;
- prs->ip_proto = parse_ipv4(prs, &parseptr, &offset,
- frame_len);
- break;
+ if (layer == ODP_PKTIO_PARSER_LAYER_L2)
+ return prs->error_flags.all != 0;
- case _ODP_ETHTYPE_IPV6:
- prs->input_flags.ipv6 = 1;
- prs->ip_proto = parse_ipv6(prs, &parseptr, &offset,
- frame_len, seg_len);
- break;
+ /* Set l3_offset+flag only for known ethtypes */
+ prs->l3_offset = offset;
+ prs->input_flags.l3 = 1;
- case _ODP_ETHTYPE_ARP:
- prs->input_flags.arp = 1;
- prs->ip_proto = 255; /* Reserved invalid by IANA */
- break;
+ /* Parse Layer 3 headers */
+ switch (ethtype) {
+ case _ODP_ETHTYPE_IPV4:
+ prs->input_flags.ipv4 = 1;
+ ip_proto = parse_ipv4(prs, &parseptr, &offset, frame_len);
+ break;
- default:
- prs->input_flags.l3 = 0;
- prs->l3_offset = ODP_PACKET_OFFSET_INVALID;
- prs->ip_proto = 255; /* Reserved invalid by IANA */
- }
+ case _ODP_ETHTYPE_IPV6:
+ prs->input_flags.ipv6 = 1;
+ ip_proto = parse_ipv6(prs, &parseptr, &offset, frame_len,
+ seg_len);
+ break;
- /* Set l4_offset+flag only for known ip_proto */
- prs->l4_offset = offset;
- prs->parsed_layers = LAYER_L3;
- if (layer == LAYER_L3)
- return prs->error_flags.all != 0;
- }
- /* Fall through */
+ case _ODP_ETHTYPE_ARP:
+ prs->input_flags.arp = 1;
+ ip_proto = 255; /* Reserved invalid by IANA */
+ break;
- case LAYER_L4:
- {
- offset = prs->l4_offset;
- parseptr = (const uint8_t *)(ptr + offset);
- prs->input_flags.l4 = 1;
+ default:
+ prs->input_flags.l3 = 0;
+ prs->l3_offset = ODP_PACKET_OFFSET_INVALID;
+ ip_proto = 255; /* Reserved invalid by IANA */
+ }
- /* Parse Layer 4 headers */
- switch (prs->ip_proto) {
- case _ODP_IPPROTO_ICMP:
- prs->input_flags.icmp = 1;
- break;
+ if (layer == ODP_PKTIO_PARSER_LAYER_L3)
+ return prs->error_flags.all != 0;
- case _ODP_IPPROTO_TCP:
- if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len))
- return -1;
- prs->input_flags.tcp = 1;
- parse_tcp(prs, &parseptr, NULL);
- break;
+ /* Set l4_offset+flag only for known ip_proto */
+ prs->l4_offset = offset;
+ prs->input_flags.l4 = 1;
- case _ODP_IPPROTO_UDP:
- if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len))
- return -1;
- prs->input_flags.udp = 1;
- parse_udp(prs, &parseptr, NULL);
- break;
+ /* Parse Layer 4 headers */
+ switch (ip_proto) {
+ case _ODP_IPPROTO_ICMPv4:
+ /* Fall through */
- case _ODP_IPPROTO_AH:
- prs->input_flags.ipsec = 1;
- prs->input_flags.ipsec_ah = 1;
- break;
+ case _ODP_IPPROTO_ICMPv6:
+ prs->input_flags.icmp = 1;
+ break;
- case _ODP_IPPROTO_ESP:
- prs->input_flags.ipsec = 1;
- prs->input_flags.ipsec_esp = 1;
- break;
+ case _ODP_IPPROTO_TCP:
+ if (odp_unlikely(offset + _ODP_TCPHDR_LEN > seg_len))
+ return -1;
+ prs->input_flags.tcp = 1;
+ parse_tcp(prs, &parseptr, NULL);
+ break;
- case _ODP_IPPROTO_SCTP:
- prs->input_flags.sctp = 1;
- break;
+ case _ODP_IPPROTO_UDP:
+ if (odp_unlikely(offset + _ODP_UDPHDR_LEN > seg_len))
+ return -1;
+ prs->input_flags.udp = 1;
+ parse_udp(prs, &parseptr, NULL);
+ break;
- default:
- prs->input_flags.l4 = 0;
- prs->l4_offset = ODP_PACKET_OFFSET_INVALID;
- break;
- }
+ case _ODP_IPPROTO_AH:
+ prs->input_flags.ipsec = 1;
+ prs->input_flags.ipsec_ah = 1;
+ break;
- prs->parsed_layers = LAYER_L4;
+ case _ODP_IPPROTO_ESP:
+ prs->input_flags.ipsec = 1;
+ prs->input_flags.ipsec_esp = 1;
break;
- }
- case LAYER_ALL:
+ case _ODP_IPPROTO_SCTP:
+ prs->input_flags.sctp = 1;
break;
default:
- ODP_ERR("Invalid parse layer: %d\n", (int)layer);
- return -1;
+ prs->input_flags.l4 = 0;
+ prs->l4_offset = ODP_PACKET_OFFSET_INVALID;
+ break;
}
-
- prs->parsed_layers = LAYER_ALL;
-
parse_exit:
return prs->error_flags.all != 0;
}
-
/**
* Simple packet parser
*/
-int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, layer_t layer)
+int packet_parse_layer(odp_packet_hdr_t *pkt_hdr,
+ odp_pktio_parser_layer_t layer)
{
uint32_t seg_len = odp_packet_seg_len((odp_packet_t)pkt_hdr);
uint32_t len = packet_len(pkt_hdr);
diff --git a/platform/linux-dpdk/odp_packet_dpdk.c b/platform/linux-dpdk/odp_packet_dpdk.c
index 26d32370b..d056a8242 100644
--- a/platform/linux-dpdk/odp_packet_dpdk.c
+++ b/platform/linux-dpdk/odp_packet_dpdk.c
@@ -369,6 +369,12 @@ static int recv_pkt_dpdk(pktio_entry_t *pktio_entry, int index,
(struct rte_mbuf **)pkt_table,
(uint16_t)RTE_MAX(len, min));
+ if (pktio_entry->s.config.pktin.bit.ts_all ||
+ pktio_entry->s.config.pktin.bit.ts_ptp) {
+ ts_val = odp_time_global();
+ ts = &ts_val;
+ }
+
if (nb_rx == 0 && !pkt_dpdk->lockless_tx) {
pool_entry_t *pool_entry =
get_pool_entry(_odp_typeval(pktio_entry->s.pool));
@@ -382,14 +388,16 @@ static int recv_pkt_dpdk(pktio_entry_t *pktio_entry, int index,
odp_ticketlock_unlock(&pkt_dpdk->rx_lock[index]);
for (i = 0; i < nb_rx; ++i) {
- _odp_packet_reset_parse(pkt_table[i]);
- odp_packet_hdr(pkt_table[i])->input = pktio_entry->s.handle;
- }
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt_table[i]);
- if (pktio_entry->s.config.pktin.bit.ts_all ||
- pktio_entry->s.config.pktin.bit.ts_ptp) {
- ts_val = odp_time_global();
- ts = &ts_val;
+ packet_parse_reset(pkt_hdr);
+ pkt_hdr->input = pktio_entry->s.handle;
+
+ if (!pktio_cls_enabled(pktio_entry) &&
+ pktio_entry->s.config.parser.layer)
+ packet_parse_layer(pkt_hdr,
+ pktio_entry->s.config.parser.layer);
+ packet_set_ts(pkt_hdr, ts);
}
if (odp_unlikely(min > len)) {
diff --git a/platform/linux-dpdk/odp_packet_flags.c b/platform/linux-dpdk/odp_packet_flags.c
index 20517e164..d8c68f021 100644
--- a/platform/linux-dpdk/odp_packet_flags.c
+++ b/platform/linux-dpdk/odp_packet_flags.c
@@ -8,26 +8,21 @@
#include <odp/api/packet_flags.h>
#include <odp_packet_internal.h>
-#define retflag(pkt, x, layer) do { \
+#define retflag(pkt, x) do { \
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); \
- if (pkt_hdr->p.parsed_layers < layer) \
- packet_parse_layer(pkt_hdr, layer); \
return pkt_hdr->p.x; \
} while (0)
-#define setflag(pkt, x, v, layer) do { \
+#define setflag(pkt, x, v) do { \
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); \
- if (pkt_hdr->p.parsed_layers < layer) \
- packet_parse_layer(pkt_hdr, layer); \
- pkt_hdr->p.x = v & 1; \
+ pkt_hdr->p.x = (v) & 1; \
} while (0)
int odp_packet_has_error(odp_packet_t pkt)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- if (packet_parse_not_complete(pkt_hdr))
- packet_parse_layer(pkt_hdr, LAYER_ALL);
- return odp_packet_hdr(pkt)->p.error_flags.all != 0;
+
+ return pkt_hdr->p.error_flags.all != 0;
}
/* Get Input Flags */
@@ -44,126 +39,117 @@ int odp_packet_has_l2_error(odp_packet_t pkt)
int odp_packet_has_l3(odp_packet_t pkt)
{
- retflag(pkt, input_flags.l3, LAYER_L3);
+ retflag(pkt, input_flags.l3);
}
int odp_packet_has_l3_error(odp_packet_t pkt)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- if (packet_parse_not_complete(pkt_hdr))
- packet_parse_layer(pkt_hdr, LAYER_ALL);
-
return pkt_hdr->p.error_flags.ip_err;
}
int odp_packet_has_l4(odp_packet_t pkt)
{
- retflag(pkt, input_flags.l4, LAYER_L4);
+ retflag(pkt, input_flags.l4);
}
int odp_packet_has_l4_error(odp_packet_t pkt)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- if (pkt_hdr->p.parsed_layers < LAYER_L4)
- packet_parse_layer(pkt_hdr, LAYER_L4);
-
return pkt_hdr->p.error_flags.tcp_err | pkt_hdr->p.error_flags.udp_err;
}
int odp_packet_has_eth_bcast(odp_packet_t pkt)
{
- retflag(pkt, input_flags.eth_bcast, LAYER_L2);
+ retflag(pkt, input_flags.eth_bcast);
}
int odp_packet_has_eth_mcast(odp_packet_t pkt)
{
- retflag(pkt, input_flags.eth_mcast, LAYER_L2);
+ retflag(pkt, input_flags.eth_mcast);
}
int odp_packet_has_vlan(odp_packet_t pkt)
{
- retflag(pkt, input_flags.vlan, LAYER_L2);
+ retflag(pkt, input_flags.vlan);
}
int odp_packet_has_vlan_qinq(odp_packet_t pkt)
{
- retflag(pkt, input_flags.vlan_qinq, LAYER_L2);
+ retflag(pkt, input_flags.vlan_qinq);
}
int odp_packet_has_arp(odp_packet_t pkt)
{
- retflag(pkt, input_flags.arp, LAYER_L3);
+ retflag(pkt, input_flags.arp);
}
int odp_packet_has_ipv4(odp_packet_t pkt)
{
- retflag(pkt, input_flags.ipv4, LAYER_L3);
+ retflag(pkt, input_flags.ipv4);
}
int odp_packet_has_ipv6(odp_packet_t pkt)
{
- retflag(pkt, input_flags.ipv6, LAYER_L3);
+ retflag(pkt, input_flags.ipv6);
}
int odp_packet_has_ip_bcast(odp_packet_t pkt)
{
- retflag(pkt, input_flags.ip_bcast, LAYER_L3);
+ retflag(pkt, input_flags.ip_bcast);
}
int odp_packet_has_ip_mcast(odp_packet_t pkt)
{
- retflag(pkt, input_flags.ip_mcast, LAYER_L3);
+ retflag(pkt, input_flags.ip_mcast);
}
int odp_packet_has_ipfrag(odp_packet_t pkt)
{
- retflag(pkt, input_flags.ipfrag, LAYER_L3);
+ retflag(pkt, input_flags.ipfrag);
}
int odp_packet_has_ipopt(odp_packet_t pkt)
{
- retflag(pkt, input_flags.ipopt, LAYER_L3);
+ retflag(pkt, input_flags.ipopt);
}
int odp_packet_has_ipsec(odp_packet_t pkt)
{
- retflag(pkt, input_flags.ipsec, LAYER_L4);
+ retflag(pkt, input_flags.ipsec);
}
int odp_packet_has_udp(odp_packet_t pkt)
{
- retflag(pkt, input_flags.udp, LAYER_L4);
+ retflag(pkt, input_flags.udp);
}
int odp_packet_has_tcp(odp_packet_t pkt)
{
- retflag(pkt, input_flags.tcp, LAYER_L4);
+ retflag(pkt, input_flags.tcp);
}
int odp_packet_has_sctp(odp_packet_t pkt)
{
- retflag(pkt, input_flags.sctp, LAYER_L4);
+ retflag(pkt, input_flags.sctp);
}
int odp_packet_has_icmp(odp_packet_t pkt)
{
- retflag(pkt, input_flags.icmp, LAYER_L4);
+ retflag(pkt, input_flags.icmp);
}
odp_packet_color_t odp_packet_color(odp_packet_t pkt)
{
- retflag(pkt, input_flags.color, LAYER_ALL);
+ retflag(pkt, input_flags.color);
}
void odp_packet_color_set(odp_packet_t pkt, odp_packet_color_t color)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- if (packet_parse_not_complete(pkt_hdr))
- packet_parse_layer(pkt_hdr, LAYER_ALL);
-
pkt_hdr->p.input_flags.color = color;
}
@@ -171,29 +157,23 @@ odp_bool_t odp_packet_drop_eligible(odp_packet_t pkt)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- if (packet_parse_not_complete(pkt_hdr))
- packet_parse_layer(pkt_hdr, LAYER_ALL);
-
return !pkt_hdr->p.input_flags.nodrop;
}
void odp_packet_drop_eligible_set(odp_packet_t pkt, odp_bool_t drop)
{
- setflag(pkt, input_flags.nodrop, !drop, LAYER_ALL);
+ setflag(pkt, input_flags.nodrop, !drop);
}
int8_t odp_packet_shaper_len_adjust(odp_packet_t pkt)
{
- retflag(pkt, output_flags.shaper_len_adj, LAYER_ALL);
+ retflag(pkt, output_flags.shaper_len_adj);
}
void odp_packet_shaper_len_adjust_set(odp_packet_t pkt, int8_t adj)
{
odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- if (packet_parse_not_complete(pkt_hdr))
- packet_parse_layer(pkt_hdr, LAYER_ALL);
-
pkt_hdr->p.output_flags.shaper_len_adj = adj;
}
@@ -201,107 +181,107 @@ void odp_packet_shaper_len_adjust_set(odp_packet_t pkt, int8_t adj)
void odp_packet_has_l2_set(odp_packet_t pkt, int val)
{
- setflag(pkt, input_flags.l2, val, LAYER_L2);
+ setflag(pkt, input_flags.l2, val);
}
void odp_packet_has_l3_set(odp_packet_t pkt, int val)
{
- setflag(pkt, input_flags.l3, val, LAYER_L3);
+ setflag(pkt, input_flags.l3, val);
}
void odp_packet_has_l4_set(odp_packet_t pkt, int val)
{
- setflag(pkt, input_flags.l4, val, LAYER_L4);
+ setflag(pkt, input_flags.l4, val);
}
void odp_packet_has_eth_set(odp_packet_t pkt, int val)
{
- setflag(pkt, input_flags.eth, val, LAYER_L2);
+ setflag(pkt, input_flags.eth, val);
}
void odp_packet_has_eth_bcast_set(odp_packet_t pkt, int val)
{
- setflag(pkt, input_flags.eth_bcast, val, LAYER_L2);
+ setflag(pkt, input_flags.eth_bcast, val);
}
void odp_packet_has_eth_mcast_set(odp_packet_t pkt, int val)
{
- setflag(pkt, input_flags.eth_mcast, val, LAYER_L2);
+ setflag(pkt, input_flags.eth_mcast, val);
}
void odp_packet_has_jumbo_set(odp_packet_t pkt, int val)
{
- setflag(pkt, input_flags.jumbo, val, LAYER_L2);
+ setflag(pkt, input_flags.jumbo, val);
}
void odp_packet_has_vlan_set(odp_packet_t pkt, int val)
{
- setflag(pkt, input_flags.vlan, val, LAYER_L2);
+ setflag(pkt, input_flags.vlan, val);
}
void odp_packet_has_vlan_qinq_set(odp_packet_t pkt, int val)
{
- setflag(pkt, input_flags.vlan_qinq, val, LAYER_L2);
+ setflag(pkt, input_flags.vlan_qinq, val);
}
void odp_packet_has_arp_set(odp_packet_t pkt, int val)
{
- setflag(pkt, input_flags.arp, val, LAYER_L3);
+ setflag(pkt, input_flags.arp, val);
}
void odp_packet_has_ipv4_set(odp_packet_t pkt, int val)
{
- setflag(pkt, input_flags.ipv4, val, LAYER_L3);
+ setflag(pkt, input_flags.ipv4, val);
}
void odp_packet_has_ipv6_set(odp_packet_t pkt, int val)
{
- setflag(pkt, input_flags.ipv6, val, LAYER_L3);
+ setflag(pkt, input_flags.ipv6, val);
}
void odp_packet_has_ip_bcast_set(odp_packet_t pkt, int val)
{
- setflag(pkt, input_flags.ip_bcast, val, LAYER_L3);
+ setflag(pkt, input_flags.ip_bcast, val);
}
void odp_packet_has_ip_mcast_set(odp_packet_t pkt, int val)
{
- setflag(pkt, input_flags.ip_mcast, val, LAYER_L3);
+ setflag(pkt, input_flags.ip_mcast, val);
}
void odp_packet_has_ipfrag_set(odp_packet_t pkt, int val)
{
- setflag(pkt, input_flags.ipfrag, val, LAYER_L3);
+ setflag(pkt, input_flags.ipfrag, val);
}
void odp_packet_has_ipopt_set(odp_packet_t pkt, int val)
{
- setflag(pkt, input_flags.ipopt, val, LAYER_L3);
+ setflag(pkt, input_flags.ipopt, val);
}
void odp_packet_has_ipsec_set(odp_packet_t pkt, int val)
{
- setflag(pkt, input_flags.ipsec, val, LAYER_L4);
+ setflag(pkt, input_flags.ipsec, val);
}
void odp_packet_has_udp_set(odp_packet_t pkt, int val)
{
- setflag(pkt, input_flags.udp, val, LAYER_L4);
+ setflag(pkt, input_flags.udp, val);
}
void odp_packet_has_tcp_set(odp_packet_t pkt, int val)
{
- setflag(pkt, input_flags.tcp, val, LAYER_L4);
+ setflag(pkt, input_flags.tcp, val);
}
void odp_packet_has_sctp_set(odp_packet_t pkt, int val)
{
- setflag(pkt, input_flags.sctp, val, LAYER_L4);
+ setflag(pkt, input_flags.sctp, val);
}
void odp_packet_has_icmp_set(odp_packet_t pkt, int val)
{
- setflag(pkt, input_flags.icmp, val, LAYER_L4);
+ setflag(pkt, input_flags.icmp, val);
}
void odp_packet_has_ts_clr(odp_packet_t pkt)
diff --git a/platform/linux-dpdk/odp_pool.c b/platform/linux-dpdk/odp_pool.c
index e6d415954..0821cdcac 100644
--- a/platform/linux-dpdk/odp_pool.c
+++ b/platform/linux-dpdk/odp_pool.c
@@ -240,7 +240,8 @@ static int check_params(odp_pool_param_t *params)
{
odp_pool_capability_t capa;
- odp_pool_capability(&capa);
+ if (odp_pool_capability(&capa) < 0)
+ return -1;
switch (params->type) {
case ODP_POOL_BUFFER:
@@ -552,6 +553,8 @@ static odp_buffer_t buffer_alloc(pool_entry_t *pool)
odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl)
{
+ ODP_ASSERT(ODP_POOL_INVALID != pool_hdl);
+
pool_entry_t *pool = odp_pool_to_entry(pool_hdl);
return buffer_alloc(pool);
@@ -560,6 +563,9 @@ odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl)
int odp_buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int num)
{
int i;
+
+ ODP_ASSERT(ODP_POOL_INVALID != pool_hdl);
+
pool_entry_t *pool = odp_pool_to_entry(pool_hdl);
for (i = 0; i < num; i++) {
diff --git a/platform/linux-dpdk/odp_time.c b/platform/linux-dpdk/odp_time.c
index c1ad973cc..6c066a5b5 100644
--- a/platform/linux-dpdk/odp_time.c
+++ b/platform/linux-dpdk/odp_time.c
@@ -13,105 +13,182 @@
#include <odp_time_internal.h>
#include <rte_cycles.h>
#include <string.h>
+#include <inttypes.h>
+
+typedef uint64_t (*time_to_ns_fn) (odp_time_t time);
+typedef odp_time_t (*time_cur_fn)(void);
+typedef odp_time_t (*time_from_ns_fn) (uint64_t ns);
+typedef uint64_t (*time_res_fn)(void);
+
+typedef struct time_handler_ {
+ time_to_ns_fn time_to_ns;
+ time_cur_fn time_cur;
+ time_from_ns_fn time_from_ns;
+ time_res_fn time_res;
+} time_handler_t;
+
+typedef struct time_global_t {
+ struct timespec spec_start;
+ int use_hw;
+ uint64_t hw_start;
+ uint64_t hw_freq_hz;
+ /* DPDK specific */
+ time_handler_t handler;
+ double tick_per_nsec;
+ double nsec_per_tick;
+} time_global_t;
+
+static time_global_t global;
+
+/*
+ * Posix timespec based functions
+ */
+
+static inline uint64_t time_spec_diff_nsec(struct timespec *t2,
+ struct timespec *t1)
+{
+ struct timespec diff;
+ uint64_t nsec;
+
+ diff.tv_sec = t2->tv_sec - t1->tv_sec;
+ diff.tv_nsec = t2->tv_nsec - t1->tv_nsec;
-typedef union {
- odp_time_t ex;
- struct timespec in;
-} _odp_time_t;
+ if (diff.tv_nsec < 0) {
+ diff.tv_nsec += ODP_TIME_SEC_IN_NS;
+ diff.tv_sec -= 1;
+ }
+
+ nsec = (diff.tv_sec * ODP_TIME_SEC_IN_NS) + diff.tv_nsec;
-static odp_time_t start_time;
-static time_handler_t time_handler;
-double tick_per_nsec;
-double nsec_per_tick;
+ return nsec;
+}
-static inline uint64_t time_local_res_dpdk(void)
+static inline odp_time_t time_spec_cur(void)
{
- return rte_get_timer_hz();
+ int ret;
+ odp_time_t time;
+ struct timespec sys_time;
+
+ ret = clock_gettime(CLOCK_MONOTONIC_RAW, &sys_time);
+ if (odp_unlikely(ret != 0))
+ ODP_ABORT("clock_gettime failed\n");
+
+ time.nsec = time_spec_diff_nsec(&sys_time, &global.spec_start);
+
+ return time;
}
-static inline
-uint64_t time_to_ns(odp_time_t time)
+static inline uint64_t time_spec_res(void)
{
- uint64_t ns;
+ int ret;
+ struct timespec tres;
- ns = time.tv_sec * ODP_TIME_SEC_IN_NS;
- ns += time.tv_nsec;
+ ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres);
+ if (odp_unlikely(ret != 0))
+ ODP_ABORT("clock_getres failed\n");
- return ns;
+ return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec;
}
-static inline uint64_t time_to_ns_dpdk(odp_time_t time)
+static inline uint64_t time_spec_to_ns(odp_time_t time)
{
- return (time.tv_sec * nsec_per_tick);
+ return time.nsec;
}
-static inline odp_time_t time_diff(odp_time_t t2, odp_time_t t1)
+static inline odp_time_t time_spec_from_ns(uint64_t ns)
{
odp_time_t time;
- time.tv_sec = t2.tv_sec - t1.tv_sec;
- time.tv_nsec = t2.tv_nsec - t1.tv_nsec;
-
- if (time.tv_nsec < 0) {
- time.tv_nsec += ODP_TIME_SEC_IN_NS;
- --time.tv_sec;
- }
+ time.nsec = ns;
return time;
}
-static inline odp_time_t time_diff_dpdk(odp_time_t t2, odp_time_t t1)
+/*
+ * HW time counter based functions
+ */
+
+static inline odp_time_t time_hw_cur(void)
{
odp_time_t time;
- time.tv_sec = t2.tv_sec - t1.tv_sec;
+ time.count = cpu_global_time() - global.hw_start;
+
return time;
}
-static inline odp_time_t time_curr(void)
+static inline uint64_t time_hw_res(void)
{
- int ret;
- _odp_time_t time;
+ /* Promise a bit lower resolution than average cycle counter
+ * frequency */
+ return global.hw_freq_hz / 10;
+}
- ret = clock_gettime(CLOCK_MONOTONIC_RAW, &time.in);
- if (odp_unlikely(ret != 0))
- ODP_ABORT("clock_gettime failed\n");
- return time.ex;
+static inline uint64_t time_hw_to_ns(odp_time_t time)
+{
+ uint64_t nsec;
+ uint64_t freq_hz = global.hw_freq_hz;
+ uint64_t count = time.count;
+ uint64_t sec = 0;
+
+ if (count >= freq_hz) {
+ sec = count / freq_hz;
+ count = count - sec * freq_hz;
+ }
+
+ nsec = (ODP_TIME_SEC_IN_NS * count) / freq_hz;
+
+ return (sec * ODP_TIME_SEC_IN_NS) + nsec;
}
-static inline odp_time_t time_curr_dpdk(void)
+static inline odp_time_t time_hw_from_ns(uint64_t ns)
{
odp_time_t time;
+ uint64_t count;
+ uint64_t freq_hz = global.hw_freq_hz;
+ uint64_t sec = 0;
+
+ if (ns >= ODP_TIME_SEC_IN_NS) {
+ sec = ns / ODP_TIME_SEC_IN_NS;
+ ns = ns - sec * ODP_TIME_SEC_IN_NS;
+ }
+
+ count = sec * freq_hz;
+ count += (ns * freq_hz) / ODP_TIME_SEC_IN_NS;
+
+ time.count = count;
- time.tv_sec = rte_get_timer_cycles();
return time;
}
-static inline odp_time_t time_local(void)
+/*
+ * Common functions
+ */
+
+static inline odp_time_t time_cur(void)
{
- odp_time_t time;
+ if (global.use_hw)
+ return time_hw_cur();
- time = time_handler.time_curr();
- return time_handler.time_diff(time, start_time);
+ return time_spec_cur();
}
-static inline int time_cmp(odp_time_t t2, odp_time_t t1)
+static inline uint64_t time_res(void)
{
- if (t2.tv_sec < t1.tv_sec)
- return -1;
-
- if (t2.tv_sec > t1.tv_sec)
- return 1;
+ if (global.use_hw)
+ return time_hw_res();
- return t2.tv_nsec - t1.tv_nsec;
+ return time_spec_res();
}
-static inline int time_cmp_dpdk(odp_time_t t2, odp_time_t t1)
+static inline int time_cmp(odp_time_t t2, odp_time_t t1)
{
- if (t2.tv_sec < t1.tv_sec)
- return -1;
- if (t2.tv_sec > t1.tv_sec)
+ if (odp_likely(t2.u64 > t1.u64))
return 1;
+
+ if (t2.u64 < t1.u64)
+ return -1;
+
return 0;
}
@@ -119,61 +196,67 @@ static inline odp_time_t time_sum(odp_time_t t1, odp_time_t t2)
{
odp_time_t time;
- time.tv_sec = t2.tv_sec + t1.tv_sec;
- time.tv_nsec = t2.tv_nsec + t1.tv_nsec;
-
- if (time.tv_nsec >= (long)ODP_TIME_SEC_IN_NS) {
- time.tv_nsec -= ODP_TIME_SEC_IN_NS;
- ++time.tv_sec;
- }
+ time.u64 = t1.u64 + t2.u64;
return time;
}
-static inline odp_time_t time_sum_dpdk(odp_time_t t1, odp_time_t t2)
+static inline uint64_t time_to_ns(odp_time_t time)
{
- odp_time_t time;
+ if (global.use_hw)
+ return time_hw_to_ns(time);
- time.tv_sec = t2.tv_sec + t1.tv_sec;
- return time;
+ return time_spec_to_ns(time);
+}
+
+static inline odp_time_t time_from_ns(uint64_t ns)
+{
+ if (global.use_hw)
+ return time_hw_from_ns(ns);
+
+ return time_spec_from_ns(ns);
+}
+
+static inline uint64_t time_res_dpdk(void)
+{
+ return rte_get_timer_hz();
}
-static inline odp_time_t time_local_from_ns(uint64_t ns)
+static inline uint64_t time_to_ns_dpdk(odp_time_t time)
+{
+ return (time.u64 * global.nsec_per_tick);
+}
+
+static inline odp_time_t time_cur_dpdk(void)
{
odp_time_t time;
- time.tv_sec = ns / ODP_TIME_SEC_IN_NS;
- time.tv_nsec = ns - time.tv_sec * ODP_TIME_SEC_IN_NS;
+ time.u64 = rte_get_timer_cycles() - global.hw_start;
return time;
}
-static inline odp_time_t time_local_from_ns_dpdk(uint64_t ns)
+static inline odp_time_t time_from_ns_dpdk(uint64_t ns)
{
odp_time_t time;
- time.tv_sec = ns * tick_per_nsec;
+
+ time.u64 = ns * global.tick_per_nsec;
+
return time;
}
+static inline odp_time_t time_local(void)
+{
+ return global.handler.time_cur();
+}
+
static inline void time_wait_until(odp_time_t time)
{
odp_time_t cur;
do {
cur = time_local();
- } while (time_handler.time_cmp(time, cur) > 0);
-}
-
-static inline uint64_t time_local_res(void)
-{
- int ret;
- struct timespec tres;
-
- ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres);
- if (odp_unlikely(ret != 0))
- ODP_ABORT("clock_getres failed\n");
-
- return ODP_TIME_SEC_IN_NS / (uint64_t)tres.tv_nsec;
+ } while (time_cmp(time, cur) > 0);
}
odp_time_t odp_time_local(void)
@@ -188,49 +271,53 @@ odp_time_t odp_time_global(void)
odp_time_t odp_time_diff(odp_time_t t2, odp_time_t t1)
{
- return time_handler.time_diff(t2, t1);
+ odp_time_t time;
+
+ time.u64 = t2.u64 - t1.u64;
+
+ return time;
}
uint64_t odp_time_to_ns(odp_time_t time)
{
- return time_handler.time_to_ns(time);
+ return global.handler.time_to_ns(time);
}
odp_time_t odp_time_local_from_ns(uint64_t ns)
{
- return time_handler.time_local_from_ns(ns);
+ return global.handler.time_from_ns(ns);
}
odp_time_t odp_time_global_from_ns(uint64_t ns)
{
- return time_handler.time_local_from_ns(ns);
+ return global.handler.time_from_ns(ns);
}
int odp_time_cmp(odp_time_t t2, odp_time_t t1)
{
- return time_handler.time_cmp(t2, t1);
+ return time_cmp(t2, t1);
}
odp_time_t odp_time_sum(odp_time_t t1, odp_time_t t2)
{
- return time_handler.time_sum(t1, t2);
+ return time_sum(t1, t2);
}
uint64_t odp_time_local_res(void)
{
- return time_handler.time_local_res();
+ return global.handler.time_res();
}
uint64_t odp_time_global_res(void)
{
- return time_handler.time_local_res();
+ return global.handler.time_res();
}
void odp_time_wait_ns(uint64_t ns)
{
odp_time_t cur = time_local();
- odp_time_t wait = time_handler.time_local_from_ns(ns);
- odp_time_t end_time = time_handler.time_sum(cur, wait);
+ odp_time_t wait = global.handler.time_from_ns(ns);
+ odp_time_t end_time = time_sum(cur, wait);
time_wait_until(end_time);
}
@@ -240,31 +327,6 @@ void odp_time_wait_until(odp_time_t time)
return time_wait_until(time);
}
-static uint64_t time_to_u64(odp_time_t time)
-{
- int ret;
- struct timespec tres;
- uint64_t resolution;
-
- ret = clock_getres(CLOCK_MONOTONIC_RAW, &tres);
- if (odp_unlikely(ret != 0))
- ODP_ABORT("clock_getres failed\n");
-
- resolution = (uint64_t)tres.tv_nsec;
-
- return time_handler.time_to_ns(time) / resolution;
-}
-
-static uint64_t time_to_u64_dpdk(odp_time_t time)
-{
- return time.tv_sec;
-}
-
-uint64_t odp_time_to_u64(odp_time_t time)
-{
- return time_handler.time_to_u64(time);
-}
-
static odp_bool_t is_invariant_tsc_supported(void)
{
FILE *file;
@@ -311,37 +373,52 @@ static inline odp_bool_t is_dpdk_timer_cycles_support(void)
int odp_time_init_global(void)
{
+ int ret = 0;
+
+ memset(&global, 0, sizeof(time_global_t));
+
if (is_dpdk_timer_cycles_support()) {
- time_handler.time_to_ns = time_to_ns_dpdk;
- time_handler.time_diff = time_diff_dpdk;
- time_handler.time_curr = time_curr_dpdk;
- time_handler.time_cmp = time_cmp_dpdk;
- time_handler.time_sum = time_sum_dpdk;
- time_handler.time_local_from_ns = time_local_from_ns_dpdk;
- time_handler.time_local_res = time_local_res_dpdk;
- time_handler.time_to_u64 = time_to_u64_dpdk;
- tick_per_nsec = (double)time_local_res_dpdk() /
+ global.handler.time_to_ns = time_to_ns_dpdk;
+ global.handler.time_cur = time_cur_dpdk;
+ global.handler.time_from_ns = time_from_ns_dpdk;
+ global.handler.time_res = time_res_dpdk;
+ global.tick_per_nsec = (double)time_res_dpdk() /
(double)ODP_TIME_SEC_IN_NS;
- nsec_per_tick = (double)ODP_TIME_SEC_IN_NS /
- (double)time_local_res_dpdk();
- } else {
- time_handler.time_to_ns = time_to_ns;
- time_handler.time_diff = time_diff;
- time_handler.time_curr = time_curr;
- time_handler.time_cmp = time_cmp;
- time_handler.time_sum = time_sum;
- time_handler.time_local_from_ns = time_local_from_ns;
- time_handler.time_local_res = time_local_res;
- time_handler.time_to_u64 = time_to_u64;
+ global.nsec_per_tick = (double)ODP_TIME_SEC_IN_NS /
+ (double)time_res_dpdk();
+
+ global.hw_start = rte_get_timer_cycles();
+ if (global.hw_start == 0)
+ return -1;
+ else
+ return 0;
}
- start_time = time_handler.time_curr();
- if (time_handler.time_cmp(start_time, ODP_TIME_NULL) == 0) {
- ODP_ABORT("initiate odp time failed\n");
- return -1;
- } else {
+ global.handler.time_to_ns = time_to_ns;
+ global.handler.time_cur = time_cur;
+ global.handler.time_from_ns = time_from_ns;
+ global.handler.time_res = time_res;
+
+ if (cpu_has_global_time()) {
+ global.use_hw = 1;
+ global.hw_freq_hz = cpu_global_time_freq();
+
+ if (global.hw_freq_hz == 0)
+ return -1;
+
+ printf("HW time counter freq: %" PRIu64 " hz\n\n",
+ global.hw_freq_hz);
+
+ global.hw_start = cpu_global_time();
return 0;
}
+
+ global.spec_start.tv_sec = 0;
+ global.spec_start.tv_nsec = 0;
+
+ ret = clock_gettime(CLOCK_MONOTONIC_RAW, &global.spec_start);
+
+ return ret;
}
int odp_time_term_global(void)
generated by cgit v1.2.3 (git 2.25.1) at 2024-04-28 18:12:19 +0000