linux-gen: ipsec: separate hot r/w data from r/o data in an SA

Group very frequently updated SA state together, separately from
read-only data to reduce false sharing of cache lines and resulting
cache missed (not done for tun_ipv4.hdr_id which should be removed).

Signed-off-by: Janne Peltonen <janne.peltonen@nokia.com>
Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov@linaro.org>
Reviewed-by: Bill Fischofer <bill.fischofer@linaro.org>
Signed-off-by: Maxim Uvarov <maxim.uvarov@linaro.org>

3 files changed, 37 insertions, 23 deletions

diff --git a/platform/linux-generic/include/odp_ipsec_internal.h b/platform/linux-generic/include/odp_ipsec_internal.h
index 4941fbbab..d9884f886 100644
--- a/platform/linux-generic/include/odp_ipsec_internal.h
+++ b/platform/linux-generic/include/odp_ipsec_internal.h
@@ -94,6 +94,27 @@ int _odp_ipsec_status_send(odp_queue_t queue,
 struct ipsec_sa_s {
 	odp_atomic_u32_t ODP_ALIGNED_CACHE state;
 
+	/*
+	 * State that gets updated very frequently. Grouped separately
+	 * to avoid false cache line sharing with other data.
+	 */
+	struct ODP_ALIGNED_CACHE {
+		/* Statistics for soft/hard expiration */
+		odp_atomic_u64_t bytes;
+		odp_atomic_u64_t packets;
+
+		union {
+			struct {
+				odp_atomic_u64_t antireplay;
+			} in;
+
+			struct {
+				odp_atomic_u64_t counter; /* for CTR/GCM */
+				odp_atomic_u32_t seq;
+			} out;
+		};
+	} hot;
+
 	uint32_t	ipsec_sa_idx;
 	odp_ipsec_sa_t	ipsec_sa_hdl;
 
@@ -108,10 +129,6 @@ struct ipsec_sa_s {
 	uint64_t hard_limit_bytes;
 	uint64_t hard_limit_packets;
 
-	/* Statistics for soft/hard expiration */
-	odp_atomic_u64_t bytes;
-	odp_atomic_u64_t packets;
-
 	odp_crypto_session_t session;
 	void		*context;
 	odp_queue_t	queue;
@@ -150,12 +167,9 @@ struct ipsec_sa_s {
 				odp_u32be_t	lookup_dst_ipv4;
 				uint8_t lookup_dst_ipv6[_ODP_IPV6ADDR_LEN];
 			};
-			odp_atomic_u64_t antireplay;
 		} in;
 
 		struct {
-			odp_atomic_u64_t counter; /* for CTR/GCM */
-			odp_atomic_u32_t seq;
 			odp_ipsec_frag_mode_t frag_mode;
 			uint32_t mtu;
 
diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c
index a71efffe8..6a44277d8 100644
--- a/platform/linux-generic/odp_ipsec.c
+++ b/platform/linux-generic/odp_ipsec.c
@@ -813,7 +813,7 @@ err:
 static inline
 uint32_t ipsec_seq_no(ipsec_sa_t *ipsec_sa)
 {
-	return odp_atomic_fetch_add_u32(&ipsec_sa->out.seq, 1);
+	return odp_atomic_fetch_add_u32(&ipsec_sa->hot.out.seq, 1);
 }
 
 /* Helper for calculating encode length using data length and block size */
@@ -1010,7 +1010,7 @@ static int ipsec_out_iv(ipsec_state_t *state,
 		/* Both GCM and CTR use 8-bit counters */
 		ODP_ASSERT(sizeof(ctr) == ipsec_sa->esp_iv_len);
 
-		ctr = odp_atomic_fetch_add_u64(&ipsec_sa->out.counter,
+		ctr = odp_atomic_fetch_add_u64(&ipsec_sa->hot.out.counter,
 					       1);
 		/* Check for overrun */
 		if (ctr == 0)
diff --git a/platform/linux-generic/odp_ipsec_sad.c b/platform/linux-generic/odp_ipsec_sad.c
index aa1c337de..31a7ac922 100644
--- a/platform/linux-generic/odp_ipsec_sad.c
+++ b/platform/linux-generic/odp_ipsec_sad.c
@@ -75,8 +75,8 @@ int _odp_ipsec_sad_init_global(void)
 		ipsec_sa->ipsec_sa_hdl = ipsec_sa_index_to_handle(i);
 		ipsec_sa->ipsec_sa_idx = i;
 		odp_atomic_init_u32(&ipsec_sa->state, IPSEC_SA_STATE_FREE);
-		odp_atomic_init_u64(&ipsec_sa->bytes, 0);
-		odp_atomic_init_u64(&ipsec_sa->packets, 0);
+		odp_atomic_init_u64(&ipsec_sa->hot.bytes, 0);
+		odp_atomic_init_u64(&ipsec_sa->hot.packets, 0);
 	}
 
 	return 0;
@@ -318,10 +318,10 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param)
 		if (param->inbound.antireplay_ws > IPSEC_ANTIREPLAY_WS)
 			goto error;
 		ipsec_sa->antireplay = (param->inbound.antireplay_ws != 0);
-		odp_atomic_init_u64(&ipsec_sa->in.antireplay, 0);
+		odp_atomic_init_u64(&ipsec_sa->hot.in.antireplay, 0);
 	} else {
 		ipsec_sa->lookup_mode = ODP_IPSEC_LOOKUP_DISABLED;
-		odp_atomic_store_u32(&ipsec_sa->out.seq, 1);
+		odp_atomic_store_u32(&ipsec_sa->hot.out.seq, 1);
 		ipsec_sa->out.frag_mode = param->outbound.frag_mode;
 		ipsec_sa->out.mtu = param->outbound.mtu;
 	}
@@ -331,8 +331,8 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param)
 	ipsec_sa->copy_flabel = param->opt.copy_flabel;
 	ipsec_sa->udp_encap = param->opt.udp_encap;
 
-	odp_atomic_store_u64(&ipsec_sa->bytes, 0);
-	odp_atomic_store_u64(&ipsec_sa->packets, 0);
+	odp_atomic_store_u64(&ipsec_sa->hot.bytes, 0);
+	odp_atomic_store_u64(&ipsec_sa->hot.packets, 0);
 	ipsec_sa->soft_limit_bytes = param->lifetime.soft_limit.bytes;
 	ipsec_sa->soft_limit_packets = param->lifetime.soft_limit.packets;
 	ipsec_sa->hard_limit_bytes = param->lifetime.hard_limit.bytes;
@@ -472,7 +472,7 @@ odp_ipsec_sa_t odp_ipsec_sa_create(const odp_ipsec_sa_param_t *param)
 
 	if (1 == ipsec_sa->use_counter_iv &&
 	    ODP_IPSEC_DIR_OUTBOUND == param->dir)
-		odp_atomic_init_u64(&ipsec_sa->out.counter, 1);
+		odp_atomic_init_u64(&ipsec_sa->hot.out.counter, 1);
 
 	ipsec_sa->icv_len = crypto_param.auth_digest_len;
 
@@ -636,13 +636,13 @@ int _odp_ipsec_sa_stats_precheck(ipsec_sa_t *ipsec_sa,
 	int rc = 0;
 
 	if (ipsec_sa->hard_limit_bytes > 0 &&
-	    odp_atomic_load_u64(&ipsec_sa->bytes) >
+	    odp_atomic_load_u64(&ipsec_sa->hot.bytes) >
 	    ipsec_sa->hard_limit_bytes) {
 		status->error.hard_exp_bytes = 1;
 		rc = -1;
 	}
 	if (ipsec_sa->hard_limit_packets > 0 &&
-	    odp_atomic_load_u64(&ipsec_sa->packets) >
+	    odp_atomic_load_u64(&ipsec_sa->hot.packets) >
 	    ipsec_sa->hard_limit_packets) {
 		status->error.hard_exp_packets = 1;
 		rc = -1;
@@ -654,8 +654,8 @@ int _odp_ipsec_sa_stats_precheck(ipsec_sa_t *ipsec_sa,
 int _odp_ipsec_sa_stats_update(ipsec_sa_t *ipsec_sa, uint32_t len,
 			       odp_ipsec_op_status_t *status)
 {
-	uint64_t bytes = odp_atomic_fetch_add_u64(&ipsec_sa->bytes, len) + len;
-	uint64_t packets = odp_atomic_fetch_add_u64(&ipsec_sa->packets, 1) + 1;
+	uint64_t bytes = odp_atomic_fetch_add_u64(&ipsec_sa->hot.bytes, len) + len;
+	uint64_t packets = odp_atomic_fetch_add_u64(&ipsec_sa->hot.packets, 1) + 1;
 	int rc = 0;
 
 	if (ipsec_sa->soft_limit_bytes > 0 &&
@@ -686,7 +686,7 @@ int _odp_ipsec_sa_replay_precheck(ipsec_sa_t *ipsec_sa, uint32_t seq,
 	/* Try to be as quick as possible, we will discard packets later */
 	if (ipsec_sa->antireplay &&
 	    seq + IPSEC_ANTIREPLAY_WS <=
-	    (odp_atomic_load_u64(&ipsec_sa->in.antireplay) & 0xffffffff)) {
+	    (odp_atomic_load_u64(&ipsec_sa->hot.in.antireplay) & 0xffffffff)) {
 		status->error.antireplay = 1;
 		return -1;
 	}
@@ -703,7 +703,7 @@ int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq,
 	if (!ipsec_sa->antireplay)
 		return 0;
 
-	state = odp_atomic_load_u64(&ipsec_sa->in.antireplay);
+	state = odp_atomic_load_u64(&ipsec_sa->hot.in.antireplay);
 
 	while (0 == cas) {
 		uint32_t max_seq = state & 0xffffffff;
@@ -728,7 +728,7 @@ int _odp_ipsec_sa_replay_update(ipsec_sa_t *ipsec_sa, uint32_t seq,
 
 		new_state = (((uint64_t)mask) << 32) | max_seq;
 
-		cas = odp_atomic_cas_acq_rel_u64(&ipsec_sa->in.antireplay,
+		cas = odp_atomic_cas_acq_rel_u64(&ipsec_sa->hot.in.antireplay,
 						 &state, new_state);
 	}