aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMatias Elo <matias.elo@nokia.com>2019-04-08 13:48:54 +0300
committerMatias Elo <matias.elo@nokia.com>2019-04-15 10:32:36 +0300
commit0b9d1e85d1f4d9b98c88c4d34fb26bc801556210 (patch)
tree00deadabe6936f853b466e5217b0630ee87f9f48
parent12eaa3d2ce51cc5cd48e54deded2740161162022 (diff)
downloadodp-0b9d1e85d1f4d9b98c88c4d34fb26bc801556210.tar.gz
linux-gen: pool: optimize buffer caching
Refactor local buffer caching into separate functions and optimize implementation by caching buffer header pointers instead of indices. Signed-off-by: Matias Elo <matias.elo@nokia.com> Reviewed-by: Petri Savolainen <petri.savolainen@nokia.com>
-rw-r--r--platform/linux-generic/include/odp_pool_internal.h7
-rw-r--r--platform/linux-generic/odp_pool.c161
2 files changed, 93 insertions, 75 deletions
diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h
index cfd0872af..a5242d60e 100644
--- a/platform/linux-generic/include/odp_pool_internal.h
+++ b/platform/linux-generic/include/odp_pool_internal.h
@@ -1,4 +1,5 @@
-/* Copyright (c) 2013-2018, Linaro Limited
+/* Copyright (c) 2019, Nokia
+ * Copyright (c) 2013-2018, Linaro Limited
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -26,8 +27,8 @@ extern "C" {
#include <odp/api/plat/strong_types.h>
typedef struct ODP_ALIGNED_CACHE pool_cache_t {
- uint32_t num;
- uint32_t buf_index[CONFIG_POOL_CACHE_SIZE];
+ uint32_t num; /* Number of buffers in cache */
+ odp_buffer_hdr_t *buf_hdr[CONFIG_POOL_CACHE_SIZE]; /* Cached buffers */
} pool_cache_t;
diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c
index feb25cf39..2e0898461 100644
--- a/platform/linux-generic/odp_pool.c
+++ b/platform/linux-generic/odp_pool.c
@@ -91,6 +91,58 @@ static inline pool_t *pool_from_buf(odp_buffer_t buf)
return buf_hdr->pool_ptr;
}
+static inline void cache_init(pool_cache_t *cache)
+{
+ memset(cache, 0, sizeof(pool_cache_t));
+}
+
+static inline uint32_t cache_pop(pool_cache_t *cache,
+ odp_buffer_hdr_t *buf_hdr[], int max_num)
+{
+ uint32_t cache_num = cache->num;
+ uint32_t num_ch = max_num;
+ uint32_t cache_begin;
+ uint32_t i;
+
+ /* Cache does not have enough buffers */
+ if (odp_unlikely(cache_num < (uint32_t)max_num))
+ num_ch = cache_num;
+
+ /* Get buffers from the cache */
+ cache_begin = cache_num - num_ch;
+ for (i = 0; i < num_ch; i++)
+ buf_hdr[i] = cache->buf_hdr[cache_begin + i];
+
+ cache->num = cache_num - num_ch;
+
+ return num_ch;
+}
+
+static inline void cache_push(pool_cache_t *cache, odp_buffer_hdr_t *buf_hdr[],
+ uint32_t num)
+{
+ uint32_t cache_num = cache->num;
+ uint32_t i;
+
+ for (i = 0; i < num; i++)
+ cache->buf_hdr[cache_num + i] = buf_hdr[i];
+
+ cache->num = cache_num + num;
+}
+
+static void cache_flush(pool_cache_t *cache, pool_t *pool)
+{
+ odp_buffer_hdr_t *buf_hdr;
+ ring_t *ring;
+ uint32_t mask;
+
+ ring = &pool->ring->hdr;
+ mask = pool->ring_mask;
+
+ while (cache_pop(cache, &buf_hdr, 1))
+ ring_enq(ring, mask, buf_hdr->index.buffer);
+}
+
static int read_config_file(pool_table_t *pool_tbl)
{
const char *str;
@@ -193,29 +245,13 @@ int odp_pool_init_local(void)
for (i = 0; i < ODP_CONFIG_POOLS; i++) {
pool = pool_entry(i);
local.cache[i] = &pool->local_cache[thr_id];
- local.cache[i]->num = 0;
+ cache_init(local.cache[i]);
}
local.thr_id = thr_id;
return 0;
}
-static void flush_cache(pool_cache_t *cache, pool_t *pool)
-{
- ring_t *ring;
- uint32_t mask;
- uint32_t cache_num, i;
-
- ring = &pool->ring->hdr;
- mask = pool->ring_mask;
- cache_num = cache->num;
-
- for (i = 0; i < cache_num; i++)
- ring_enq(ring, mask, cache->buf_index[i]);
-
- cache->num = 0;
-}
-
int odp_pool_term_local(void)
{
int i;
@@ -223,7 +259,7 @@ int odp_pool_term_local(void)
for (i = 0; i < ODP_CONFIG_POOLS; i++) {
pool_t *pool = pool_entry(i);
- flush_cache(local.cache[i], pool);
+ cache_flush(local.cache[i], pool);
}
return 0;
@@ -725,7 +761,7 @@ int odp_pool_destroy(odp_pool_t pool_hdl)
/* Make sure local caches are empty */
for (i = 0; i < ODP_THREAD_COUNT_MAX; i++)
- flush_cache(&pool->local_cache[i], pool);
+ cache_flush(&pool->local_cache[i], pool);
odp_shm_free(pool->shm);
@@ -791,40 +827,26 @@ int odp_pool_info(odp_pool_t pool_hdl, odp_pool_info_t *info)
int buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int max_num)
{
+ pool_cache_t *cache = local.cache[pool->pool_idx];
ring_t *ring;
- uint32_t mask, i;
- pool_cache_t *cache;
- uint32_t cache_num, num_ch, num_deq, burst;
odp_buffer_hdr_t *hdr;
+ uint32_t mask, num_ch, i;
+ uint32_t num_deq = 0;
- cache = local.cache[pool->pool_idx];
-
- cache_num = cache->num;
- num_ch = max_num;
- num_deq = 0;
- burst = CACHE_BURST;
+ /* First pull packets from local cache */
+ num_ch = cache_pop(cache, buf_hdr, max_num);
- if (odp_unlikely(cache_num < (uint32_t)max_num)) {
- /* Cache does not have enough buffers */
- num_ch = cache_num;
- num_deq = max_num - cache_num;
+ /* If needed, get more from the global pool */
+ if (odp_unlikely(num_ch != (uint32_t)max_num)) {
+ uint32_t burst = CACHE_BURST;
+ uint32_t cache_num;
+ num_deq = max_num - num_ch;
if (odp_unlikely(num_deq > CACHE_BURST))
burst = num_deq;
- }
-
- /* Get buffers from the cache */
- for (i = 0; i < num_ch; i++) {
- uint32_t j = cache_num - num_ch + i;
-
- buf_hdr[i] = buf_hdr_from_index(pool, cache->buf_index[j]);
- }
- /* Declare variable here to fix clang compilation bug */
- uint32_t data[burst];
+ uint32_t data[burst];
- /* If needed, get more from the global pool */
- if (odp_unlikely(num_deq)) {
/* Temporary copy to data[] needed since odp_buffer_t is
* uintptr_t and not uint32_t. */
ring = &pool->ring->hdr;
@@ -845,13 +867,18 @@ int buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int max_num)
buf_hdr[idx] = hdr;
}
- /* Cache extra buffers. Cache is currently empty. */
- for (i = 0; i < cache_num; i++)
- cache->buf_index[i] = data[num_deq + i];
+ /* Cache possible extra buffers. Cache is currently empty. */
+ if (cache_num) {
+ odp_buffer_hdr_t *buf_hdr[cache_num];
- cache->num = cache_num;
- } else {
- cache->num = cache_num - num_ch;
+ for (i = 0; i < cache_num; i++) {
+ uint32_t idx = num_deq + i;
+
+ buf_hdr[i] = buf_hdr_from_index(pool,
+ data[idx]);
+ }
+ cache_push(cache, buf_hdr, cache_num);
+ }
}
return num_ch + num_deq;
@@ -860,13 +887,10 @@ int buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int max_num)
static inline void buffer_free_to_pool(pool_t *pool,
odp_buffer_hdr_t *buf_hdr[], int num)
{
- int i;
+ pool_cache_t *cache = local.cache[pool->pool_idx];
ring_t *ring;
- uint32_t mask;
- pool_cache_t *cache;
- uint32_t cache_num;
-
- cache = local.cache[pool->pool_idx];
+ int i;
+ uint32_t cache_num, mask;
/* Special case of a very large free. Move directly to
* the global pool. */
@@ -888,7 +912,6 @@ static inline void buffer_free_to_pool(pool_t *pool,
cache_num = cache->num;
if (odp_unlikely((int)(CONFIG_POOL_CACHE_SIZE - cache_num) < num)) {
- uint32_t index;
int burst = CACHE_BURST;
ring = &pool->ring->hdr;
@@ -899,26 +922,20 @@ static inline void buffer_free_to_pool(pool_t *pool,
if (odp_unlikely((uint32_t)num > cache_num))
burst = cache_num;
- {
- /* Temporary copy needed since odp_buffer_t is
- * uintptr_t and not uint32_t. */
- uint32_t data[burst];
-
- index = cache_num - burst;
+ /* Temporary copy needed since odp_buffer_t is
+ * uintptr_t and not uint32_t. */
+ odp_buffer_hdr_t *buf_hdr[burst];
+ uint32_t data[burst];
- for (i = 0; i < burst; i++)
- data[i] = cache->buf_index[index + i];
+ cache_pop(cache, buf_hdr, burst);
- ring_enq_multi(ring, mask, data, burst);
- }
+ for (i = 0; i < burst; i++)
+ data[i] = buf_hdr[i]->index.buffer;
- cache_num -= burst;
+ ring_enq_multi(ring, mask, data, burst);
}
- for (i = 0; i < num; i++)
- cache->buf_index[cache_num + i] = buf_hdr[i]->index.buffer;
-
- cache->num = cache_num + num;
+ cache_push(cache, buf_hdr, num);
}
void buffer_free_multi(odp_buffer_hdr_t *buf_hdr[], int num_total)