aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMatias Elo <matias.elo@nokia.com>2017-10-30 15:39:50 +0200
committerBalakrishna Garapati <balakrishna.garapati@linaro.org>2017-11-14 10:23:56 +0100
commita65ece22d0aec860fbbdc63266d75d6794a132e1 (patch)
tree8d20f7f0038cb427ff3e662b15422e3818abc3b2
parentafdf175a5ea644ce599fe4843bb26c7a984989f9 (diff)
downloadodp-dpdk-master.tar.gz
linux-dpdk: shm: use native dpdk memzonesHEADmaster
Replace odp-linux shared memory implementation with native DPDK memzones. Signed-off-by: Matias Elo <matias.elo@nokia.com>
-rw-r--r--platform/linux-dpdk/Makefile.am10
-rw-r--r--platform/linux-dpdk/include/odp_shm_internal.h32
-rw-r--r--platform/linux-dpdk/odp_init.c31
-rw-r--r--platform/linux-dpdk/odp_shared_memory.c441
-rw-r--r--test/common_plat/validation/api/shmem/shmem.c4
5 files changed, 487 insertions, 31 deletions
diff --git a/platform/linux-dpdk/Makefile.am b/platform/linux-dpdk/Makefile.am
index c1497f0..ec695b9 100644
--- a/platform/linux-dpdk/Makefile.am
+++ b/platform/linux-dpdk/Makefile.am
@@ -121,9 +121,6 @@ odpdrvinclude_HEADERS = \
$(srcdir)/include/odp/drv/compiler.h
noinst_HEADERS = \
- ${top_srcdir}/platform/linux-generic/include/_fdserver_internal.h \
- ${top_srcdir}/platform/linux-generic/include/_ishm_internal.h \
- ${top_srcdir}/platform/linux-generic/include/_ishmphy_internal.h \
${top_srcdir}/platform/linux-generic/include/odp_align_internal.h \
${top_srcdir}/platform/linux-generic/include/odp_atomic_internal.h \
${srcdir}/include/odp_buffer_inlines.h \
@@ -154,7 +151,7 @@ noinst_HEADERS = \
${top_srcdir}/platform/linux-generic/include/odp_ring_internal.h \
${top_srcdir}/platform/linux-generic/include/odp_schedule_if.h \
${top_srcdir}/platform/linux-generic/include/odp_sorted_list_internal.h \
- ${top_srcdir}/platform/linux-generic/include/odp_shm_internal.h \
+ ${srcdir}/include/odp_shm_internal.h \
${top_srcdir}/platform/linux-generic/include/odp_time_internal.h \
${top_srcdir}/platform/linux-generic/include/odp_timer_internal.h \
${top_srcdir}/platform/linux-generic/include/odp_timer_wheel_internal.h \
@@ -167,9 +164,6 @@ noinst_HEADERS = \
${srcdir}/Makefile.inc
__LIB__libodp_dpdk_la_SOURCES = \
- ../linux-generic/_fdserver.c \
- ../linux-generic/_ishm.c \
- ../linux-generic/_ishmphy.c \
../linux-generic/odp_atomic.c \
../linux-generic/odp_barrier.c \
../linux-generic/odp_bitmap.c \
@@ -199,7 +193,7 @@ __LIB__libodp_dpdk_la_SOURCES = \
../linux-generic/odp_schedule.c \
../linux-generic/odp_schedule_if.c \
../linux-generic/odp_schedule_iquery.c \
- ../linux-generic/odp_shared_memory.c \
+ odp_shared_memory.c \
../linux-generic/odp_sorted_list.c \
../linux-generic/odp_spinlock.c \
../linux-generic/odp_spinlock_recursive.c \
diff --git a/platform/linux-dpdk/include/odp_shm_internal.h b/platform/linux-dpdk/include/odp_shm_internal.h
new file mode 100644
index 0000000..072a4f1
--- /dev/null
+++ b/platform/linux-dpdk/include/odp_shm_internal.h
@@ -0,0 +1,32 @@
+/* Copyright (c) 2017, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODP shared memory - implementation internal
+ */
+
+#ifndef ODP_SHM_INTERNAL_H_
+#define ODP_SHM_INTERNAL_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int _odp_shm_init_global(void);
+
+int _odp_shm_init_local(void);
+
+int _odp_shm_term_global(void);
+
+int _odp_shm_term_local(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-dpdk/odp_init.c b/platform/linux-dpdk/odp_init.c
index 6cea393..fdd9d43 100644
--- a/platform/linux-dpdk/odp_init.c
+++ b/platform/linux-dpdk/odp_init.c
@@ -12,6 +12,7 @@
#include <unistd.h>
#include <odp_internal.h>
#include <odp_schedule_if.h>
+#include <odp_shm_internal.h>
#include <string.h>
#include <stdio.h>
#include <linux/limits.h>
@@ -238,14 +239,8 @@ int odp_init_global(odp_instance_t *instance,
cleanup_files(hpdir, odp_global_data.main_pid);
stage = SYSINFO_INIT;
- if (_odp_fdserver_init_global()) {
- ODP_ERR("ODP fdserver init failed.\n");
- goto init_failed;
- }
- stage = FDSERVER_INIT;
-
- if (_odp_ishm_init_global()) {
- ODP_ERR("ODP ishm init failed.\n");
+ if (_odp_shm_init_global()) {
+ ODP_ERR("ODP shm init failed.\n");
goto init_failed;
}
stage = ISHM_INIT;
@@ -405,19 +400,13 @@ int _odp_term_global(enum init_stage stage)
/* Fall through */
case ISHM_INIT:
- if (_odp_ishm_term_global()) {
- ODP_ERR("ODP ishm term failed.\n");
+ if (_odp_shm_term_global()) {
+ ODP_ERR("ODP shm term failed.\n");
rc = -1;
}
/* Fall through */
-
+ /* Needed to prevent compiler warning */
case FDSERVER_INIT:
- if (_odp_fdserver_term_global()) {
- ODP_ERR("ODP fdserver term failed.\n");
- rc = -1;
- }
- /* Fall through */
-
case SYSINFO_INIT:
if (odp_system_info_term()) {
ODP_ERR("ODP system info term failed.\n");
@@ -455,8 +444,8 @@ int odp_init_local(odp_instance_t instance, odp_thread_type_t thr_type)
goto init_fail;
}
- if (_odp_ishm_init_local()) {
- ODP_ERR("ODP ishm local init failed.\n");
+ if (_odp_shm_init_local()) {
+ ODP_ERR("ODP shm local init failed.\n");
goto init_fail;
}
stage = ISHM_INIT;
@@ -531,8 +520,8 @@ int _odp_term_local(enum init_stage stage)
/* Fall through */
case ISHM_INIT:
- if (_odp_ishm_term_local()) {
- ODP_ERR("ODP ishm local term failed.\n");
+ if (_odp_shm_term_local()) {
+ ODP_ERR("ODP shm local term failed.\n");
rc = -1;
}
/* Fall through */
diff --git a/platform/linux-dpdk/odp_shared_memory.c b/platform/linux-dpdk/odp_shared_memory.c
new file mode 100644
index 0000000..5af02f4
--- /dev/null
+++ b/platform/linux-dpdk/odp_shared_memory.c
@@ -0,0 +1,441 @@
+/* Copyright (c) 2017, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#include <odp_align_internal.h>
+#include <odp_config_internal.h>
+#include <odp/api/debug.h>
+#include <odp_debug_internal.h>
+#include <odp/api/shared_memory.h>
+#include <odp/api/spinlock.h>
+#include <odp/api/plat/strong_types.h>
+#include <odp_shm_internal.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+#include <inttypes.h>
+
+#include <rte_lcore.h>
+#include <rte_memzone.h>
+
+#define SHM_MAX_ALIGN (0x80000000)
+#define SHM_BLOCK_NAME "%" PRIu64 "-%d-%s"
+
+ODP_STATIC_ASSERT(ODP_CONFIG_SHM_BLOCKS >= ODP_CONFIG_POOLS,
+ "ODP_CONFIG_SHM_BLOCKS < ODP_CONFIG_POOLS");
+
+ODP_STATIC_ASSERT(ODP_SHM_NAME_LEN >= RTE_MEMZONE_NAMESIZE,
+ "ODP_SHM_NAME_LEN < RTE_MEMZONE_NAMESIZE");
+
+typedef enum {
+ SHM_TYPE_LOCAL = 0,
+ SHM_TYPE_REMOTE
+} shm_type_t;
+
+/**
+ * Memory zone descriptor
+ *
+ * This struct is stored inside DPDK memzone to make it available for
+ * odp_shm_import().
+ */
+typedef struct {
+ /* Shared memory flags */
+ uint32_t flags;
+} shm_zone_t;
+
+/**
+ * Memory block descriptor
+ */
+typedef struct {
+ /* Memory block type */
+ shm_type_t type;
+ /* Memory block name */
+ char name[ODP_SHM_NAME_LEN];
+ /* DPDK memzone. If this pointer != NULL, the shm block is interpreted
+ * as reserved. */
+ const struct rte_memzone *mz;
+} shm_block_t;
+
+/**
+ * Table of blocks describing allocated shared memory. This table is visible to
+ * every ODP thread (linux process or pthreads). It is allocated shared at odp
+ * init time and is therefore inherited by all.
+ */
+typedef struct {
+ odp_spinlock_t lock;
+ shm_block_t block[ODP_CONFIG_SHM_BLOCKS];
+} shm_table_t;
+
+static shm_table_t *shm_tbl;
+
+/**
+ * Check if DPDK memzone name has been used already
+ */
+static odp_bool_t mz_name_used(const char *name)
+{
+ int idx;
+
+ for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) {
+ if (shm_tbl->block[idx].mz &&
+ strncmp(name, shm_tbl->block[idx].mz->name,
+ RTE_MEMZONE_NAMESIZE) == 0)
+ return 1;
+ }
+ return 0;
+}
+
+/**
+ * Convert ODP shm name into unique DPDK memzone name
+ */
+static void name_to_mz_name(const char *name, char *mz_name)
+{
+ int i = 0;
+
+ /* Use pid and counter to make name unique */
+ do {
+ snprintf(mz_name, RTE_MEMZONE_NAMESIZE - 1, SHM_BLOCK_NAME,
+ (odp_instance_t)odp_global_data.main_pid, i++, name);
+ mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0;
+ } while (mz_name_used(mz_name));
+}
+
+/**
+ * Convert DPDK memzone length into ODP shm block size
+ */
+static uint64_t shm_size(const struct rte_memzone *mz)
+{
+ return mz->len - sizeof(shm_zone_t);
+}
+
+/**
+ * Return a pointer to shm zone descriptor stored at the end of DPDK memzone
+ */
+static shm_zone_t *shm_zone(const struct rte_memzone *mz)
+{
+ return (shm_zone_t *)((uint8_t *)mz->addr + shm_size(mz));
+}
+
+static int find_free_block(void)
+{
+ int idx;
+
+ for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) {
+ if (shm_tbl->block[idx].mz == NULL)
+ return idx;
+ }
+ return -1;
+}
+
+static inline uint32_t handle_to_idx(odp_shm_t shm)
+{
+ return _odp_typeval(shm) - 1;
+}
+
+static inline odp_shm_t idx_to_handle(uint32_t idx)
+{
+ return _odp_cast_scalar(odp_shm_t, idx + 1);
+}
+
+static inline odp_bool_t handle_is_valid(odp_shm_t shm)
+{
+ int idx = handle_to_idx(shm);
+
+ if (idx < 0 || idx >= ODP_CONFIG_SHM_BLOCKS ||
+ shm_tbl->block[idx].mz == NULL) {
+ ODP_ERR("Invalid odp_shm_t handle: %" PRIu64 "\n",
+ odp_shm_to_u64(shm));
+ return 0;
+ }
+ return 1;
+}
+
+int _odp_shm_init_global(void)
+{
+ void *addr;
+
+ if ((getpid() != odp_global_data.main_pid) ||
+ (syscall(SYS_gettid) != getpid())) {
+ ODP_ERR("shm_init_global() must be performed by the main "
+ "ODP process!\n.");
+ return -1;
+ }
+
+ /* Allocate space for the internal shared mem block table */
+ addr = mmap(NULL, sizeof(shm_table_t), PROT_READ | PROT_WRITE,
+ MAP_SHARED | MAP_ANONYMOUS, -1, 0);
+ if (addr == MAP_FAILED) {
+ ODP_ERR("Unable to mmap the shm block table\n");
+ return -1;
+ }
+
+ shm_tbl = addr;
+ memset(shm_tbl, 0, sizeof(shm_table_t));
+
+ odp_spinlock_init(&shm_tbl->lock);
+
+ return 0;
+}
+
+int _odp_shm_init_local(void)
+{
+ return 0;
+}
+
+int _odp_shm_term_global(void)
+{
+ shm_block_t *block;
+ int idx;
+
+ if ((getpid() != odp_global_data.main_pid) ||
+ (syscall(SYS_gettid) != getpid())) {
+ ODP_ERR("shm_term_global() must be performed by the main "
+ "ODP process!\n.");
+ return -1;
+ }
+
+ /* Cleanup possibly non freed memory (and complain a bit) */
+ for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) {
+ block = &shm_tbl->block[idx];
+ if (block->mz) {
+ ODP_ERR("block '%s' was never freed (cleaning up...)\n",
+ block->name);
+ rte_memzone_free(block->mz);
+ }
+ }
+ /* Free the shared memory block table */
+ if (munmap(shm_tbl, sizeof(shm_table_t)) < 0) {
+ ODP_ERR("Unable to munmap the shm block table\n");
+ return -1;
+ }
+ return 0;
+}
+
+int _odp_shm_term_local(void)
+{
+ return 0;
+}
+
+int odp_shm_capability(odp_shm_capability_t *capa)
+{
+ memset(capa, 0, sizeof(odp_shm_capability_t));
+
+ capa->max_blocks = ODP_CONFIG_SHM_BLOCKS;
+ capa->max_size = 0;
+ capa->max_align = SHM_MAX_ALIGN;
+
+ return 0;
+}
+
+odp_shm_t odp_shm_reserve(const char *name, uint64_t size, uint64_t align,
+ uint32_t flags)
+{
+ shm_block_t *block;
+ const struct rte_memzone *mz;
+ char mz_name[RTE_MEMZONE_NAMESIZE];
+ uint32_t mz_flags = RTE_MEMZONE_1GB | RTE_MEMZONE_SIZE_HINT_ONLY;
+ int idx;
+
+ if (align > SHM_MAX_ALIGN) {
+ ODP_ERR("Align too large: %" PRIu64 "\n", align);
+ return ODP_SHM_INVALID;
+ }
+
+ /* DPDK requires alignment to be power of two */
+ if (!rte_is_power_of_2(align))
+ align = ROUNDUP_POWER2_U32(align);
+
+ odp_spinlock_lock(&shm_tbl->lock);
+
+ idx = find_free_block();
+ if (idx < 0) {
+ odp_spinlock_unlock(&shm_tbl->lock);
+ ODP_ERR("No free SHM blocks left\n");
+ return ODP_SHM_INVALID;
+ }
+ block = &shm_tbl->block[idx];
+
+ /* DPDK requires unique memzone names */
+ name_to_mz_name(name, mz_name);
+ /* Reserve extra space for storing shm zone descriptor */
+ mz = rte_memzone_reserve_aligned(mz_name, size + sizeof(shm_zone_t),
+ rte_socket_id(), mz_flags, align);
+ if (mz == NULL) {
+ odp_spinlock_unlock(&shm_tbl->lock);
+ ODP_ERR("Reserving DPDK memzone failed\n");
+ return ODP_SHM_INVALID;
+ }
+
+ block->mz = mz;
+ snprintf(block->name, ODP_SHM_NAME_LEN - 1, "%s", name);
+ block->name[ODP_SHM_NAME_LEN - 1] = 0;
+ block->type = SHM_TYPE_LOCAL;
+ /* Note: ODP_SHM_SW_ONLY/ODP_SHM_PROC/ODP_SHM_SINGLE_VA flags are
+ * currently ignored. */
+ shm_zone(mz)->flags = flags;
+
+ odp_spinlock_unlock(&shm_tbl->lock);
+
+ return idx_to_handle(idx);
+}
+
+odp_shm_t odp_shm_import(const char *remote_name, odp_instance_t odp_inst,
+ const char *local_name)
+{
+ shm_block_t *block;
+ const struct rte_memzone *mz;
+ char mz_name[RTE_MEMZONE_NAMESIZE];
+ int idx;
+
+ snprintf(mz_name, RTE_MEMZONE_NAMESIZE - 1, SHM_BLOCK_NAME, odp_inst, 0,
+ remote_name);
+ mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0;
+
+ mz = rte_memzone_lookup(mz_name);
+ if (mz == NULL) {
+ ODP_ERR("Unable to find remote SHM block: %s\n", remote_name);
+ return ODP_SHM_INVALID;
+ }
+
+ if (!(shm_zone(mz)->flags & ODP_SHM_EXPORT)) {
+ ODP_ERR("Not exported SHM block!\n");
+ return ODP_SHM_INVALID;
+ }
+
+ odp_spinlock_lock(&shm_tbl->lock);
+
+ idx = find_free_block();
+ if (idx < 0) {
+ odp_spinlock_unlock(&shm_tbl->lock);
+ ODP_ERR("No free SHM blocks left\n");
+ return ODP_SHM_INVALID;
+ }
+ block = &shm_tbl->block[idx];
+
+ block->mz = mz;
+ snprintf(block->name, ODP_SHM_NAME_LEN - 1, "%s", local_name);
+ block->name[ODP_SHM_NAME_LEN - 1] = 0;
+ block->type = SHM_TYPE_REMOTE;
+
+ odp_spinlock_unlock(&shm_tbl->lock);
+
+ return idx_to_handle(idx);
+}
+
+int odp_shm_free(odp_shm_t shm)
+{
+ shm_block_t *block;
+ int ret = 0;
+
+ odp_spinlock_lock(&shm_tbl->lock);
+
+ if (!handle_is_valid(shm)) {
+ odp_spinlock_unlock(&shm_tbl->lock);
+ return -1;
+ }
+
+ block = &shm_tbl->block[handle_to_idx(shm)];
+
+ /* Only the creator of memzone can free it */
+ if (block->type == SHM_TYPE_LOCAL)
+ ret = rte_memzone_free(block->mz);
+
+ block->mz = NULL;
+
+ odp_spinlock_unlock(&shm_tbl->lock);
+
+ return ret;
+}
+
+odp_shm_t odp_shm_lookup(const char *name)
+{
+ int idx;
+
+ odp_spinlock_lock(&shm_tbl->lock);
+
+ for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) {
+ if (shm_tbl->block[idx].mz &&
+ strncmp(name, shm_tbl->block[idx].name,
+ ODP_SHM_NAME_LEN) == 0) {
+ odp_spinlock_unlock(&shm_tbl->lock);
+ return idx_to_handle(idx);
+ }
+ }
+
+ odp_spinlock_unlock(&shm_tbl->lock);
+
+ return ODP_SHM_INVALID;
+}
+
+void *odp_shm_addr(odp_shm_t shm)
+{
+ void *addr;
+
+ odp_spinlock_lock(&shm_tbl->lock);
+
+ if (!handle_is_valid(shm)) {
+ odp_spinlock_unlock(&shm_tbl->lock);
+ return NULL;
+ }
+
+ addr = shm_tbl->block[handle_to_idx(shm)].mz->addr;
+
+ odp_spinlock_unlock(&shm_tbl->lock);
+
+ return addr;
+}
+
+int odp_shm_info(odp_shm_t shm, odp_shm_info_t *info)
+{
+ shm_block_t *block;
+ int idx = handle_to_idx(shm);
+
+ odp_spinlock_lock(&shm_tbl->lock);
+
+ if (!handle_is_valid(shm)) {
+ odp_spinlock_unlock(&shm_tbl->lock);
+ return -1;
+ }
+
+ block = &shm_tbl->block[idx];
+
+ memset(info, 0, sizeof(odp_shm_info_t));
+
+ info->name = block->name;
+ info->addr = block->mz->addr;
+ info->size = shm_size(block->mz);
+ info->page_size = block->mz->hugepage_sz;
+ info->flags = shm_zone(block->mz)->flags;
+
+ odp_spinlock_unlock(&shm_tbl->lock);
+
+ return 0;
+}
+
+void odp_shm_print_all(void)
+{
+ shm_block_t *block;
+ int idx;
+
+ odp_spinlock_lock(&shm_tbl->lock);
+
+ printf("\nShared memory blocks\n--------------------\n");
+
+ for (idx = 0; idx < ODP_CONFIG_SHM_BLOCKS; idx++) {
+ block = &shm_tbl->block[idx];
+ if (block->mz == NULL)
+ continue;
+ printf(" %s: addr: %p, len: %" PRIu64 " page size: "
+ "%" PRIu64 "\n", block->name, block->mz->addr,
+ shm_size(block->mz), block->mz->hugepage_sz);
+ }
+
+ odp_spinlock_unlock(&shm_tbl->lock);
+}
+
+uint64_t odp_shm_to_u64(odp_shm_t hdl)
+{
+ return _odp_pri(hdl);
+}
diff --git a/test/common_plat/validation/api/shmem/shmem.c b/test/common_plat/validation/api/shmem/shmem.c
index 0e757a7..9a54ecc 100644
--- a/test/common_plat/validation/api/shmem/shmem.c
+++ b/test/common_plat/validation/api/shmem/shmem.c
@@ -96,8 +96,8 @@ static int run_test_basic_thread(void *arg ODP_UNUSED)
CU_ASSERT(0 == info.flags);
CU_ASSERT(shared_test_data == info.addr);
CU_ASSERT(sizeof(shared_test_data_t) <= info.size);
- CU_ASSERT((info.page_size == odp_sys_huge_page_size()) ||
- (info.page_size == odp_sys_page_size()))
+ CU_ASSERT(info.page_size > 0);
+
odp_shm_print_all();
fflush(stdout);