1 files changed, 390 insertions, 132 deletions

diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c
index d2ba539ce..1ada5581b 100644
--- a/platform/linux-generic/odp_schedule_sp.c
+++ b/platform/linux-generic/odp_schedule_sp.c
@@ -1,30 +1,50 @@
-/* Copyright (c) 2016, Linaro Limited
- * All rights reserved.
- *
- * SPDX-License-Identifier:     BSD-3-Clause
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2016-2018 Linaro Limited
+ * Copyright (c) 2019-2021 Nokia
  */
 
-#include <string.h>
+/*
+ * Suppress bounds warnings about interior zero length arrays. Such an array
+ * is used intentionally in prio_queue_t.
+ */
+#if __GNUC__ >= 10
+#pragma GCC diagnostic ignored "-Wzero-length-bounds"
+#endif
+
+#include <odp/api/packet.h>
 #include <odp/api/ticketlock.h>
 #include <odp/api/thread.h>
+#include <odp/api/plat/thread_inlines.h>
 #include <odp/api/time.h>
+#include <odp/api/plat/time_inlines.h>
 #include <odp/api/schedule.h>
 #include <odp/api/shared_memory.h>
+
+#include <odp/api/plat/schedule_inline_types.h>
+
 #include <odp_schedule_if.h>
 #include <odp_debug_internal.h>
-#include <odp_align_internal.h>
 #include <odp_config_internal.h>
-#include <odp_ring_internal.h>
+#include <odp_event_internal.h>
+#include <odp_macros_internal.h>
+#include <odp_ring_u32_internal.h>
+#include <odp_timer_internal.h>
+#include <odp_queue_basic_internal.h>
+#include <odp_string_internal.h>
+#include <odp_global_data.h>
+
+#include <string.h>
 
 #define NUM_THREAD        ODP_THREAD_COUNT_MAX
-#define NUM_QUEUE         ODP_CONFIG_QUEUES
-#define NUM_PKTIO         ODP_CONFIG_PKTIO_ENTRIES
+#define NUM_QUEUE         CONFIG_MAX_SCHED_QUEUES
+#define NUM_PKTIO         CONFIG_PKTIO_ENTRIES
 #define NUM_ORDERED_LOCKS 1
-#define NUM_PRIO          3
 #define NUM_STATIC_GROUP  3
 #define NUM_GROUP         (NUM_STATIC_GROUP + 9)
 #define NUM_PKTIN         32
-#define LOWEST_QUEUE_PRIO (NUM_PRIO - 2)
+#define NUM_PRIO          3
+#define MAX_API_PRIO      (NUM_PRIO - 2)
+/* Lowest internal priority */
 #define PKTIN_PRIO        (NUM_PRIO - 1)
 #define CMD_QUEUE         0
 #define CMD_PKTIO         1
@@ -34,20 +54,17 @@
 #define GROUP_PKTIN       GROUP_ALL
 
 /* Maximum number of commands: one priority/group for all queues and pktios */
-#define RING_SIZE         (ROUNDUP_POWER2_U32(NUM_QUEUE + NUM_PKTIO))
+#define RING_SIZE         (_ODP_ROUNDUP_POWER2_U32(NUM_QUEUE + NUM_PKTIO))
 #define RING_MASK         (RING_SIZE - 1)
 
 /* Ring size must be power of two */
-ODP_STATIC_ASSERT(CHECK_IS_POWER2(RING_SIZE),
+ODP_STATIC_ASSERT(_ODP_CHECK_IS_POWER2(RING_SIZE),
 		  "Ring_size_is_not_power_of_two");
 
 ODP_STATIC_ASSERT(NUM_ORDERED_LOCKS <= CONFIG_QUEUE_MAX_ORD_LOCKS,
 		  "Too_many_ordered_locks");
 
-struct sched_cmd_t;
-
-struct sched_cmd_s {
-	struct sched_cmd_t *next;
+typedef struct ODP_ALIGNED_CACHE {
 	uint32_t           index;
 	uint32_t           ring_idx;
 	int                type;
@@ -56,22 +73,20 @@ struct sched_cmd_s {
 	int                init;
 	int                num_pktin;
 	int                pktin_idx[NUM_PKTIN];
-};
-
-typedef struct sched_cmd_t {
-	struct sched_cmd_s s;
-	uint8_t            pad[ROUNDUP_CACHE_LINE(sizeof(struct sched_cmd_s)) -
-			       sizeof(struct sched_cmd_s)];
-} sched_cmd_t ODP_ALIGNED_CACHE;
+	odp_queue_t        queue[NUM_PKTIN];
+} sched_cmd_t;
 
-typedef struct {
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpedantic"
+typedef struct ODP_ALIGNED_CACHE {
 	/* Ring header */
-	ring_t ring;
+	ring_u32_t ring;
 
 	/* Ring data: queue indexes */
-	uint32_t ring_idx[RING_SIZE];
+	uint32_t ring_idx[RING_SIZE]; /* overlaps with ring.data[] */
 
-} prio_queue_t ODP_ALIGNED_CACHE;
+} prio_queue_t;
+#pragma GCC diagnostic pop
 
 typedef struct thr_group_t {
 	/* A generation counter for fast comparison if groups have changed */
@@ -85,13 +100,13 @@ typedef struct thr_group_t {
 
 } thr_group_t;
 
-typedef struct sched_group_t {
+typedef struct ODP_ALIGNED_CACHE sched_group_t {
 	struct {
 		odp_ticketlock_t  lock;
 
 		/* All groups */
 		struct {
-			char          name[ODP_SCHED_GROUP_NAME_LEN + 1];
+			char          name[ODP_SCHED_GROUP_NAME_LEN];
 			odp_thrmask_t mask;
 			int           allocated;
 		} group[NUM_GROUP];
@@ -101,14 +116,20 @@ typedef struct sched_group_t {
 
 	} s;
 
-} sched_group_t ODP_ALIGNED_CACHE;
+} sched_group_t;
 
 typedef struct {
 	sched_cmd_t   queue_cmd[NUM_QUEUE];
 	sched_cmd_t   pktio_cmd[NUM_PKTIO];
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpedantic"
 	prio_queue_t  prio_queue[NUM_GROUP][NUM_PRIO];
+#pragma GCC diagnostic pop
 	sched_group_t sched_group;
 	odp_shm_t     shm;
+	/* Scheduler interface config options (not used in fast path) */
+	schedule_config_t config_if;
 } sched_global_t;
 
 typedef struct {
@@ -123,6 +144,8 @@ typedef struct {
 static sched_global_t *sched_global;
 static __thread sched_local_t sched_local;
 
+static void remove_group(sched_group_t *sched_group, int thr, int group);
+
 static inline uint32_t index_to_ring_idx(int pktio, uint32_t index)
 {
 	if (pktio)
@@ -149,16 +172,16 @@ static int init_global(void)
 	odp_shm_t shm;
 	sched_group_t *sched_group = NULL;
 
-	ODP_DBG("Using SP scheduler\n");
+	_ODP_DBG("Using SP scheduler\n");
 
-	shm = odp_shm_reserve("sp_scheduler",
+	shm = odp_shm_reserve("_odp_sched_sp_global",
 			      sizeof(sched_global_t),
 			      ODP_CACHE_LINE_SIZE, 0);
 
 	sched_global = odp_shm_addr(shm);
 
 	if (sched_global == NULL) {
-		ODP_ERR("Schedule init: Shm reserve failed.\n");
+		_ODP_ERR("Schedule init: Shm reserve failed.\n");
 		return -1;
 	}
 
@@ -166,22 +189,22 @@ static int init_global(void)
 	sched_global->shm = shm;
 
 	for (i = 0; i < NUM_QUEUE; i++) {
-		sched_global->queue_cmd[i].s.type     = CMD_QUEUE;
-		sched_global->queue_cmd[i].s.index    = i;
-		sched_global->queue_cmd[i].s.ring_idx = index_to_ring_idx(0, i);
+		sched_global->queue_cmd[i].type     = CMD_QUEUE;
+		sched_global->queue_cmd[i].index    = i;
+		sched_global->queue_cmd[i].ring_idx = index_to_ring_idx(0, i);
 	}
 
 	for (i = 0; i < NUM_PKTIO; i++) {
-		sched_global->pktio_cmd[i].s.type     = CMD_PKTIO;
-		sched_global->pktio_cmd[i].s.index    = i;
-		sched_global->pktio_cmd[i].s.ring_idx = index_to_ring_idx(1, i);
-		sched_global->pktio_cmd[i].s.prio     = PKTIN_PRIO;
-		sched_global->pktio_cmd[i].s.group    = GROUP_PKTIN;
+		sched_global->pktio_cmd[i].type     = CMD_PKTIO;
+		sched_global->pktio_cmd[i].index    = i;
+		sched_global->pktio_cmd[i].ring_idx = index_to_ring_idx(1, i);
+		sched_global->pktio_cmd[i].prio     = PKTIN_PRIO;
+		sched_global->pktio_cmd[i].group    = GROUP_PKTIN;
 	}
 
 	for (i = 0; i < NUM_GROUP; i++)
 		for (j = 0; j < NUM_PRIO; j++)
-			ring_init(&sched_global->prio_queue[i][j].ring);
+			ring_u32_init(&sched_global->prio_queue[i][j].ring);
 
 	sched_group = &sched_global->sched_group;
 	odp_ticketlock_init(&sched_group->s.lock);
@@ -189,21 +212,25 @@ static int init_global(void)
 	for (i = 0; i < NUM_THREAD; i++)
 		odp_atomic_init_u32(&sched_group->s.thr[i].gen_cnt, 0);
 
-	strncpy(sched_group->s.group[GROUP_ALL].name, "__group_all",
-		ODP_SCHED_GROUP_NAME_LEN);
+	_odp_strcpy(sched_group->s.group[GROUP_ALL].name, "__group_all",
+		    ODP_SCHED_GROUP_NAME_LEN);
 	odp_thrmask_zero(&sched_group->s.group[GROUP_ALL].mask);
 	sched_group->s.group[GROUP_ALL].allocated = 1;
 
-	strncpy(sched_group->s.group[GROUP_WORKER].name, "__group_worker",
-		ODP_SCHED_GROUP_NAME_LEN);
+	_odp_strcpy(sched_group->s.group[GROUP_WORKER].name, "__group_worker",
+		    ODP_SCHED_GROUP_NAME_LEN);
 	odp_thrmask_zero(&sched_group->s.group[GROUP_WORKER].mask);
 	sched_group->s.group[GROUP_WORKER].allocated = 1;
 
-	strncpy(sched_group->s.group[GROUP_CONTROL].name, "__group_control",
-		ODP_SCHED_GROUP_NAME_LEN);
+	_odp_strcpy(sched_group->s.group[GROUP_CONTROL].name, "__group_control",
+		    ODP_SCHED_GROUP_NAME_LEN);
 	odp_thrmask_zero(&sched_group->s.group[GROUP_CONTROL].mask);
 	sched_group->s.group[GROUP_CONTROL].allocated = 1;
 
+	sched_global->config_if.group_enable.all = 1;
+	sched_global->config_if.group_enable.control = 1;
+	sched_global->config_if.group_enable.worker = 1;
+
 	return 0;
 }
 
@@ -217,18 +244,26 @@ static int init_local(void)
 
 static int term_global(void)
 {
+	odp_event_t event;
 	int qi, ret = 0;
 
 	for (qi = 0; qi < NUM_QUEUE; qi++) {
-		if (sched_global->queue_cmd[qi].s.init) {
-			/* todo: dequeue until empty ? */
-			sched_cb_queue_destroy_finalize(qi);
+		int report = 1;
+
+		if (sched_global->queue_cmd[qi].init) {
+			while (_odp_sched_queue_deq(qi, &event, 1, 1) > 0) {
+				if (report) {
+					_ODP_ERR("Queue not empty\n");
+					report = 0;
+				}
+				odp_event_free(event);
+			}
 		}
 	}
 
 	ret = odp_shm_free(sched_global->shm);
 	if (ret < 0) {
-		ODP_ERR("Shm free failed for sp_scheduler");
+		_ODP_ERR("Shm free failed for sp_scheduler");
 		ret = -1;
 	}
 
@@ -240,7 +275,61 @@ static int term_local(void)
 	return 0;
 }
 
-static unsigned max_ordered_locks(void)
+static void schedule_config_init(odp_schedule_config_t *config)
+{
+	config->num_queues = CONFIG_MAX_SCHED_QUEUES;
+	config->queue_size = _odp_queue_glb->config.max_queue_size;
+	config->sched_group.all = true;
+	config->sched_group.control = true;
+	config->sched_group.worker = true;
+}
+
+static void schedule_group_clear(odp_schedule_group_t group)
+{
+	sched_group_t *sched_group = &sched_global->sched_group;
+	int thr;
+	const odp_thrmask_t *thrmask;
+
+	if (group < 0 || group >= NUM_STATIC_GROUP)
+		_ODP_ABORT("Invalid scheduling group\n");
+
+	thrmask = &sched_group->s.group[group].mask;
+
+	thr = odp_thrmask_first(thrmask);
+	while (thr >= 0) {
+		remove_group(sched_group, thr, group);
+		thr = odp_thrmask_next(thrmask, thr);
+	}
+
+	memset(&sched_group->s.group[group], 0, sizeof(sched_group->s.group[0]));
+}
+
+static int schedule_config(const odp_schedule_config_t *config)
+{
+	sched_group_t *sched_group = &sched_global->sched_group;
+
+	odp_ticketlock_lock(&sched_group->s.lock);
+
+	sched_global->config_if.group_enable.all = config->sched_group.all;
+	sched_global->config_if.group_enable.control = config->sched_group.control;
+	sched_global->config_if.group_enable.worker = config->sched_group.worker;
+
+	/* Remove existing threads from predefined scheduling groups. */
+	if (!config->sched_group.all)
+		schedule_group_clear(ODP_SCHED_GROUP_ALL);
+
+	if (!config->sched_group.worker)
+		schedule_group_clear(ODP_SCHED_GROUP_WORKER);
+
+	if (!config->sched_group.control)
+		schedule_group_clear(ODP_SCHED_GROUP_CONTROL);
+
+	odp_ticketlock_unlock(&sched_group->s.lock);
+
+	return 0;
+}
+
+static uint32_t max_ordered_locks(void)
 {
 	return NUM_ORDERED_LOCKS;
 }
@@ -255,7 +344,7 @@ static void add_group(sched_group_t *sched_group, int thr, int group)
 	thr_group->group[num] = group;
 	thr_group->num_group  = num + 1;
 	gen_cnt = odp_atomic_load_u32(&thr_group->gen_cnt);
-	odp_atomic_store_u32(&thr_group->gen_cnt, gen_cnt + 1);
+	odp_atomic_store_rel_u32(&thr_group->gen_cnt, gen_cnt + 1);
 }
 
 static void remove_group(sched_group_t *sched_group, int thr, int group)
@@ -266,6 +355,12 @@ static void remove_group(sched_group_t *sched_group, int thr, int group)
 
 	num = thr_group->num_group;
 
+	/* Extra array bounds check to suppress warning on GCC 7.4 with -O3 */
+	if (num >= NUM_GROUP) {
+		_ODP_ERR("Too many groups");
+		return;
+	}
+
 	for (i = 0; i < num; i++) {
 		if (thr_group->group[i] == group) {
 			found = 1;
@@ -282,7 +377,7 @@ static void remove_group(sched_group_t *sched_group, int thr, int group)
 
 		thr_group->num_group = num - 1;
 		gen_cnt = odp_atomic_load_u32(&thr_group->gen_cnt);
-		odp_atomic_store_u32(&thr_group->gen_cnt, gen_cnt + 1);
+		odp_atomic_store_rel_u32(&thr_group->gen_cnt, gen_cnt + 1);
 	}
 }
 
@@ -290,7 +385,7 @@ static int thr_add(odp_schedule_group_t group, int thr)
 {
 	sched_group_t *sched_group = &sched_global->sched_group;
 
-	if (group < 0 || group >= NUM_GROUP)
+	if (group < 0 || group >= NUM_STATIC_GROUP)
 		return -1;
 
 	if (thr < 0 || thr >= NUM_THREAD)
@@ -300,7 +395,7 @@ static int thr_add(odp_schedule_group_t group, int thr)
 
 	if (!sched_group->s.group[group].allocated) {
 		odp_ticketlock_unlock(&sched_group->s.lock);
-		return -1;
+		return 0;
 	}
 
 	odp_thrmask_set(&sched_group->s.group[group].mask, thr);
@@ -315,14 +410,14 @@ static int thr_rem(odp_schedule_group_t group, int thr)
 {
 	sched_group_t *sched_group = &sched_global->sched_group;
 
-	if (group < 0 || group >= NUM_GROUP)
+	if (group < 0 || group >= NUM_STATIC_GROUP)
 		return -1;
 
 	odp_ticketlock_lock(&sched_group->s.lock);
 
 	if (!sched_group->s.group[group].allocated) {
 		odp_ticketlock_unlock(&sched_group->s.lock);
-		return -1;
+		return 0;
 	}
 
 	odp_thrmask_clr(&sched_group->s.group[group].mask, thr);
@@ -339,44 +434,49 @@ static int num_grps(void)
 	return NUM_GROUP - NUM_STATIC_GROUP;
 }
 
-static int init_queue(uint32_t qi, const odp_schedule_param_t *sched_param)
+static int create_queue(uint32_t qi, const odp_schedule_param_t *sched_param)
 {
 	sched_group_t *sched_group = &sched_global->sched_group;
 	odp_schedule_group_t group = sched_param->group;
 	int prio = 0;
 
+	if (odp_global_rw->schedule_configured == 0) {
+		_ODP_ERR("Scheduler has not been configured\n");
+		return -1;
+	}
+
 	if (group < 0 || group >= NUM_GROUP)
 		return -1;
 
 	if (!sched_group->s.group[group].allocated)
 		return -1;
 
-	if (sched_param->prio > 0)
-		prio = LOWEST_QUEUE_PRIO;
+	/* Inverted prio value (max = 0) vs API */
+	prio = MAX_API_PRIO - sched_param->prio;
 
-	sched_global->queue_cmd[qi].s.prio  = prio;
-	sched_global->queue_cmd[qi].s.group = group;
-	sched_global->queue_cmd[qi].s.init  = 1;
+	sched_global->queue_cmd[qi].prio  = prio;
+	sched_global->queue_cmd[qi].group = group;
+	sched_global->queue_cmd[qi].init  = 1;
 
 	return 0;
 }
 
 static void destroy_queue(uint32_t qi)
 {
-	sched_global->queue_cmd[qi].s.prio  = 0;
-	sched_global->queue_cmd[qi].s.group = 0;
-	sched_global->queue_cmd[qi].s.init  = 0;
+	sched_global->queue_cmd[qi].prio  = 0;
+	sched_global->queue_cmd[qi].group = 0;
+	sched_global->queue_cmd[qi].init  = 0;
 }
 
 static inline void add_tail(sched_cmd_t *cmd)
 {
 	prio_queue_t *prio_queue;
-	int group    = cmd->s.group;
-	int prio     = cmd->s.prio;
-	uint32_t idx = cmd->s.ring_idx;
+	int group    = cmd->group;
+	int prio     = cmd->prio;
+	uint32_t idx = cmd->ring_idx;
 
 	prio_queue = &sched_global->prio_queue[group][prio];
-	ring_enq(&prio_queue->ring, RING_MASK, idx);
+	ring_u32_enq(&prio_queue->ring, RING_MASK, idx);
 }
 
 static inline sched_cmd_t *rem_head(int group, int prio)
@@ -386,9 +486,8 @@ static inline sched_cmd_t *rem_head(int group, int prio)
 	int pktio;
 
 	prio_queue = &sched_global->prio_queue[group][prio];
-	ring_idx = ring_deq(&prio_queue->ring, RING_MASK);
 
-	if (ring_idx == RING_EMPTY)
+	if (ring_u32_deq(&prio_queue->ring, RING_MASK, &ring_idx) == 0)
 		return NULL;
 
 	pktio = index_from_ring_idx(&index, ring_idx);
@@ -409,10 +508,10 @@ static int sched_queue(uint32_t qi)
 	return 0;
 }
 
-static int ord_enq_multi(queue_t q_int, void *buf_hdr[], int num,
+static int ord_enq_multi(odp_queue_t queue, void *buf_hdr[], int num,
 			 int *ret)
 {
-	(void)q_int;
+	(void)queue;
 	(void)buf_hdr;
 	(void)num;
 	(void)ret;
@@ -421,24 +520,27 @@ static int ord_enq_multi(queue_t q_int, void *buf_hdr[], int num,
 	return 0;
 }
 
-static void pktio_start(int pktio_index, int num, int pktin_idx[])
+static void pktio_start(int pktio_index,
+			int num,
+			int pktin_idx[],
+			odp_queue_t queue[])
 {
 	int i;
 	sched_cmd_t *cmd;
 
-	ODP_DBG("pktio index: %i, %i pktin queues %i\n",
-		pktio_index, num, pktin_idx[0]);
+	_ODP_DBG("pktio index: %i, %i pktin queues %i\n", pktio_index, num, pktin_idx[0]);
 
 	cmd = &sched_global->pktio_cmd[pktio_index];
 
 	if (num > NUM_PKTIN)
-		ODP_ABORT("Supports only %i pktin queues per interface\n",
-			  NUM_PKTIN);
+		_ODP_ABORT("Supports only %i pktin queues per interface\n", NUM_PKTIN);
 
-	for (i = 0; i < num; i++)
-		cmd->s.pktin_idx[i] = pktin_idx[i];
+	for (i = 0; i < num; i++) {
+		cmd->pktin_idx[i] = pktin_idx[i];
+		cmd->queue[i]     = queue[i];
+	}
 
-	cmd->s.num_pktin = num;
+	cmd->num_pktin = num;
 
 	add_tail(cmd);
 }
@@ -490,6 +592,26 @@ static uint64_t schedule_wait_time(uint64_t ns)
 	return ns;
 }
 
+static inline void enqueue_packets(odp_queue_t queue,
+				   _odp_event_hdr_t *hdr_tbl[], int num_pkt)
+{
+	int num_enq, num_drop;
+
+	num_enq = odp_queue_enq_multi(queue, (odp_event_t *)hdr_tbl,
+				      num_pkt);
+
+	if (num_enq < 0)
+		num_enq = 0;
+
+	if (num_enq < num_pkt) {
+		num_drop = num_pkt - num_enq;
+
+		_ODP_DBG("Dropped %i packets\n", num_drop);
+		odp_packet_free_multi((odp_packet_t *)&hdr_tbl[num_enq],
+				      num_drop);
+	}
+}
+
 static int schedule_multi(odp_queue_t *from, uint64_t wait,
 			  odp_event_t events[], int max_events ODP_UNUSED)
 {
@@ -498,7 +620,7 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait,
 
 	if (sched_local.cmd) {
 		/* Continue scheduling if queue is not empty */
-		if (sched_cb_queue_empty(sched_local.cmd->s.index) == 0)
+		if (_odp_sched_queue_empty(sched_local.cmd->index) == 0)
 			add_tail(sched_local.cmd);
 
 		sched_local.cmd = NULL;
@@ -514,12 +636,34 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait,
 
 		cmd = sched_cmd();
 
-		if (cmd && cmd->s.type == CMD_PKTIO) {
-			if (sched_cb_pktin_poll(cmd->s.index, cmd->s.num_pktin,
-						cmd->s.pktin_idx)) {
-				/* Pktio stopped or closed. */
-				sched_cb_pktio_stop_finalize(cmd->s.index);
-			} else {
+		if (cmd && cmd->type == CMD_PKTIO) {
+			_odp_event_hdr_t *hdr_tbl[CONFIG_BURST_SIZE];
+			int i;
+			int num_pkt = 0;
+			int max_num = CONFIG_BURST_SIZE;
+			int pktio_idx = cmd->index;
+			int num_pktin = cmd->num_pktin;
+			int *pktin_idx = cmd->pktin_idx;
+			odp_queue_t *queue = cmd->queue;
+
+			for (i = 0; i < num_pktin; i++) {
+				num_pkt = _odp_sched_cb_pktin_poll(pktio_idx,
+								   pktin_idx[i],
+								   hdr_tbl, max_num);
+
+				if (num_pkt < 0) {
+					/* Pktio stopped or closed. */
+					_odp_sched_cb_pktio_stop_finalize(pktio_idx);
+					break;
+				}
+
+				if (num_pkt == 0)
+					continue;
+
+				enqueue_packets(queue[i], hdr_tbl, num_pkt);
+			}
+
+			if (num_pkt >= 0) {
 				/* Continue polling pktio. */
 				add_tail(cmd);
 			}
@@ -529,6 +673,7 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait,
 		}
 
 		if (cmd == NULL) {
+			timer_run(1);
 			/* All priority queues are empty */
 			if (wait == ODP_SCHED_NO_WAIT)
 				return 0;
@@ -537,8 +682,7 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait,
 				continue;
 
 			if (update_t1) {
-				t1 = odp_time_sum(odp_time_local(),
-						  odp_time_local_from_ns(wait));
+				t1 = odp_time_add_ns(odp_time_local(), wait);
 				update_t1 = 0;
 				continue;
 			}
@@ -549,31 +693,26 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait,
 			return 0;
 		}
 
-		qi  = cmd->s.index;
-		num = sched_cb_queue_deq_multi(qi, events, 1);
+		qi  = cmd->index;
+		num = _odp_sched_queue_deq(qi, events, 1, 1);
 
-		if (num > 0) {
-			sched_local.cmd = cmd;
+		if (num <= 0) {
+			timer_run(1);
+			/* Destroyed or empty queue. Remove empty queue from
+			 * scheduling. A dequeue operation to on an already
+			 * empty queue moves it to NOTSCHED state and
+			 * sched_queue() will be called on next enqueue. */
+			continue;
+		}
 
-			if (from)
-				*from = sched_cb_queue_handle(qi);
+		timer_run(2);
 
-			return num;
-		}
+		sched_local.cmd = cmd;
 
-		if (num < 0) {
-			/* Destroyed queue */
-			sched_cb_queue_destroy_finalize(qi);
-			continue;
-		}
+		if (from)
+			*from = queue_from_index(qi);
 
-		if (num == 0) {
-			/* Remove empty queue from scheduling. A dequeue
-			 * operation to on an already empty queue moves
-			 * it to NOTSCHED state and sched_queue() will
-			 * be called on next enqueue. */
-			continue;
-		}
+		return num;
 	}
 }
 
@@ -587,6 +726,18 @@ static odp_event_t schedule(odp_queue_t *from, uint64_t wait)
 	return ODP_EVENT_INVALID;
 }
 
+static int schedule_multi_wait(odp_queue_t *from, odp_event_t events[],
+			       int max_num)
+{
+	return schedule_multi(from, ODP_SCHED_WAIT, events, max_num);
+}
+
+static int schedule_multi_no_wait(odp_queue_t *from, odp_event_t events[],
+				  int max_num)
+{
+	return schedule_multi(from, ODP_SCHED_NO_WAIT, events, max_num);
+}
+
 static void schedule_pause(void)
 {
 	sched_local.pause = 1;
@@ -599,10 +750,12 @@ static void schedule_resume(void)
 
 static void schedule_release_atomic(void)
 {
+	/* Nothing to do */
 }
 
 static void schedule_release_ordered(void)
 {
+	/* Nothing to do */
 }
 
 static void schedule_prefetch(int num)
@@ -610,6 +763,21 @@ static void schedule_prefetch(int num)
 	(void)num;
 }
 
+static int schedule_min_prio(void)
+{
+	return 0;
+}
+
+static int schedule_max_prio(void)
+{
+	return MAX_API_PRIO;
+}
+
+static int schedule_default_prio(void)
+{
+	return schedule_max_prio() / 2;
+}
+
 static int schedule_num_prio(void)
 {
 	/* Lowest priority is used for pktin polling and is internal
@@ -622,7 +790,7 @@ static odp_schedule_group_t schedule_group_create(const char *name,
 {
 	odp_schedule_group_t group = ODP_SCHED_GROUP_INVALID;
 	sched_group_t *sched_group = &sched_global->sched_group;
-	int i;
+	int i, thr;
 
 	odp_ticketlock_lock(&sched_group->s.lock);
 
@@ -630,17 +798,27 @@ static odp_schedule_group_t schedule_group_create(const char *name,
 		if (!sched_group->s.group[i].allocated) {
 			char *grp_name = sched_group->s.group[i].name;
 
-			if (name == NULL) {
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Warray-bounds"
+#if __GNUC__ >= 13
+#pragma GCC diagnostic ignored "-Wstringop-overflow"
+#endif
+			if (name == NULL)
 				grp_name[0] = 0;
-			} else {
-				strncpy(grp_name, name,
-					ODP_SCHED_GROUP_NAME_LEN - 1);
-				grp_name[ODP_SCHED_GROUP_NAME_LEN - 1] = 0;
-			}
-			odp_thrmask_copy(&sched_group->s.group[i].mask,
-					 thrmask);
+			else
+				_odp_strcpy(grp_name, name,
+					    ODP_SCHED_GROUP_NAME_LEN);
+#pragma GCC diagnostic pop
+
+			odp_thrmask_copy(&sched_group->s.group[i].mask, thrmask);
 			sched_group->s.group[i].allocated = 1;
 			group = i;
+
+			thr = odp_thrmask_first(thrmask);
+			while (thr >= 0) {
+				add_group(sched_group, thr, group);
+				thr = odp_thrmask_next(thrmask, thr);
+			}
 			break;
 		}
 	}
@@ -653,6 +831,8 @@ static odp_schedule_group_t schedule_group_create(const char *name,
 static int schedule_group_destroy(odp_schedule_group_t group)
 {
 	sched_group_t *sched_group = &sched_global->sched_group;
+	int thr;
+	const odp_thrmask_t *thrmask;
 
 	if (group < NUM_STATIC_GROUP || group >= NUM_GROUP)
 		return -1;
@@ -664,6 +844,14 @@ static int schedule_group_destroy(odp_schedule_group_t group)
 		return -1;
 	}
 
+	thrmask = &sched_group->s.group[group].mask;
+
+	thr = odp_thrmask_first(thrmask);
+	while (thr >= 0) {
+		remove_group(sched_group, thr, group);
+		thr = odp_thrmask_next(thrmask, thr);
+	}
+
 	memset(&sched_group->s.group[group], 0,
 	       sizeof(sched_group->s.group[0]));
 
@@ -804,32 +992,89 @@ static int schedule_group_info(odp_schedule_group_t group,
 	return 0;
 }
 
-static void schedule_order_lock(unsigned lock_index)
+static void schedule_order_lock(uint32_t lock_index)
 {
 	(void)lock_index;
 }
 
-static void schedule_order_unlock(unsigned lock_index)
+static void schedule_order_unlock(uint32_t lock_index)
+{
+	(void)lock_index;
+}
+
+static void schedule_order_unlock_lock(uint32_t unlock_index,
+				       uint32_t lock_index)
+{
+	(void)unlock_index;
+	(void)lock_index;
+}
+
+static void schedule_order_lock_start(uint32_t lock_index)
+{
+	(void)lock_index;
+}
+
+static void schedule_order_lock_wait(uint32_t lock_index)
 {
 	(void)lock_index;
 }
 
 static void order_lock(void)
 {
+	/* Nothing to do */
 }
 
 static void order_unlock(void)
 {
+	/* Nothing to do */
+}
+
+static int schedule_capability(odp_schedule_capability_t *capa)
+{
+	memset(capa, 0, sizeof(odp_schedule_capability_t));
+
+	capa->max_ordered_locks = max_ordered_locks();
+	capa->max_groups = num_grps();
+	capa->max_prios = schedule_num_prio();
+	capa->max_queues = CONFIG_MAX_SCHED_QUEUES;
+	capa->max_queue_size = _odp_queue_glb->config.max_queue_size;
+
+	return 0;
+}
+
+static void schedule_print(void)
+{
+	odp_schedule_capability_t capa;
+
+	(void)schedule_capability(&capa);
+
+	_ODP_PRINT("\nScheduler debug info\n");
+	_ODP_PRINT("--------------------\n");
+	_ODP_PRINT("  scheduler:         sp\n");
+	_ODP_PRINT("  max groups:        %u\n", capa.max_groups);
+	_ODP_PRINT("  max priorities:    %u\n", capa.max_prios);
+	_ODP_PRINT("\n");
+}
+
+static void get_config(schedule_config_t *config)
+{
+	*config = sched_global->config_if;
+}
+
+const _odp_schedule_api_fn_t _odp_schedule_sp_api;
+
+static const _odp_schedule_api_fn_t *sched_api(void)
+{
+	return &_odp_schedule_sp_api;
 }
 
 /* Fill in scheduler interface */
-const schedule_fn_t schedule_sp_fn = {
-	.status_sync   = 0,
+const schedule_fn_t _odp_schedule_sp_fn = {
 	.pktio_start   = pktio_start,
 	.thr_add       = thr_add,
 	.thr_rem       = thr_rem,
 	.num_grps      = num_grps,
-	.init_queue    = init_queue,
+	.create_queue  = create_queue,
 	.destroy_queue = destroy_queue,
 	.sched_queue   = sched_queue,
 	.ord_enq_multi = ord_enq_multi,
@@ -840,20 +1085,28 @@ const schedule_fn_t schedule_sp_fn = {
 	.order_lock    = order_lock,
 	.order_unlock  = order_unlock,
 	.max_ordered_locks = max_ordered_locks,
-	.unsched_queue = NULL,
-	.save_context  = NULL
+	.get_config    = get_config,
+	.sched_api     = sched_api,
 };
 
 /* Fill in scheduler API calls */
-const schedule_api_t schedule_sp_api = {
+const _odp_schedule_api_fn_t _odp_schedule_sp_api = {
 	.schedule_wait_time       = schedule_wait_time,
+	.schedule_capability      = schedule_capability,
+	.schedule_config_init     = schedule_config_init,
+	.schedule_config          = schedule_config,
 	.schedule                 = schedule,
 	.schedule_multi           = schedule_multi,
+	.schedule_multi_wait      = schedule_multi_wait,
+	.schedule_multi_no_wait   = schedule_multi_no_wait,
 	.schedule_pause           = schedule_pause,
 	.schedule_resume          = schedule_resume,
 	.schedule_release_atomic  = schedule_release_atomic,
 	.schedule_release_ordered = schedule_release_ordered,
 	.schedule_prefetch        = schedule_prefetch,
+	.schedule_min_prio        = schedule_min_prio,
+	.schedule_max_prio        = schedule_max_prio,
+	.schedule_default_prio    = schedule_default_prio,
 	.schedule_num_prio        = schedule_num_prio,
 	.schedule_group_create    = schedule_group_create,
 	.schedule_group_destroy   = schedule_group_destroy,
@@ -863,5 +1116,10 @@ const schedule_api_t schedule_sp_api = {
 	.schedule_group_thrmask   = schedule_group_thrmask,
 	.schedule_group_info      = schedule_group_info,
 	.schedule_order_lock      = schedule_order_lock,
-	.schedule_order_unlock    = schedule_order_unlock
+	.schedule_order_unlock    = schedule_order_unlock,
+	.schedule_order_unlock_lock = schedule_order_unlock_lock,
+	.schedule_order_lock_start  = schedule_order_lock_start,
+	.schedule_order_lock_wait   = schedule_order_lock_wait,
+	.schedule_order_wait      = order_lock,
+	.schedule_print           = schedule_print
 };