linux-gen: timer: decrease inline timer polling interval under load

Decrease inline timer polling interval after receiving events to
compansate for event processing delay.

Signed-off-by: Matias Elo <matias.elo@nokia.com>
Reviewed-by: Bill Fischofer <bill.fischofer@linaro.org>
Signed-off-by: Maxim Uvarov <maxim.uvarov@linaro.org>

7 files changed, 40 insertions, 27 deletions

diff --git a/platform/linux-generic/include/odp_timer_internal.h b/platform/linux-generic/include/odp_timer_internal.h
index 1f612448e..cd80778a5 100644
--- a/platform/linux-generic/include/odp_timer_internal.h
+++ b/platform/linux-generic/include/odp_timer_internal.h
@@ -36,12 +36,14 @@ typedef struct {
 	odp_timer_t timer;
 } odp_timeout_hdr_t;
 
-unsigned _timer_run(void);
+/* A larger decrement value should be used after receiving events compared to
+ * an 'empty' call. */
+unsigned int _timer_run(int dec);
 
 /* Static inline wrapper to minimize modification of schedulers. */
-static inline unsigned timer_run(void)
+static inline unsigned int timer_run(int dec)
 {
-	return odp_global_rw->inline_timers ? _timer_run() : 0;
+	return odp_global_rw->inline_timers ? _timer_run(dec) : 0;
 }
 
 #endif
diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c
index c36548848..37a2fad12 100644
--- a/platform/linux-generic/odp_queue_basic.c
+++ b/platform/linux-generic/odp_queue_basic.c
@@ -984,27 +984,30 @@ static int queue_api_enq(odp_queue_t handle, odp_event_t ev)
 static int queue_api_deq_multi(odp_queue_t handle, odp_event_t ev[], int num)
 {
 	queue_entry_t *queue = qentry_from_handle(handle);
-
-	if (odp_global_rw->inline_timers &&
-	    odp_atomic_load_u64(&queue->s.num_timers))
-		timer_run();
+	int ret;
 
 	if (num > QUEUE_MULTI_MAX)
 		num = QUEUE_MULTI_MAX;
 
-	return queue->s.dequeue_multi(handle,
-				      (odp_buffer_hdr_t **)ev, num);
+	ret = queue->s.dequeue_multi(handle, (odp_buffer_hdr_t **)ev, num);
+
+	if (odp_global_rw->inline_timers &&
+	    odp_atomic_load_u64(&queue->s.num_timers))
+		timer_run(ret ? 2 : 1);
+
+	return ret;
 }
 
 static odp_event_t queue_api_deq(odp_queue_t handle)
 {
 	queue_entry_t *queue = qentry_from_handle(handle);
+	odp_event_t ev = (odp_event_t)queue->s.dequeue(handle);
 
 	if (odp_global_rw->inline_timers &&
 	    odp_atomic_load_u64(&queue->s.num_timers))
-		timer_run();
+		timer_run(ev != ODP_EVENT_INVALID ? 2 : 1);
 
-	return (odp_event_t)queue->s.dequeue(handle);
+	return ev;
 }
 
 /* API functions */
diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c
index 2de4d52d8..88abe8c78 100644
--- a/platform/linux-generic/odp_queue_scalable.c
+++ b/platform/linux-generic/odp_queue_scalable.c
@@ -850,30 +850,32 @@ static odp_buffer_hdr_t *_queue_deq(odp_queue_t handle)
 static int queue_deq_multi(odp_queue_t handle, odp_event_t ev[], int num)
 {
 	queue_entry_t *queue;
+	int ret;
 
 	if (num > QUEUE_MULTI_MAX)
 		num = QUEUE_MULTI_MAX;
 
 	queue = qentry_from_ext(handle);
 
+	ret = queue->s.dequeue_multi(handle, (odp_buffer_hdr_t **)ev, num);
+
 	if (odp_global_rw->inline_timers &&
 	    odp_atomic_load_u64(&queue->s.num_timers))
-		timer_run();
+		timer_run(ret ? 2 : 1);
 
-	return queue->s.dequeue_multi(handle, (odp_buffer_hdr_t **)ev, num);
+	return ret;
 }
 
 static odp_event_t queue_deq(odp_queue_t handle)
 {
-	queue_entry_t *queue;
-
-	queue = qentry_from_ext(handle);
+	queue_entry_t *queue = qentry_from_ext(handle);
+	odp_event_t ev = (odp_event_t)queue->s.dequeue(handle);
 
 	if (odp_global_rw->inline_timers &&
 	    odp_atomic_load_u64(&queue->s.num_timers))
-		timer_run();
+		timer_run(ev != ODP_EVENT_INVALID ? 2 : 1);
 
-	return (odp_event_t)queue->s.dequeue(handle);
+	return ev;
 }
 
 static void queue_param_init(odp_queue_param_t *params)
diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c
index cf566eb88..48f232e6b 100644
--- a/platform/linux-generic/odp_schedule_basic.c
+++ b/platform/linux-generic/odp_schedule_basic.c
@@ -1181,7 +1181,7 @@ static inline int do_schedule(odp_queue_t *out_queue, odp_event_t out_ev[],
 static inline int schedule_run(odp_queue_t *out_queue, odp_event_t out_ev[],
 			       unsigned int max_num)
 {
-	timer_run();
+	timer_run(1);
 
 	return do_schedule(out_queue, out_ev, max_num);
 }
@@ -1194,10 +1194,13 @@ static inline int schedule_loop(odp_queue_t *out_queue, uint64_t wait,
 	int ret;
 
 	while (1) {
-		ret = schedule_run(out_queue, out_ev, max_num);
 
-		if (ret)
+		ret = do_schedule(out_queue, out_ev, max_num);
+		if (ret) {
+			timer_run(2);
 			break;
+		}
+		timer_run(1);
 
 		if (wait == ODP_SCHED_WAIT)
 			continue;
diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c
index 5d83ed088..4e9dd7717 100644
--- a/platform/linux-generic/odp_schedule_scalable.c
+++ b/platform/linux-generic/odp_schedule_scalable.c
@@ -883,7 +883,7 @@ static int _schedule(odp_queue_t *from, odp_event_t ev[], int num_evts)
 	ts = sched_ts;
 	atomq = ts->atomq;
 
-	timer_run();
+	timer_run(1);
 
 	/* Once an atomic queue has been scheduled to a thread, it will stay
 	 * on that thread until empty or 'rotated' by WRR
diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c
index b8f805ac0..eec88a60a 100644
--- a/platform/linux-generic/odp_schedule_sp.c
+++ b/platform/linux-generic/odp_schedule_sp.c
@@ -571,8 +571,6 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait,
 		uint32_t qi;
 		int num;
 
-		timer_run();
-
 		cmd = sched_cmd();
 
 		if (cmd && cmd->s.type == CMD_PKTIO) {
@@ -612,6 +610,7 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait,
 		}
 
 		if (cmd == NULL) {
+			timer_run(1);
 			/* All priority queues are empty */
 			if (wait == ODP_SCHED_NO_WAIT)
 				return 0;
@@ -636,6 +635,7 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait,
 		num = sched_queue_deq(qi, events, 1, 1);
 
 		if (num <= 0) {
+			timer_run(1);
 			/* Destroyed or empty queue. Remove empty queue from
 			 * scheduling. A dequeue operation to on an already
 			 * empty queue moves it to NOTSCHED state and
@@ -643,6 +643,8 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait,
 			continue;
 		}
 
+		timer_run(2);
+
 		sched_local.cmd = cmd;
 
 		if (from)
diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c
index d502c21ff..6446b1c91 100644
--- a/platform/linux-generic/odp_timer.c
+++ b/platform/linux-generic/odp_timer.c
@@ -883,10 +883,10 @@ static unsigned process_timer_pools(void)
 	return nexp;
 }
 
-unsigned _timer_run(void)
+unsigned int _timer_run(int dec)
 {
 	static __thread odp_time_t last_timer_run;
-	static __thread unsigned timer_run_cnt = 1;
+	static __thread int timer_run_cnt = 1;
 	odp_time_t now;
 
 	if (timer_global->num_timer_pools == 0)
@@ -895,7 +895,8 @@ unsigned _timer_run(void)
 	/* Rate limit how often this thread checks the timer pools. */
 
 	if (timer_global->inline_poll_interval > 1) {
-		if (--timer_run_cnt)
+		timer_run_cnt -= dec;
+		if (timer_run_cnt > 0)
 			return 0;
 		timer_run_cnt = timer_global->inline_poll_interval;
 	}