percpu_ida: Use local locks
the local_irq_save() + spin_lock() does not work that well on -RT
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
diff --git a/lib/percpu_ida.c b/lib/percpu_ida.c
index 6016f1d..cdd4308 100644
--- a/lib/percpu_ida.c
+++ b/lib/percpu_ida.c
@@ -27,6 +27,9 @@
#include <linux/string.h>
#include <linux/spinlock.h>
#include <linux/percpu_ida.h>
+#include <linux/locallock.h>
+
+static DEFINE_LOCAL_IRQ_LOCK(irq_off_lock);
struct percpu_ida_cpu {
/*
@@ -149,13 +152,13 @@
unsigned long flags;
int tag;
- local_irq_save(flags);
+ local_lock_irqsave(irq_off_lock, flags);
tags = this_cpu_ptr(pool->tag_cpu);
/* Fastpath */
tag = alloc_local_tag(tags);
if (likely(tag >= 0)) {
- local_irq_restore(flags);
+ local_unlock_irqrestore(irq_off_lock, flags);
return tag;
}
@@ -174,6 +177,7 @@
if (!tags->nr_free)
alloc_global_tags(pool, tags);
+
if (!tags->nr_free)
steal_tags(pool, tags);
@@ -185,7 +189,7 @@
}
spin_unlock(&pool->lock);
- local_irq_restore(flags);
+ local_unlock_irqrestore(irq_off_lock, flags);
if (tag >= 0 || state == TASK_RUNNING)
break;
@@ -197,7 +201,7 @@
schedule();
- local_irq_save(flags);
+ local_lock_irqsave(irq_off_lock, flags);
tags = this_cpu_ptr(pool->tag_cpu);
}
if (state != TASK_RUNNING)
@@ -222,7 +226,7 @@
BUG_ON(tag >= pool->nr_tags);
- local_irq_save(flags);
+ local_lock_irqsave(irq_off_lock, flags);
tags = this_cpu_ptr(pool->tag_cpu);
spin_lock(&tags->lock);
@@ -254,7 +258,7 @@
spin_unlock(&pool->lock);
}
- local_irq_restore(flags);
+ local_unlock_irqrestore(irq_off_lock, flags);
}
EXPORT_SYMBOL_GPL(percpu_ida_free);
@@ -346,7 +350,7 @@
struct percpu_ida_cpu *remote;
unsigned cpu, i, err = 0;
- local_irq_save(flags);
+ local_lock_irqsave(irq_off_lock, flags);
for_each_possible_cpu(cpu) {
remote = per_cpu_ptr(pool->tag_cpu, cpu);
spin_lock(&remote->lock);
@@ -368,7 +372,7 @@
}
spin_unlock(&pool->lock);
out:
- local_irq_restore(flags);
+ local_unlock_irqrestore(irq_off_lock, flags);
return err;
}
EXPORT_SYMBOL_GPL(percpu_ida_for_each_free);