Merge tag 'v3.14.44-rt44-lno1' of git://git.linaro.org/people/anders.roxell/linux-rt into linux-linaro-lsk-v3.14-rt
Linux 3.14.44-rt44
Changes since v3.14.43-rt42:
- irq_work: Split raised and lazy lists
- irq_work: Introduce arch_irq_work_has_interrupt()
* tag 'v3.14.44-rt44-lno1' of git://git.linaro.org/people/anders.roxell/linux-rt: (360 commits)
Linux 3.14.44-rt44 REBASE
irq_work: Split raised and lazy lists
irq_work: Introduce arch_irq_work_has_interrupt()
rt, nohz_full: fix nohz_full for PREEMPT_RT_FULL
hotplug: Use set_cpus_allowed_ptr() in sync_unplug_thread()
KVM: use simple waitqueue for vcpu->wq
KVM: lapic: mark LAPIC timer handler as irqsafe
kernel/irq_work: fix no_hz deadlock
netpoll: guard the access to dev->npinfo with rcu_read_lock/unlock_bh() for CONFIG_PREEMPT_RT_FULL=y
Revert "timers: do not raise softirq unconditionally"
fs,btrfs: fix rt deadlock on extent_buffer->lock
staging: Mark rtl8821ae as broken
timers: Reduce future __run_timers() latency for first add to empty list
timers: Reduce future __run_timers() latency for newly emptied list
timers: Reduce __run_timers() latency for empty list
timers: Track total number of timers in list
fs/aio: simple simple work
lockdep: selftest: fix warnings due to missing PREEMPT_RT conditionals
thermal: Defer thermal wakups to threads
locking: ww_mutex: fix ww_mutex vs self-deadlock
...
diff --git a/arch/alpha/include/asm/Kbuild b/arch/alpha/include/asm/Kbuild
index a73a8e2..aca4359 100644
--- a/arch/alpha/include/asm/Kbuild
+++ b/arch/alpha/include/asm/Kbuild
@@ -5,3 +5,4 @@
generic-y += trace_clock.h
generic-y += preempt.h
generic-y += hash.h
+generic-y += irq_work.h
diff --git a/arch/arc/include/asm/Kbuild b/arch/arc/include/asm/Kbuild
index 0d33629..1c828e4 100644
--- a/arch/arc/include/asm/Kbuild
+++ b/arch/arc/include/asm/Kbuild
@@ -18,6 +18,7 @@
generic-y += ioctls.h
generic-y += ipcbuf.h
generic-y += irq_regs.h
+generic-y += irq_work.h
generic-y += kmap_types.h
generic-y += kvm_para.h
generic-y += local.h
diff --git a/arch/arm/include/asm/Kbuild b/arch/arm/include/asm/Kbuild
index 3278afe..0657adc 100644
--- a/arch/arm/include/asm/Kbuild
+++ b/arch/arm/include/asm/Kbuild
@@ -10,6 +10,7 @@
generic-y += ioctl.h
generic-y += ipcbuf.h
generic-y += irq_regs.h
+generic-y += irq_work.h
generic-y += kdebug.h
generic-y += local.h
generic-y += local64.h
diff --git a/arch/arm64/include/asm/Kbuild b/arch/arm64/include/asm/Kbuild
index bd36b30..55f6c27 100644
--- a/arch/arm64/include/asm/Kbuild
+++ b/arch/arm64/include/asm/Kbuild
@@ -18,6 +18,7 @@
generic-y += ioctls.h
generic-y += ipcbuf.h
generic-y += irq_regs.h
+generic-y += irq_work.h
generic-y += kdebug.h
generic-y += kmap_types.h
generic-y += kvm_para.h
diff --git a/arch/avr32/include/asm/Kbuild b/arch/avr32/include/asm/Kbuild
index c7c64a6..83b594e 100644
--- a/arch/avr32/include/asm/Kbuild
+++ b/arch/avr32/include/asm/Kbuild
@@ -9,6 +9,7 @@
generic-y += futex.h
generic-y += preempt.h
generic-y += irq_regs.h
+generic-y += irq_work.h
generic-y += param.h
generic-y += local.h
generic-y += local64.h
diff --git a/arch/blackfin/include/asm/Kbuild b/arch/blackfin/include/asm/Kbuild
index 359d36f..c80efff 100644
--- a/arch/blackfin/include/asm/Kbuild
+++ b/arch/blackfin/include/asm/Kbuild
@@ -14,6 +14,7 @@
generic-y += ioctl.h
generic-y += ipcbuf.h
generic-y += irq_regs.h
+generic-y += irq_work.h
generic-y += kdebug.h
generic-y += kmap_types.h
generic-y += kvm_para.h
diff --git a/arch/c6x/include/asm/Kbuild b/arch/c6x/include/asm/Kbuild
index d73bb85..2250450 100644
--- a/arch/c6x/include/asm/Kbuild
+++ b/arch/c6x/include/asm/Kbuild
@@ -21,6 +21,7 @@
generic-y += ioctls.h
generic-y += ipcbuf.h
generic-y += irq_regs.h
+generic-y += irq_work.h
generic-y += kdebug.h
generic-y += kmap_types.h
generic-y += local.h
diff --git a/arch/cris/include/asm/Kbuild b/arch/cris/include/asm/Kbuild
index f3fd876..05c068a 100644
--- a/arch/cris/include/asm/Kbuild
+++ b/arch/cris/include/asm/Kbuild
@@ -14,3 +14,4 @@
generic-y += vga.h
generic-y += xor.h
generic-y += preempt.h
+generic-y += irq_work.h
diff --git a/arch/frv/include/asm/Kbuild b/arch/frv/include/asm/Kbuild
index bc42f14..9467583 100644
--- a/arch/frv/include/asm/Kbuild
+++ b/arch/frv/include/asm/Kbuild
@@ -4,3 +4,4 @@
generic-y += trace_clock.h
generic-y += preempt.h
generic-y += hash.h
+generic-y += irq_work.h
diff --git a/arch/hexagon/include/asm/Kbuild b/arch/hexagon/include/asm/Kbuild
index 38ca45d..42490ec 100644
--- a/arch/hexagon/include/asm/Kbuild
+++ b/arch/hexagon/include/asm/Kbuild
@@ -23,6 +23,7 @@
generic-y += iomap.h
generic-y += ipcbuf.h
generic-y += irq_regs.h
+generic-y += irq_work.h
generic-y += kdebug.h
generic-y += kmap_types.h
generic-y += local64.h
diff --git a/arch/ia64/include/asm/Kbuild b/arch/ia64/include/asm/Kbuild
index 283a831..0007e55 100644
--- a/arch/ia64/include/asm/Kbuild
+++ b/arch/ia64/include/asm/Kbuild
@@ -3,6 +3,7 @@
generic-y += exec.h
generic-y += kvm_para.h
generic-y += trace_clock.h
+generic-y += irq_work.h
generic-y += preempt.h
generic-y += vtime.h
generic-y += hash.h
diff --git a/arch/m32r/include/asm/Kbuild b/arch/m32r/include/asm/Kbuild
index 932435a..afd0830 100644
--- a/arch/m32r/include/asm/Kbuild
+++ b/arch/m32r/include/asm/Kbuild
@@ -5,3 +5,4 @@
generic-y += trace_clock.h
generic-y += preempt.h
generic-y += hash.h
+generic-y += irq_work.h
diff --git a/arch/m68k/include/asm/Kbuild b/arch/m68k/include/asm/Kbuild
index 6fb9e81..bb51ed1 100644
--- a/arch/m68k/include/asm/Kbuild
+++ b/arch/m68k/include/asm/Kbuild
@@ -11,6 +11,7 @@
generic-y += ioctl.h
generic-y += ipcbuf.h
generic-y += irq_regs.h
+generic-y += irq_work.h
generic-y += kdebug.h
generic-y += kmap_types.h
generic-y += kvm_para.h
diff --git a/arch/metag/include/asm/Kbuild b/arch/metag/include/asm/Kbuild
index b716d80..d64c4a6 100644
--- a/arch/metag/include/asm/Kbuild
+++ b/arch/metag/include/asm/Kbuild
@@ -18,6 +18,7 @@
generic-y += ioctls.h
generic-y += ipcbuf.h
generic-y += irq_regs.h
+generic-y += irq_work.h
generic-y += kdebug.h
generic-y += kmap_types.h
generic-y += kvm_para.h
diff --git a/arch/microblaze/include/asm/Kbuild b/arch/microblaze/include/asm/Kbuild
index 2b98bc7..b8263e5 100644
--- a/arch/microblaze/include/asm/Kbuild
+++ b/arch/microblaze/include/asm/Kbuild
@@ -3,6 +3,7 @@
generic-y += clkdev.h
generic-y += exec.h
generic-y += hash.h
+generic-y += irq_work.h
generic-y += trace_clock.h
generic-y += syscalls.h
generic-y += preempt.h
diff --git a/arch/mips/include/asm/Kbuild b/arch/mips/include/asm/Kbuild
index 2d7f650..e15b31a 100644
--- a/arch/mips/include/asm/Kbuild
+++ b/arch/mips/include/asm/Kbuild
@@ -15,3 +15,4 @@
generic-y += ucontext.h
generic-y += xor.h
generic-y += hash.h
+generic-y += irq_work.h
diff --git a/arch/mn10300/include/asm/Kbuild b/arch/mn10300/include/asm/Kbuild
index 992e989..76459a9 100644
--- a/arch/mn10300/include/asm/Kbuild
+++ b/arch/mn10300/include/asm/Kbuild
@@ -5,3 +5,4 @@
generic-y += hash.h
generic-y += trace_clock.h
generic-y += preempt.h
+generic-y += irq_work.h
diff --git a/arch/openrisc/include/asm/Kbuild b/arch/openrisc/include/asm/Kbuild
index 2e40f1c..3bafc67 100644
--- a/arch/openrisc/include/asm/Kbuild
+++ b/arch/openrisc/include/asm/Kbuild
@@ -30,6 +30,7 @@
generic-y += ioctls.h
generic-y += ipcbuf.h
generic-y += irq_regs.h
+generic-y += irq_work.h
generic-y += kdebug.h
generic-y += kmap_types.h
generic-y += kvm_para.h
diff --git a/arch/parisc/include/asm/Kbuild b/arch/parisc/include/asm/Kbuild
index 752c981..aa1a01e 100644
--- a/arch/parisc/include/asm/Kbuild
+++ b/arch/parisc/include/asm/Kbuild
@@ -4,6 +4,7 @@
segment.h topology.h vga.h device.h percpu.h hw_irq.h mutex.h \
div64.h irq_regs.h kdebug.h kvm_para.h local64.h local.h param.h \
poll.h xor.h clkdev.h exec.h
+generic-y += irq_work.h
generic-y += trace_clock.h
generic-y += preempt.h
generic-y += hash.h
diff --git a/arch/powerpc/include/asm/Kbuild b/arch/powerpc/include/asm/Kbuild
index 6c0a955..80444f4 100644
--- a/arch/powerpc/include/asm/Kbuild
+++ b/arch/powerpc/include/asm/Kbuild
@@ -5,3 +5,4 @@
generic-y += preempt.h
generic-y += vtime.h
generic-y += hash.h
+generic-y += irq_work.h
diff --git a/arch/s390/include/asm/Kbuild b/arch/s390/include/asm/Kbuild
index 8386a4a..7cb4ef1 100644
--- a/arch/s390/include/asm/Kbuild
+++ b/arch/s390/include/asm/Kbuild
@@ -4,3 +4,4 @@
generic-y += trace_clock.h
generic-y += preempt.h
generic-y += hash.h
+generic-y += irq_work.h
diff --git a/arch/score/include/asm/Kbuild b/arch/score/include/asm/Kbuild
index 146b9d5..9c630ea 100644
--- a/arch/score/include/asm/Kbuild
+++ b/arch/score/include/asm/Kbuild
@@ -7,4 +7,5 @@
generic-y += trace_clock.h
generic-y += xor.h
generic-y += preempt.h
+generic-y += irq_work.h
diff --git a/arch/sh/include/asm/Kbuild b/arch/sh/include/asm/Kbuild
index 0cd7198..60037fd 100644
--- a/arch/sh/include/asm/Kbuild
+++ b/arch/sh/include/asm/Kbuild
@@ -11,6 +11,7 @@
generic-y += ioctl.h
generic-y += ipcbuf.h
generic-y += irq_regs.h
+generic-y += irq_work.h
generic-y += kvm_para.h
generic-y += local.h
generic-y += local64.h
diff --git a/arch/sparc/include/asm/Kbuild b/arch/sparc/include/asm/Kbuild
index 4b60a0c..abf1a73 100644
--- a/arch/sparc/include/asm/Kbuild
+++ b/arch/sparc/include/asm/Kbuild
@@ -10,6 +10,7 @@
generic-y += local64.h
generic-y += mutex.h
generic-y += irq_regs.h
+generic-y += irq_work.h
generic-y += local.h
generic-y += module.h
generic-y += serial.h
diff --git a/arch/tile/include/asm/Kbuild b/arch/tile/include/asm/Kbuild
index 3793c75..09699d5 100644
--- a/arch/tile/include/asm/Kbuild
+++ b/arch/tile/include/asm/Kbuild
@@ -16,6 +16,7 @@
generic-y += ioctls.h
generic-y += ipcbuf.h
generic-y += irq_regs.h
+generic-y += irq_work.h
generic-y += local.h
generic-y += local64.h
generic-y += msgbuf.h
diff --git a/arch/um/include/asm/Kbuild b/arch/um/include/asm/Kbuild
index 88a330d..bd0b7a0 100644
--- a/arch/um/include/asm/Kbuild
+++ b/arch/um/include/asm/Kbuild
@@ -5,4 +5,5 @@
generic-y += trace_clock.h
generic-y += preempt.h
generic-y += hash.h
+generic-y += irq_work.h
generic-y += barrier.h
diff --git a/arch/unicore32/include/asm/Kbuild b/arch/unicore32/include/asm/Kbuild
index 3ef4f9d..9c0dd43 100644
--- a/arch/unicore32/include/asm/Kbuild
+++ b/arch/unicore32/include/asm/Kbuild
@@ -21,6 +21,7 @@
generic-y += ioctls.h
generic-y += ipcbuf.h
generic-y += irq_regs.h
+generic-y += irq_work.h
generic-y += kdebug.h
generic-y += kmap_types.h
generic-y += local.h
diff --git a/arch/x86/include/asm/Kbuild b/arch/x86/include/asm/Kbuild
index 7f66985..1eb77ac 100644
--- a/arch/x86/include/asm/Kbuild
+++ b/arch/x86/include/asm/Kbuild
@@ -5,3 +5,4 @@
genhdr-y += unistd_x32.h
generic-y += clkdev.h
+generic-y += irq_work.h
diff --git a/arch/xtensa/include/asm/Kbuild b/arch/xtensa/include/asm/Kbuild
index 0a337e4..b749817 100644
--- a/arch/xtensa/include/asm/Kbuild
+++ b/arch/xtensa/include/asm/Kbuild
@@ -11,6 +11,7 @@
generic-y += hardirq.h
generic-y += ioctl.h
generic-y += irq_regs.h
+generic-y += irq_work.h
generic-y += kdebug.h
generic-y += kmap_types.h
generic-y += kvm_para.h
diff --git a/include/asm-generic/irq_work.h b/include/asm-generic/irq_work.h
new file mode 100644
index 0000000..a44f452
--- /dev/null
+++ b/include/asm-generic/irq_work.h
@@ -0,0 +1,10 @@
+#ifndef __ASM_IRQ_WORK_H
+#define __ASM_IRQ_WORK_H
+
+static inline bool arch_irq_work_has_interrupt(void)
+{
+ return false;
+}
+
+#endif /* __ASM_IRQ_WORK_H */
+
diff --git a/include/linux/irq_work.h b/include/linux/irq_work.h
index 60c19ee..4a8c7a2 100644
--- a/include/linux/irq_work.h
+++ b/include/linux/irq_work.h
@@ -33,9 +33,12 @@
void irq_work_queue(struct irq_work *work);
void irq_work_run(void);
+void irq_work_tick(void);
void irq_work_sync(struct irq_work *work);
#ifdef CONFIG_IRQ_WORK
+#include <asm/irq_work.h>
+
bool irq_work_needs_cpu(void);
#else
static inline bool irq_work_needs_cpu(void) { return false; }
diff --git a/kernel/irq_work.c b/kernel/irq_work.c
index 5f7d93d..af8ceaf 100644
--- a/kernel/irq_work.c
+++ b/kernel/irq_work.c
@@ -20,11 +20,8 @@
#include <asm/processor.h>
-static DEFINE_PER_CPU(struct llist_head, irq_work_list);
-#ifdef CONFIG_PREEMPT_RT_FULL
-static DEFINE_PER_CPU(struct llist_head, hirq_work_list);
-#endif
-static DEFINE_PER_CPU(int, irq_work_raised);
+static DEFINE_PER_CPU(struct llist_head, raised_list);
+static DEFINE_PER_CPU(struct llist_head, lazy_list);
/*
* Claim the entry so that no one else will poke at it.
@@ -67,6 +64,9 @@
*/
void irq_work_queue(struct irq_work *work)
{
+ bool lazy_work, realtime = IS_ENABLED(CONFIG_PREEMPT_RT_FULL);
+ struct llist_head *list;
+
/* Only queue if not already pending */
if (!irq_work_claim(work))
return;
@@ -74,19 +74,16 @@
/* Queue the entry and raise the IPI if needed. */
preempt_disable();
-#ifdef CONFIG_PREEMPT_RT_FULL
- if (work->flags & IRQ_WORK_HARD_IRQ)
- llist_add(&work->llnode, &__get_cpu_var(hirq_work_list));
+ /* If the work is "lazy", handle it from next tick if any */
+ lazy_work = work->flags & IRQ_WORK_LAZY;
+
+ if (lazy_work || (realtime && !(work->flags & IRQ_WORK_HARD_IRQ)))
+ list = this_cpu_ptr(&lazy_list);
else
-#endif
- llist_add(&work->llnode, &__get_cpu_var(irq_work_list));
- /*
- * If the work is not "lazy" or the tick is stopped, raise the irq
- * work interrupt (if supported by the arch), otherwise, just wait
- * for the next tick.
- */
- if (!(work->flags & IRQ_WORK_LAZY) || tick_nohz_tick_stopped()) {
- if (!this_cpu_cmpxchg(irq_work_raised, 0, 1))
+ list = this_cpu_ptr(&raised_list);
+
+ if (llist_add(&work->llnode, list)) {
+ if (!lazy_work || tick_nohz_tick_stopped())
arch_irq_work_raise();
}
@@ -96,10 +93,11 @@
bool irq_work_needs_cpu(void)
{
- struct llist_head *this_list;
+ struct llist_head *raised, *lazy;
- this_list = &__get_cpu_var(irq_work_list);
- if (llist_empty(this_list))
+ raised = &__get_cpu_var(raised_list);
+ lazy = &__get_cpu_var(lazy_list);
+ if (llist_empty(raised) && llist_empty(lazy))
return false;
/* All work should have been flushed before going offline */
@@ -108,34 +106,18 @@
return true;
}
-static void __irq_work_run(void)
+static void irq_work_run_list(struct llist_head *list)
{
unsigned long flags;
struct irq_work *work;
- struct llist_head *this_list;
struct llist_node *llnode;
+ BUG_ON_NONRT(!irqs_disabled());
- /*
- * Reset the "raised" state right before we check the list because
- * an NMI may enqueue after we find the list empty from the runner.
- */
- __this_cpu_write(irq_work_raised, 0);
- barrier();
-
-#ifdef CONFIG_PREEMPT_RT_FULL
- if (in_irq())
- this_list = &__get_cpu_var(hirq_work_list);
- else
-#endif
- this_list = &__get_cpu_var(irq_work_list);
- if (llist_empty(this_list))
+ if (llist_empty(list))
return;
-#ifndef CONFIG_PREEMPT_RT_FULL
- BUG_ON(!irqs_disabled());
-#endif
- llnode = llist_del_all(this_list);
+ llnode = llist_del_all(list);
while (llnode != NULL) {
work = llist_entry(llnode, struct irq_work, llnode);
@@ -166,13 +148,24 @@
*/
void irq_work_run(void)
{
-#ifndef CONFIG_PREEMPT_RT_FULL
- BUG_ON(!in_irq());
-#endif
- __irq_work_run();
+ irq_work_run_list(this_cpu_ptr(&raised_list));
+ if (IS_ENABLED(CONFIG_PREEMPT_RT_FULL)) {
+ if (!llist_empty(this_cpu_ptr(&lazy_list)))
+ raise_softirq(TIMER_SOFTIRQ);
+ } else
+ irq_work_run_list(this_cpu_ptr(&lazy_list));
}
EXPORT_SYMBOL_GPL(irq_work_run);
+void irq_work_tick(void)
+{
+ struct llist_head *raised = this_cpu_ptr(&raised_list);
+
+ if (!llist_empty(raised) && !arch_irq_work_has_interrupt())
+ irq_work_run_list(raised);
+ irq_work_run_list(this_cpu_ptr(&lazy_list));
+}
+
/*
* Synchronize against the irq_work @entry, ensures the entry is not
* currently in use.
@@ -197,7 +190,7 @@
/* Called from stop_machine */
if (WARN_ON_ONCE(cpu != smp_processor_id()))
break;
- __irq_work_run();
+ irq_work_tick();
break;
default:
break;
diff --git a/kernel/timer.c b/kernel/timer.c
index 36b9f10..3008703 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -1467,7 +1467,7 @@
hrtimer_run_pending();
#if defined(CONFIG_IRQ_WORK) && defined(CONFIG_PREEMPT_RT_FULL)
- irq_work_run();
+ irq_work_tick();
#endif
if (time_after_eq(jiffies, base->timer_jiffies))
diff --git a/localversion-rt b/localversion-rt
index 8bdfb9a..ac4d836 100644
--- a/localversion-rt
+++ b/localversion-rt
@@ -1 +1 @@
--rt42
+-rt44