HACK: change irqstat to be per irq per cpu array.cpuidle/irq-governor

5 files changed, 97 insertions, 145 deletions

diff --git a/include/linux/irqdesc.h b/include/linux/irqdesc.h
index 5bc2cd7fe19f..ac22297516a9 100644
--- a/include/linux/irqdesc.h
+++ b/include/linux/irqdesc.h
@@ -48,9 +48,6 @@ struct irq_desc {
 	struct irq_common_data	irq_common_data;
 	struct irq_data		irq_data;
 	unsigned int __percpu	*kstat_irqs;
-#ifdef CONFIG_IRQ_TIMINGS
-	struct irqt_stat *irq_timings;
-#endif
 	irq_flow_handler_t	handle_irq;
 #ifdef CONFIG_IRQ_PREFLOW_FASTEOI
 	irq_preflow_handler_t	preflow_handler;
diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h
index e29d44f922e5..2fcd3cc3a052 100644
--- a/kernel/irq/internals.h
+++ b/kernel/irq/internals.h
@@ -105,18 +105,17 @@ static inline void unregister_handler_proc(unsigned int irq,
 
 #ifdef CONFIG_IRQ_TIMINGS
 extern void __init irqt_init(void);
-extern void irqt_process(unsigned int irq, struct irqt_stat *s);
+extern void irqt_process(unsigned int irq);
 static inline void irqt_event(int irq, struct irq_desc *desc)
 {
-	if (desc->irq_timings)
-		irqt_process(irq, desc->irq_timings);
+	irqt_process(irq);
 }
-extern int irqt_register(struct irq_desc *desc);
+extern int irqt_register(int irq, struct irq_desc *desc);
 extern void irqt_unregister(struct irq_desc *desc);
 #else
 static inline void irqt_init(void) { }
 static inline void irqt_event(int irq, struct irq_desc *desc) { }
-static inline int irqt_register(struct irq_desc *desc) { return 0; }
+static inline int irqt_register(int irq, struct irq_desc *desc) { return 0; }
 static inline void irqt_unregister(struct irq_desc *desc) { }
 #endif
 
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index ef4709c7d6c4..548de1683e43 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -1300,7 +1300,7 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
 	free_cpumask_var(mask);
 
 	if (new->flags & IRQF_TIMINGS)
-		irqt_register(desc);
+		irqt_register(irq, desc);
 
 	return 0;
 
diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c
index 534ae42e7d4c..5956b583f89f 100644
--- a/kernel/irq/proc.c
+++ b/kernel/irq/proc.c
@@ -287,7 +287,7 @@ static int irq_timings_proc_show(struct seq_file *m, void *v)
 {
 	struct irq_desc *desc = irq_to_desc((long) m->private);
 
-	seq_printf(m, "%d\n", desc->irq_timings ? 1 : 0);
+	seq_printf(m, "%d\n", 0);
 
 	return 0;
 }
@@ -319,7 +319,7 @@ static ssize_t irq_timings_proc_write(struct file *file,
 		goto out;
 
  	if (enable) {
- 		ret = irqt_register(desc);
+ 		ret = irqt_register(irq, desc);
  	} else {
  		unsigned long flags;
  		raw_spin_lock_irqsave(&desc->lock, flags);
diff --git a/kernel/irq/timings.c b/kernel/irq/timings.c
index d8f11ceae18c..1c875e0d475b 100644
--- a/kernel/irq/timings.c
+++ b/kernel/irq/timings.c
@@ -20,53 +20,22 @@
 
 #include <trace/events/irq.h>
 
-
-/*
- * This is the size of the IRQ interval window used to compute the
- * mean interval and its variance.  This has to be at least 3 to still
- * make sense.  Higher values may improve prediction confidence but more
- * false negatives are to be expected.
- */
-#define IRQT_INTERVAL_WINDOW	3
-
-
-struct irqt_prediction {
-	struct list_head node;
-	ktime_t		 time;		/* expected occurrence time */
-	int		 cpu;		/* CPU for which this was queued for */
-};
-
-struct irqt_stat {
-	ktime_t		last_time;	/* previous IRQ occurrence */
-	u64		n_M2;		/* IRQ interval variance (n scaled) */
-	u32		n_mean;		/* IRQ mean interval (n scaled) */
-	u32     	intervals[IRQT_INTERVAL_WINDOW];
-					/* window of recent IRQ intervals */
-	unsigned int	w_ptr;		/* current window pointer */
-	u32		predictable;	/* # of IRQs that were predictable */
-	u32		unpredictable;	/* # of IRQs that were not */
-	struct irqt_prediction prediction;
-};
-
-static DEFINE_PER_CPU(struct list_head, irqt_predictions);
-static DEFINE_PER_CPU(raw_spinlock_t, irqt_predictions_lock);
-
 #ifdef CONFIG_DEBUG_FS
 
-struct irqt_stats {
+struct irqt_stats_debug {
 	atomic_t correct;
 	atomic_t early;
 	atomic_t late;
 	atomic_t total;
 };
 
-static DEFINE_PER_CPU(struct irqt_stats, irqt_stats[NR_IRQS]);
+static DEFINE_PER_CPU(struct irqt_stats_debug, irqt_stats_debug[NR_IRQS]);
 static unsigned irqt_registered[NR_IRQS];
 
 static int irqt_debugfs_stats_show(struct seq_file *m, void *v)
 {
 	int i, cpu;
-	struct irqt_stats *s;
+	struct irqt_stats_debug *s;
 
 	seq_printf(m, "# cpu\t irq\tcorrect\tearly\tlate\ttotal\n");
 	
@@ -77,7 +46,7 @@ static int irqt_debugfs_stats_show(struct seq_file *m, void *v)
 			if (!irqt_registered[i])
 				continue;
 
-			s = &per_cpu(irqt_stats[i], cpu);
+			s = &per_cpu(irqt_stats_debug[i], cpu);
 
 			if (!atomic_read(&s->total))
 				continue;
@@ -107,21 +76,21 @@ static const struct file_operations stats_fops = {
 
 static void irqt_debugfs_correct_inc(int cpu, int irq)
 {
-	struct irqt_stats *s = &per_cpu(irqt_stats[irq], cpu);
+	struct irqt_stats_debug *s = &per_cpu(irqt_stats_debug[irq], cpu);
 	atomic_inc(&s->correct);
 	atomic_inc(&s->total);
 }
 
 static void irqt_debugfs_early_inc(int cpu, int irq)
 {
-	struct irqt_stats *s = &per_cpu(irqt_stats[irq], cpu);
+	struct irqt_stats_debug *s = &per_cpu(irqt_stats_debug[irq], cpu);
 	atomic_inc(&s->early);
 	atomic_inc(&s->total);
 }
 
 static void irqt_debugfs_late_inc(int cpu, int irq)
 {
-	struct irqt_stats *s = &per_cpu(irqt_stats[irq], cpu);
+	struct irqt_stats_debug *s = &per_cpu(irqt_stats_debug[irq], cpu);
 	atomic_inc(&s->late);
 	atomic_inc(&s->total);
 }
@@ -150,14 +119,36 @@ late_initcall(irqt_debugfs_init);
 
 #endif
 
+/*
+ * This is the size of the IRQ interval window used to compute the
+ * mean interval and its variance.  This has to be at least 3 to still
+ * make sense.  Higher values may improve prediction confidence but more
+ * false negatives are to be expected.
+ */
+#define IRQT_INTERVAL_WINDOW	3
+
+struct irqt_prediction {
+	ktime_t		 time;		/* expected occurrence time */
+	int		 cpu;		/* CPU for which this was queued for */
+};
+
+struct irqt_stat {
+	struct list_head node;
+	ktime_t		last_time;	/* previous IRQ occurrence */
+	u64		n_M2;		/* IRQ interval variance (n scaled) */
+	u32		n_mean;		/* IRQ mean interval (n scaled) */
+	u32     	intervals[IRQT_INTERVAL_WINDOW];
+					/* window of recent IRQ intervals */
+	unsigned int	w_ptr;		/* current window pointer */
+	u32		predictable;	/* # of IRQs that were predictable */
+	u32		unpredictable;	/* # of IRQs that were not */
+	struct irqt_prediction prediction;
+};
+
+static DEFINE_PER_CPU(struct irqt_stat, *irqt_stats[NR_IRQS]);
+
 void __init irqt_init(void)
 {
-	int cpu;
-
-	for_each_possible_cpu(cpu) {
-		INIT_LIST_HEAD(&per_cpu(irqt_predictions, cpu));
-		raw_spin_lock_init(&per_cpu(irqt_predictions_lock, cpu));
-	}
 }
 
 /*
@@ -166,13 +157,6 @@ void __init irqt_init(void)
  */
 static void irqt_purge(ktime_t now, struct list_head *head)
 {
-	struct irqt_prediction *entry, *n;
-
-	list_for_each_entry_safe(entry, n, head, node) {
-		if (ktime_after(entry->time, now))
-			break;
-		list_del_init(&entry->node);
-	}
 }
 
 /*
@@ -183,30 +167,12 @@ static void irqt_enqueue_prediction(ktime_t now, struct irqt_stat *s)
 {
 	int this_cpu = raw_smp_processor_id();
 	int prev_cpu = s->prediction.cpu;
-	struct list_head *head = &per_cpu(irqt_predictions, this_cpu);
 	u32 predicted_interval = s->n_mean / IRQT_INTERVAL_WINDOW;
-	struct irqt_prediction *list_entry, *new_entry;
-	raw_spinlock_t *lock;
-
-	if (unlikely(prev_cpu != this_cpu && prev_cpu != -1)) {
-		lock = &per_cpu(irqt_predictions_lock, prev_cpu);
-		raw_spin_lock(lock);
-		list_del_init(&s->prediction.node);
-		raw_spin_unlock(lock);
-	}
+
+	if (unlikely(prev_cpu != this_cpu && prev_cpu != -1))
+		WARN_ONCE(1, "CPUs are not the same");
 		
-	lock = &per_cpu(irqt_predictions_lock, this_cpu);
-	raw_spin_lock(lock);
-	irqt_purge(now, head);
-	__list_del_entry(&s->prediction.node);
-	new_entry = &s->prediction;
-	new_entry->time = ktime_add_us(now, predicted_interval);
-	new_entry->cpu = this_cpu;
-	list_for_each_entry(list_entry, head, node)
-		if (ktime_after(new_entry->time, list_entry->time))
-			break;
-	list_add_tail(&new_entry->node, &list_entry->node);
-	raw_spin_unlock(lock);
+	s->prediction.time = ktime_add_us(now, predicted_interval);
 }
 
 /**
@@ -221,20 +187,26 @@ static void irqt_enqueue_prediction(ktime_t now, struct irqt_stat *s)
  */
 s64 irqt_get_next_prediction(int cpu)
 {
-	raw_spinlock_t *lock = &per_cpu(irqt_predictions_lock, cpu);
-	struct list_head *head = &per_cpu(irqt_predictions, cpu);
-	unsigned long flags;
-	ktime_t now;
-	struct irqt_prediction *next;
-	s64 result;
-
-	raw_spin_lock_irqsave(lock, flags);
-	now = ktime_get();
-	irqt_purge(now, head);
-	next = list_first_entry_or_null(head, struct irqt_prediction, node);
-	result = next ? ktime_us_delta(next->time, now) : 0;
-	raw_spin_unlock_irqrestore(lock, flags);
-	return result;
+	struct irqt_stat *s;
+	int i;
+	ktime_t now = ktime_get();
+	s64 value, min = S64_MAX;
+
+	for (i = 0; i < NR_IRQS; i++) {
+
+		s = per_cpu(irqt_stats[i], cpu);
+		if (!s)
+			continue;
+
+		value = ktime_us_delta(s->prediction.time, now);
+		if (value <= 0)
+			continue;
+
+		if (value < min)
+			min = value;
+	}
+	
+	return min != S64_MAX ? min : 0;
 }
 
 /*
@@ -246,13 +218,18 @@ s64 irqt_get_next_prediction(int cpu)
  * This is assumed to be called in IRQ context with desc->lock held and
  * IRQs turned off.
  */
-void irqt_process(unsigned int irq, struct irqt_stat *s)
+void irqt_process(unsigned int irq)
 {
+	struct irqt_stat *s = per_cpu(irqt_stats[irq], raw_smp_processor_id());
 	ktime_t now = ktime_get();
-	ktime_t ktime_interval = ktime_sub(now, s->last_time);
+	ktime_t ktime_interval;
 	u32 oldX, newX, n = IRQT_INTERVAL_WINDOW;
 	s32 delta, n_dold, n_dnew;
 
+	if (!s)
+		return;
+
+	ktime_interval = ktime_sub(now, s->last_time);
 	s->last_time = now;
 
 	/* An interval needs at least two events */
@@ -376,42 +353,38 @@ void irqt_process(unsigned int irq, struct irqt_stat *s)
  * Called from __setup_irq() after successful registration of a new action
  * handler.
  */
-int irqt_register(struct irq_desc *desc)
+int irqt_register(int irq, struct irq_desc *desc)
 {
 	struct irqt_stat *s;
 	unsigned long flags;
-	int ret;
-
-	if (desc->irq_timings)
-		return 0;
-
-	s = kzalloc(sizeof(*s), GFP_KERNEL);
-	if (!s)
-		return -ENOMEM;
-	INIT_LIST_HEAD(&s->prediction.node);
-	s->prediction.cpu = -1;
+	int cpu, ret = 0;
 
 	raw_spin_lock_irqsave(&desc->lock, flags);
-	if (desc->irq_timings) {
-		/* someone else raced ahead of us */
-		ret = 0;
-	} else if (!desc->action) {
+	if (!desc->action) {
 		/* unused IRQ? */
 		ret = -ENXIO;
-	} else if (irq_settings_is_per_cpu(desc)) {
-		/* we're not set for per-CPU accounting */
-		pr_warn("IRQ %d: can't do timing stats on per-CPU IRQs\n",
-			desc->action->irq);
-		ret = -ENOSYS;
-	} else {
-		desc->irq_timings = s;
-		irqt_registered[desc->action->irq] = 1;
-		s = NULL;
-		ret = 0;
+	} /* else if (irq_settings_is_per_cpu(desc)) { */
+	/* 	/\* we're not set for per-CPU accounting *\/ */
+	/* 	pr_warn("IRQ %d: can't do timing stats on per-CPU IRQs\n", */
+	/* 		desc->action->irq); */
+	/* 	ret = -ENOSYS; */
+	/* } */ else {
+		for_each_possible_cpu(cpu) {
+			
+			if (!per_cpu(irqt_stats[irq], cpu)) {
+				s = kzalloc(sizeof(*s), GFP_KERNEL);
+				if (!s)
+					return -ENOMEM;
+				s->prediction.cpu = cpu;
+				per_cpu(irqt_stats[irq], cpu) = s;
+			}
+		}
+
+		irqt_registered[irq] = 1;
 	}
+
 	raw_spin_unlock_irqrestore(&desc->lock, flags);
-	if (s)
-		kfree(s);
+
 	return ret;
 }
 
@@ -421,22 +394,5 @@ int irqt_register(struct irq_desc *desc)
  */
 void irqt_unregister(struct irq_desc *desc)
 {
-	struct irqt_stat *s;
-	int cpu;
-	raw_spinlock_t *lock;
-
-	assert_raw_spin_locked(&desc->lock);
-	if (!desc->irq_timings)
-		return;
-	s = desc->irq_timings;
-	desc->irq_timings = NULL;
-	irqt_registered[desc->action->irq] = 0;
-	cpu = s->prediction.cpu;
-	if (cpu != -1) {
-		lock = &per_cpu(irqt_predictions_lock, cpu);
-		raw_spin_lock(lock);
-		__list_del_entry(&s->prediction.node);
-		raw_spin_unlock(lock);
-	}
-	kfree(s);
+	;
 }