aboutsummaryrefslogtreecommitdiff
path: root/kernel/sched.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/sched.c')
-rw-r--r--kernel/sched.c313
1 files changed, 107 insertions, 206 deletions
diff --git a/kernel/sched.c b/kernel/sched.c
index df80bae6815..6acf749d333 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -571,8 +571,10 @@ struct rq {
#endif
#ifdef CONFIG_SCHED_HRTICK
- unsigned long hrtick_flags;
- ktime_t hrtick_expire;
+#ifdef CONFIG_SMP
+ int hrtick_csd_pending;
+ struct call_single_data hrtick_csd;
+#endif
struct hrtimer hrtick_timer;
#endif
@@ -983,13 +985,6 @@ static struct rq *this_rq_lock(void)
return rq;
}
-static void __resched_task(struct task_struct *p, int tif_bit);
-
-static inline void resched_task(struct task_struct *p)
-{
- __resched_task(p, TIF_NEED_RESCHED);
-}
-
#ifdef CONFIG_SCHED_HRTICK
/*
* Use HR-timers to deliver accurate preemption points.
@@ -1001,25 +996,6 @@ static inline void resched_task(struct task_struct *p)
* When we get rescheduled we reprogram the hrtick_timer outside of the
* rq->lock.
*/
-static inline void resched_hrt(struct task_struct *p)
-{
- __resched_task(p, TIF_HRTICK_RESCHED);
-}
-
-static inline void resched_rq(struct rq *rq)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&rq->lock, flags);
- resched_task(rq->curr);
- spin_unlock_irqrestore(&rq->lock, flags);
-}
-
-enum {
- HRTICK_SET, /* re-programm hrtick_timer */
- HRTICK_RESET, /* not a new slice */
- HRTICK_BLOCK, /* stop hrtick operations */
-};
/*
* Use hrtick when:
@@ -1030,40 +1006,11 @@ static inline int hrtick_enabled(struct rq *rq)
{
if (!sched_feat(HRTICK))
return 0;
- if (unlikely(test_bit(HRTICK_BLOCK, &rq->hrtick_flags)))
+ if (!cpu_active(cpu_of(rq)))
return 0;
return hrtimer_is_hres_active(&rq->hrtick_timer);
}
-/*
- * Called to set the hrtick timer state.
- *
- * called with rq->lock held and irqs disabled
- */
-static void hrtick_start(struct rq *rq, u64 delay, int reset)
-{
- assert_spin_locked(&rq->lock);
-
- /*
- * preempt at: now + delay
- */
- rq->hrtick_expire =
- ktime_add_ns(rq->hrtick_timer.base->get_time(), delay);
- /*
- * indicate we need to program the timer
- */
- __set_bit(HRTICK_SET, &rq->hrtick_flags);
- if (reset)
- __set_bit(HRTICK_RESET, &rq->hrtick_flags);
-
- /*
- * New slices are called from the schedule path and don't need a
- * forced reschedule.
- */
- if (reset)
- resched_hrt(rq->curr);
-}
-
static void hrtick_clear(struct rq *rq)
{
if (hrtimer_active(&rq->hrtick_timer))
@@ -1071,32 +1018,6 @@ static void hrtick_clear(struct rq *rq)
}
/*
- * Update the timer from the possible pending state.
- */
-static void hrtick_set(struct rq *rq)
-{
- ktime_t time;
- int set, reset;
- unsigned long flags;
-
- WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
-
- spin_lock_irqsave(&rq->lock, flags);
- set = __test_and_clear_bit(HRTICK_SET, &rq->hrtick_flags);
- reset = __test_and_clear_bit(HRTICK_RESET, &rq->hrtick_flags);
- time = rq->hrtick_expire;
- clear_thread_flag(TIF_HRTICK_RESCHED);
- spin_unlock_irqrestore(&rq->lock, flags);
-
- if (set) {
- hrtimer_start(&rq->hrtick_timer, time, HRTIMER_MODE_ABS);
- if (reset && !hrtimer_active(&rq->hrtick_timer))
- resched_rq(rq);
- } else
- hrtick_clear(rq);
-}
-
-/*
* High-resolution timer tick.
* Runs from hardirq context with interrupts disabled.
*/
@@ -1115,27 +1036,37 @@ static enum hrtimer_restart hrtick(struct hrtimer *timer)
}
#ifdef CONFIG_SMP
-static void hotplug_hrtick_disable(int cpu)
+/*
+ * called from hardirq (IPI) context
+ */
+static void __hrtick_start(void *arg)
{
- struct rq *rq = cpu_rq(cpu);
- unsigned long flags;
-
- spin_lock_irqsave(&rq->lock, flags);
- rq->hrtick_flags = 0;
- __set_bit(HRTICK_BLOCK, &rq->hrtick_flags);
- spin_unlock_irqrestore(&rq->lock, flags);
+ struct rq *rq = arg;
- hrtick_clear(rq);
+ spin_lock(&rq->lock);
+ hrtimer_restart(&rq->hrtick_timer);
+ rq->hrtick_csd_pending = 0;
+ spin_unlock(&rq->lock);
}
-static void hotplug_hrtick_enable(int cpu)
+/*
+ * Called to set the hrtick timer state.
+ *
+ * called with rq->lock held and irqs disabled
+ */
+static void hrtick_start(struct rq *rq, u64 delay)
{
- struct rq *rq = cpu_rq(cpu);
- unsigned long flags;
+ struct hrtimer *timer = &rq->hrtick_timer;
+ ktime_t time = ktime_add_ns(timer->base->get_time(), delay);
- spin_lock_irqsave(&rq->lock, flags);
- __clear_bit(HRTICK_BLOCK, &rq->hrtick_flags);
- spin_unlock_irqrestore(&rq->lock, flags);
+ timer->expires = time;
+
+ if (rq == this_rq()) {
+ hrtimer_restart(timer);
+ } else if (!rq->hrtick_csd_pending) {
+ __smp_call_function_single(cpu_of(rq), &rq->hrtick_csd);
+ rq->hrtick_csd_pending = 1;
+ }
}
static int
@@ -1150,16 +1081,7 @@ hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu)
case CPU_DOWN_PREPARE_FROZEN:
case CPU_DEAD:
case CPU_DEAD_FROZEN:
- hotplug_hrtick_disable(cpu);
- return NOTIFY_OK;
-
- case CPU_UP_PREPARE:
- case CPU_UP_PREPARE_FROZEN:
- case CPU_DOWN_FAILED:
- case CPU_DOWN_FAILED_FROZEN:
- case CPU_ONLINE:
- case CPU_ONLINE_FROZEN:
- hotplug_hrtick_enable(cpu);
+ hrtick_clear(cpu_rq(cpu));
return NOTIFY_OK;
}
@@ -1170,46 +1092,45 @@ static void init_hrtick(void)
{
hotcpu_notifier(hotplug_hrtick, 0);
}
-#endif /* CONFIG_SMP */
+#else
+/*
+ * Called to set the hrtick timer state.
+ *
+ * called with rq->lock held and irqs disabled
+ */
+static void hrtick_start(struct rq *rq, u64 delay)
+{
+ hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), HRTIMER_MODE_REL);
+}
-static void init_rq_hrtick(struct rq *rq)
+static void init_hrtick(void)
{
- rq->hrtick_flags = 0;
- hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
- rq->hrtick_timer.function = hrtick;
- rq->hrtick_timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ;
}
+#endif /* CONFIG_SMP */
-void hrtick_resched(void)
+static void init_rq_hrtick(struct rq *rq)
{
- struct rq *rq;
- unsigned long flags;
+#ifdef CONFIG_SMP
+ rq->hrtick_csd_pending = 0;
- if (!test_thread_flag(TIF_HRTICK_RESCHED))
- return;
+ rq->hrtick_csd.flags = 0;
+ rq->hrtick_csd.func = __hrtick_start;
+ rq->hrtick_csd.info = rq;
+#endif
- local_irq_save(flags);
- rq = cpu_rq(smp_processor_id());
- hrtick_set(rq);
- local_irq_restore(flags);
+ hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ rq->hrtick_timer.function = hrtick;
+ rq->hrtick_timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ;
}
#else
static inline void hrtick_clear(struct rq *rq)
{
}
-static inline void hrtick_set(struct rq *rq)
-{
-}
-
static inline void init_rq_hrtick(struct rq *rq)
{
}
-void hrtick_resched(void)
-{
-}
-
static inline void init_hrtick(void)
{
}
@@ -1228,16 +1149,16 @@ static inline void init_hrtick(void)
#define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG)
#endif
-static void __resched_task(struct task_struct *p, int tif_bit)
+static void resched_task(struct task_struct *p)
{
int cpu;
assert_spin_locked(&task_rq(p)->lock);
- if (unlikely(test_tsk_thread_flag(p, tif_bit)))
+ if (unlikely(test_tsk_thread_flag(p, TIF_NEED_RESCHED)))
return;
- set_tsk_thread_flag(p, tif_bit);
+ set_tsk_thread_flag(p, TIF_NEED_RESCHED);
cpu = task_cpu(p);
if (cpu == smp_processor_id())
@@ -1303,10 +1224,10 @@ void wake_up_idle_cpu(int cpu)
#endif /* CONFIG_NO_HZ */
#else /* !CONFIG_SMP */
-static void __resched_task(struct task_struct *p, int tif_bit)
+static void resched_task(struct task_struct *p)
{
assert_spin_locked(&task_rq(p)->lock);
- set_tsk_thread_flag(p, tif_bit);
+ set_tsk_need_resched(p);
}
#endif /* CONFIG_SMP */
@@ -2881,7 +2802,7 @@ static void sched_migrate_task(struct task_struct *p, int dest_cpu)
rq = task_rq_lock(p, &flags);
if (!cpu_isset(dest_cpu, p->cpus_allowed)
- || unlikely(cpu_is_offline(dest_cpu)))
+ || unlikely(!cpu_active(dest_cpu)))
goto out;
/* force the process onto the specified CPU */
@@ -3849,7 +3770,7 @@ int select_nohz_load_balancer(int stop_tick)
/*
* If we are going offline and still the leader, give up!
*/
- if (cpu_is_offline(cpu) &&
+ if (!cpu_active(cpu) &&
atomic_read(&nohz.load_balancer) == cpu) {
if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
BUG();
@@ -4395,7 +4316,7 @@ asmlinkage void __sched schedule(void)
struct task_struct *prev, *next;
unsigned long *switch_count;
struct rq *rq;
- int cpu, hrtick = sched_feat(HRTICK);
+ int cpu;
need_resched:
preempt_disable();
@@ -4410,7 +4331,7 @@ need_resched_nonpreemptible:
schedule_debug(prev);
- if (hrtick)
+ if (sched_feat(HRTICK))
hrtick_clear(rq);
/*
@@ -4457,9 +4378,6 @@ need_resched_nonpreemptible:
} else
spin_unlock_irq(&rq->lock);
- if (hrtick)
- hrtick_set(rq);
-
if (unlikely(reacquire_kernel_lock(current) < 0))
goto need_resched_nonpreemptible;
@@ -5876,7 +5794,7 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
struct rq *rq_dest, *rq_src;
int ret = 0, on_rq;
- if (unlikely(cpu_is_offline(dest_cpu)))
+ if (unlikely(!cpu_active(dest_cpu)))
return ret;
rq_src = cpu_rq(src_cpu);
@@ -6768,7 +6686,8 @@ static cpumask_t cpu_isolated_map = CPU_MASK_NONE;
/* Setup the mask of cpus configured for isolated domains */
static int __init isolated_cpu_setup(char *str)
{
- int ints[NR_CPUS], i;
+ static int __initdata ints[NR_CPUS];
+ int i;
str = get_options(str, ARRAY_SIZE(ints), ints);
cpus_clear(cpu_isolated_map);
@@ -7553,18 +7472,6 @@ void __attribute__((weak)) arch_update_cpu_topology(void)
}
/*
- * Free current domain masks.
- * Called after all cpus are attached to NULL domain.
- */
-static void free_sched_domains(void)
-{
- ndoms_cur = 0;
- if (doms_cur != &fallback_doms)
- kfree(doms_cur);
- doms_cur = &fallback_doms;
-}
-
-/*
* Set up scheduler domains and groups. Callers must hold the hotplug lock.
* For now this just excludes isolated cpus, but could be used to
* exclude other special cases in the future.
@@ -7642,7 +7549,7 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
* ownership of it and will kfree it when done with it. If the caller
* failed the kmalloc call, then it can pass in doms_new == NULL,
* and partition_sched_domains() will fallback to the single partition
- * 'fallback_doms'.
+ * 'fallback_doms', it also forces the domains to be rebuilt.
*
* Call with hotplug lock held
*/
@@ -7656,12 +7563,8 @@ void partition_sched_domains(int ndoms_new, cpumask_t *doms_new,
/* always unregister in case we don't destroy any domains */
unregister_sched_domain_sysctl();
- if (doms_new == NULL) {
- ndoms_new = 1;
- doms_new = &fallback_doms;
- cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map);
- dattr_new = NULL;
- }
+ if (doms_new == NULL)
+ ndoms_new = 0;
/* Destroy deleted domains */
for (i = 0; i < ndoms_cur; i++) {
@@ -7676,6 +7579,14 @@ match1:
;
}
+ if (doms_new == NULL) {
+ ndoms_cur = 0;
+ ndoms_new = 1;
+ doms_new = &fallback_doms;
+ cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map);
+ dattr_new = NULL;
+ }
+
/* Build new domains */
for (i = 0; i < ndoms_new; i++) {
for (j = 0; j < ndoms_cur; j++) {
@@ -7706,17 +7617,10 @@ match2:
#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
int arch_reinit_sched_domains(void)
{
- int err;
-
get_online_cpus();
- mutex_lock(&sched_domains_mutex);
- detach_destroy_domains(&cpu_online_map);
- free_sched_domains();
- err = arch_init_sched_domains(&cpu_online_map);
- mutex_unlock(&sched_domains_mutex);
+ rebuild_sched_domains();
put_online_cpus();
-
- return err;
+ return 0;
}
static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
@@ -7786,59 +7690,49 @@ int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
}
#endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
+#ifndef CONFIG_CPUSETS
/*
- * Force a reinitialization of the sched domains hierarchy. The domains
- * and groups cannot be updated in place without racing with the balancing
- * code, so we temporarily attach all running cpus to the NULL domain
- * which will prevent rebalancing while the sched domains are recalculated.
+ * Add online and remove offline CPUs from the scheduler domains.
+ * When cpusets are enabled they take over this function.
*/
static int update_sched_domains(struct notifier_block *nfb,
unsigned long action, void *hcpu)
{
+ switch (action) {
+ case CPU_ONLINE:
+ case CPU_ONLINE_FROZEN:
+ case CPU_DEAD:
+ case CPU_DEAD_FROZEN:
+ partition_sched_domains(0, NULL, NULL);
+ return NOTIFY_OK;
+
+ default:
+ return NOTIFY_DONE;
+ }
+}
+#endif
+
+static int update_runtime(struct notifier_block *nfb,
+ unsigned long action, void *hcpu)
+{
int cpu = (int)(long)hcpu;
switch (action) {
case CPU_DOWN_PREPARE:
case CPU_DOWN_PREPARE_FROZEN:
disable_runtime(cpu_rq(cpu));
- /* fall-through */
- case CPU_UP_PREPARE:
- case CPU_UP_PREPARE_FROZEN:
- detach_destroy_domains(&cpu_online_map);
- free_sched_domains();
return NOTIFY_OK;
-
case CPU_DOWN_FAILED:
case CPU_DOWN_FAILED_FROZEN:
case CPU_ONLINE:
case CPU_ONLINE_FROZEN:
enable_runtime(cpu_rq(cpu));
- /* fall-through */
- case CPU_UP_CANCELED:
- case CPU_UP_CANCELED_FROZEN:
- case CPU_DEAD:
- case CPU_DEAD_FROZEN:
- /*
- * Fall through and re-initialise the domains.
- */
- break;
+ return NOTIFY_OK;
+
default:
return NOTIFY_DONE;
}
-
-#ifndef CONFIG_CPUSETS
- /*
- * Create default domain partitioning if cpusets are disabled.
- * Otherwise we let cpusets rebuild the domains based on the
- * current setup.
- */
-
- /* The hotplug lock is already held by cpu_up/cpu_down */
- arch_init_sched_domains(&cpu_online_map);
-#endif
-
- return NOTIFY_OK;
}
void __init sched_init_smp(void)
@@ -7858,8 +7752,15 @@ void __init sched_init_smp(void)
cpu_set(smp_processor_id(), non_isolated_cpus);
mutex_unlock(&sched_domains_mutex);
put_online_cpus();
+
+#ifndef CONFIG_CPUSETS
/* XXX: Theoretical race here - CPU may be hotplugged now */
hotcpu_notifier(update_sched_domains, 0);
+#endif
+
+ /* RT runtime code needs to handle some hotplug events */
+ hotcpu_notifier(update_runtime, 0);
+
init_hrtick();
/* Move init over to a non-isolated CPU */