From 7997332019e336d9b05b846aa0d04910dcd2e860 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Mon, 13 May 2013 16:31:04 +0530 Subject: cpufreq: Add EXPORT_SYMBOL_GPL for have_governor_per_policy This patch adds: EXPORT_SYMBOL_GPL(have_governor_per_policy), so that this routine can be used by modules too. Signed-off-by: Viresh Kumar --- drivers/cpufreq/cpufreq.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 2d53f47d1747..2f254691ed1f 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -132,6 +132,7 @@ bool have_governor_per_policy(void) { return cpufreq_driver->have_governor_per_policy; } +EXPORT_SYMBOL_GPL(have_governor_per_policy); static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs) { -- cgit v1.2.3 From 69bbfa930a4b93beae124e10566d37d1ab3d07af Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Mon, 13 May 2013 16:26:48 +0530 Subject: cpufreq: governors: Move get_governor_parent_kobj() to cpufreq.c get_governor_parent_kobj() can be used by any governor, generic cpufreq governors or platform specific ones and so must be present in cpufreq.c instead of cpufreq_governor.c. This patch moves it to cpufreq.c. This also adds EXPORT_SYMBOL_GPL(get_governor_parent_kobj) so that modules can use this function too. Signed-off-by: Viresh Kumar --- drivers/cpufreq/cpufreq.c | 9 +++++++++ drivers/cpufreq/cpufreq_governor.c | 8 -------- include/linux/cpufreq.h | 1 + 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 2f254691ed1f..3faf62bdbad0 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -134,6 +134,15 @@ bool have_governor_per_policy(void) } EXPORT_SYMBOL_GPL(have_governor_per_policy); +struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy) +{ + if (have_governor_per_policy()) + return &policy->kobj; + else + return cpufreq_global_kobject; +} +EXPORT_SYMBOL_GPL(get_governor_parent_kobj); + static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs) { struct cpufreq_policy *data; diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c index dc9b72e25c1a..bf03b3001741 100644 --- a/drivers/cpufreq/cpufreq_governor.c +++ b/drivers/cpufreq/cpufreq_governor.c @@ -30,14 +30,6 @@ #include "cpufreq_governor.h" -static struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy) -{ - if (have_governor_per_policy()) - return &policy->kobj; - else - return cpufreq_global_kobject; -} - static struct attribute_group *get_sysfs_attr(struct dbs_data *dbs_data) { if (have_governor_per_policy()) diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index a6dc4013b0c7..400509e3a2c0 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -340,6 +340,7 @@ const char *cpufreq_get_current_driver(void); int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu); int cpufreq_update_policy(unsigned int cpu); bool have_governor_per_policy(void); +struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy); #ifdef CONFIG_CPU_FREQ /* query the current CPU frequency (in kHz). If zero, cpufreq couldn't detect it */ -- cgit v1.2.3 From a5d035da3acfefcebc2608cdc40c5f1e89784ede Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 16 May 2013 14:17:11 +0530 Subject: cpufreq: Move get_cpu_idle_time() to cpufreq.c Governors other than ondemand and conservative can also use get_cpu_idle_time() and they aren't required to compile cpufreq_governor.c. So, move these independent routines to cpufreq.c instead. Signed-off-by: Viresh Kumar --- drivers/cpufreq/cpufreq.c | 38 ++++++++++++++++++++++++++++++++++++++ drivers/cpufreq/cpufreq_governor.c | 36 ------------------------------------ drivers/cpufreq/cpufreq_governor.h | 1 - include/linux/cpufreq.h | 1 + 4 files changed, 39 insertions(+), 37 deletions(-) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 3faf62bdbad0..c6ab21880c07 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -17,7 +17,9 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include +#include #include #include #include @@ -25,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -143,6 +146,41 @@ struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy) } EXPORT_SYMBOL_GPL(get_governor_parent_kobj); +static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall) +{ + u64 idle_time; + u64 cur_wall_time; + u64 busy_time; + + cur_wall_time = jiffies64_to_cputime64(get_jiffies_64()); + + busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER]; + busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM]; + busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ]; + busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ]; + busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL]; + busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE]; + + idle_time = cur_wall_time - busy_time; + if (wall) + *wall = cputime_to_usecs(cur_wall_time); + + return cputime_to_usecs(idle_time); +} + +u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy) +{ + u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL); + + if (idle_time == -1ULL) + return get_cpu_idle_time_jiffy(cpu, wall); + else if (!io_busy) + idle_time += get_cpu_iowait_time_us(cpu, wall); + + return idle_time; +} +EXPORT_SYMBOL_GPL(get_cpu_idle_time); + static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs) { struct cpufreq_policy *data; diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c index bf03b3001741..fafeb3514669 100644 --- a/drivers/cpufreq/cpufreq_governor.c +++ b/drivers/cpufreq/cpufreq_governor.c @@ -23,7 +23,6 @@ #include #include #include -#include #include #include #include @@ -38,41 +37,6 @@ static struct attribute_group *get_sysfs_attr(struct dbs_data *dbs_data) return dbs_data->cdata->attr_group_gov_sys; } -static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall) -{ - u64 idle_time; - u64 cur_wall_time; - u64 busy_time; - - cur_wall_time = jiffies64_to_cputime64(get_jiffies_64()); - - busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE]; - - idle_time = cur_wall_time - busy_time; - if (wall) - *wall = cputime_to_usecs(cur_wall_time); - - return cputime_to_usecs(idle_time); -} - -u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy) -{ - u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL); - - if (idle_time == -1ULL) - return get_cpu_idle_time_jiffy(cpu, wall); - else if (!io_busy) - idle_time += get_cpu_iowait_time_us(cpu, wall); - - return idle_time; -} -EXPORT_SYMBOL_GPL(get_cpu_idle_time); - void dbs_check_cpu(struct dbs_data *dbs_data, int cpu) { struct cpu_dbs_common_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu); diff --git a/drivers/cpufreq/cpufreq_governor.h b/drivers/cpufreq/cpufreq_governor.h index e16a96130cb3..e7bbf767380d 100644 --- a/drivers/cpufreq/cpufreq_governor.h +++ b/drivers/cpufreq/cpufreq_governor.h @@ -256,7 +256,6 @@ static ssize_t show_sampling_rate_min_gov_pol \ return sprintf(buf, "%u\n", dbs_data->min_sampling_rate); \ } -u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy); void dbs_check_cpu(struct dbs_data *dbs_data, int cpu); bool need_load_eval(struct cpu_dbs_common_info *cdbs, unsigned int sampling_rate); diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 400509e3a2c0..79c2acf24610 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -337,6 +337,7 @@ const char *cpufreq_get_current_driver(void); /********************************************************************* * CPUFREQ 2.6. INTERFACE * *********************************************************************/ +u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy); int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu); int cpufreq_update_policy(unsigned int cpu); bool have_governor_per_policy(void); -- cgit v1.2.3 From 1551d3a9fc376b8b0898d10122033b87f0293110 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Thu, 16 May 2013 14:22:49 +0530 Subject: cpufreq: interactive: Use generic get_cpu_idle_time() from cpufreq.c Generic version of get_cpu_idle_time() is present in cpufreq.c file. We must not replicate that for Interactive governor, rather use the same one. Lets do it. Signed-off-by: Viresh Kumar --- drivers/cpufreq/cpufreq_interactive.c | 42 ++--------------------------------- 1 file changed, 2 insertions(+), 40 deletions(-) diff --git a/drivers/cpufreq/cpufreq_interactive.c b/drivers/cpufreq/cpufreq_interactive.c index 90958fdd64dc..95b7322bb509 100644 --- a/drivers/cpufreq/cpufreq_interactive.c +++ b/drivers/cpufreq/cpufreq_interactive.c @@ -30,8 +30,6 @@ #include #include #include -#include -#include #define CREATE_TRACE_POINTS #include @@ -130,42 +128,6 @@ struct cpufreq_governor cpufreq_gov_interactive = { .owner = THIS_MODULE, }; -static inline cputime64_t get_cpu_idle_time_jiffy(unsigned int cpu, - cputime64_t *wall) -{ - u64 idle_time; - u64 cur_wall_time; - u64 busy_time; - - cur_wall_time = jiffies64_to_cputime64(get_jiffies_64()); - - busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE]; - - idle_time = cur_wall_time - busy_time; - if (wall) - *wall = jiffies_to_usecs(cur_wall_time); - - return jiffies_to_usecs(idle_time); -} - -static inline cputime64_t get_cpu_idle_time(unsigned int cpu, - cputime64_t *wall) -{ - u64 idle_time = get_cpu_idle_time_us(cpu, wall); - - if (idle_time == -1ULL) - idle_time = get_cpu_idle_time_jiffy(cpu, wall); - else if (!io_is_busy) - idle_time += get_cpu_iowait_time_us(cpu, wall); - - return idle_time; -} - static void cpufreq_interactive_timer_resched( struct cpufreq_interactive_cpuinfo *pcpu) { @@ -175,7 +137,7 @@ static void cpufreq_interactive_timer_resched( spin_lock_irqsave(&pcpu->load_lock, flags); pcpu->time_in_idle = get_cpu_idle_time(smp_processor_id(), - &pcpu->time_in_idle_timestamp); + &pcpu->time_in_idle_timestamp, io_is_busy); pcpu->cputime_speedadj = 0; pcpu->cputime_speedadj_timestamp = pcpu->time_in_idle_timestamp; expires = jiffies + usecs_to_jiffies(timer_rate); @@ -348,7 +310,7 @@ static u64 update_load(int cpu) unsigned int delta_time; u64 active_time; - now_idle = get_cpu_idle_time(cpu, &now); + now_idle = get_cpu_idle_time(cpu, &now, io_is_busy); delta_idle = (unsigned int)(now_idle - pcpu->time_in_idle); delta_time = (unsigned int)(now - pcpu->time_in_idle_timestamp); -- cgit v1.2.3 From cc8a8d05fdd028e3b4da01680bc8c583a0aa1ed0 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Wed, 1 May 2013 16:31:01 +0530 Subject: cpufreq: interactive: Remove unnecessary cpu_online() check Cpufreq no longer calls governor callback for offlined cpus. i.e. All policy->cpus are guaranteed to be online. Hence we don't need explicit check to see if cpu is online or not. Signed-off-by: Viresh Kumar --- drivers/cpufreq/cpufreq_interactive.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/cpufreq/cpufreq_interactive.c b/drivers/cpufreq/cpufreq_interactive.c index 95b7322bb509..321da1f6f2d2 100644 --- a/drivers/cpufreq/cpufreq_interactive.c +++ b/drivers/cpufreq/cpufreq_interactive.c @@ -1035,9 +1035,6 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy, switch (event) { case CPUFREQ_GOV_START: - if (!cpu_online(policy->cpu)) - return -EINVAL; - mutex_lock(&gov_lock); freq_table = -- cgit v1.2.3 From 96bcd720a329c2c3145415d490e5e8611a59657c Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Tue, 14 May 2013 09:55:12 +0530 Subject: cpufreq: interactive: Move definition of cpufreq_gov_interactive downwards This moves definition of cpufreq_gov_interactive towards the bottom of file, so that we don't have to add prototype of cpufreq_governor_interactive() in the beginning of file. Signed-off-by: Viresh Kumar --- drivers/cpufreq/cpufreq_interactive.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/drivers/cpufreq/cpufreq_interactive.c b/drivers/cpufreq/cpufreq_interactive.c index 321da1f6f2d2..751fc1debb19 100644 --- a/drivers/cpufreq/cpufreq_interactive.c +++ b/drivers/cpufreq/cpufreq_interactive.c @@ -115,19 +115,6 @@ static int timer_slack_val = DEFAULT_TIMER_SLACK; static bool io_is_busy; -static int cpufreq_governor_interactive(struct cpufreq_policy *policy, - unsigned int event); - -#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE -static -#endif -struct cpufreq_governor cpufreq_gov_interactive = { - .name = "interactive", - .governor = cpufreq_governor_interactive, - .max_transition_latency = 10000000, - .owner = THIS_MODULE, -}; - static void cpufreq_interactive_timer_resched( struct cpufreq_interactive_cpuinfo *pcpu) { @@ -1144,6 +1131,16 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy, return 0; } +#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE +static +#endif +struct cpufreq_governor cpufreq_gov_interactive = { + .name = "interactive", + .governor = cpufreq_governor_interactive, + .max_transition_latency = 10000000, + .owner = THIS_MODULE, +}; + static void cpufreq_interactive_nop_timer(unsigned long data) { } -- cgit v1.2.3 From cbb39a8dbf53df05b7d9e98e6c0f6f375ed811b5 Mon Sep 17 00:00:00 2001 From: Viresh Kumar Date: Mon, 13 May 2013 16:32:58 +0530 Subject: cpufreq: Interactive: Implement per policy instances of governor If we have a multi-package system, where we have multiple instances of struct policy (per package), currently we can't have multiple instances of same governor. i.e. We can't have multiple instances of Interactive governor for multiple packages. This is a bottleneck for multicluster system, where we want different packages to use Interactive governor, but with different tunables. This patch uses the infrastructure provided by earlier patches pushed in Mainline in v3.10-rc1/rc2 and implements per policy instances of Interactive governor. Signed-off-by: Viresh Kumar --- drivers/cpufreq/cpufreq_interactive.c | 624 ++++++++++++++++++++-------------- 1 file changed, 376 insertions(+), 248 deletions(-) diff --git a/drivers/cpufreq/cpufreq_interactive.c b/drivers/cpufreq/cpufreq_interactive.c index 751fc1debb19..7f1782ade53c 100644 --- a/drivers/cpufreq/cpufreq_interactive.c +++ b/drivers/cpufreq/cpufreq_interactive.c @@ -30,12 +30,11 @@ #include #include #include +#include "cpufreq_governor.h" #define CREATE_TRACE_POINTS #include -static int active_count; - struct cpufreq_interactive_cpuinfo { struct timer_list cpu_timer; struct timer_list cpu_slack_timer; @@ -62,76 +61,92 @@ static cpumask_t speedchange_cpumask; static spinlock_t speedchange_cpumask_lock; static struct mutex gov_lock; -/* Hi speed to bump to from lo speed when load burst (default max) */ -static unsigned int hispeed_freq; - -/* Go to hi speed when CPU load at or above this value. */ -#define DEFAULT_GO_HISPEED_LOAD 99 -static unsigned long go_hispeed_load = DEFAULT_GO_HISPEED_LOAD; - -/* Target load. Lower values result in higher CPU speeds. */ #define DEFAULT_TARGET_LOAD 90 static unsigned int default_target_loads[] = {DEFAULT_TARGET_LOAD}; -static spinlock_t target_loads_lock; -static unsigned int *target_loads = default_target_loads; -static int ntarget_loads = ARRAY_SIZE(default_target_loads); -/* - * The minimum amount of time to spend at a frequency before we can ramp down. - */ -#define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC) -static unsigned long min_sample_time = DEFAULT_MIN_SAMPLE_TIME; - -/* - * The sample rate of the timer used to increase frequency - */ #define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC) -static unsigned long timer_rate = DEFAULT_TIMER_RATE; - -/* - * Wait this long before raising speed above hispeed, by default a single - * timer interval. - */ #define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE static unsigned int default_above_hispeed_delay[] = { DEFAULT_ABOVE_HISPEED_DELAY }; -static spinlock_t above_hispeed_delay_lock; -static unsigned int *above_hispeed_delay = default_above_hispeed_delay; -static int nabove_hispeed_delay = ARRAY_SIZE(default_above_hispeed_delay); -/* Non-zero means indefinite speed boost active */ -static int boost_val; -/* Duration of a boot pulse in usecs */ -static int boostpulse_duration_val = DEFAULT_MIN_SAMPLE_TIME; -/* End time of boost pulse in ktime converted to usecs */ -static u64 boostpulse_endtime; +struct cpufreq_interactive_tunables { + int usage_count; -/* - * Max additional time to wait in idle, beyond timer_rate, at speeds above - * minimum before wakeup to reduce speed, or -1 if unnecessary. - */ + /* Hi speed to bump to from lo speed when load burst (default max) */ + unsigned int hispeed_freq; + + /* Go to hi speed when CPU load at or above this value. */ +#define DEFAULT_GO_HISPEED_LOAD 99 + unsigned long go_hispeed_load; + + /* Target load. Lower values result in higher CPU speeds. */ + spinlock_t target_loads_lock; + unsigned int *target_loads; + int ntarget_loads; + + /* + * The minimum amount of time to spend at a frequency before we can ramp + * down. + */ +#define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC) + unsigned long min_sample_time; + + /* + * The sample rate of the timer used to increase frequency + */ + unsigned long timer_rate; + + /* + * Wait this long before raising speed above hispeed, by default a + * single timer interval. + */ + spinlock_t above_hispeed_delay_lock; + unsigned int *above_hispeed_delay; + int nabove_hispeed_delay; + + /* Non-zero means indefinite speed boost active */ + int boost_val; + /* Duration of a boost pulse in usecs */ + int boostpulse_duration_val; + /* End time of boost pulse in ktime converted to usecs */ + u64 boostpulse_endtime; + + /* + * Max additional time to wait in idle, beyond timer_rate, at speeds + * above minimum before wakeup to reduce speed, or -1 if unnecessary. + */ #define DEFAULT_TIMER_SLACK (4 * DEFAULT_TIMER_RATE) -static int timer_slack_val = DEFAULT_TIMER_SLACK; + int timer_slack_val; -static bool io_is_busy; + bool io_is_busy; +}; + +/* For cases where we have single governor instance for system */ +struct cpufreq_interactive_tunables *common_tunables; + +static struct attribute_group *get_sysfs_attr(void); static void cpufreq_interactive_timer_resched( struct cpufreq_interactive_cpuinfo *pcpu) { + struct cpufreq_interactive_tunables *tunables = + pcpu->policy->governor_data; unsigned long expires; unsigned long flags; spin_lock_irqsave(&pcpu->load_lock, flags); pcpu->time_in_idle = get_cpu_idle_time(smp_processor_id(), - &pcpu->time_in_idle_timestamp, io_is_busy); + &pcpu->time_in_idle_timestamp, + tunables->io_is_busy); pcpu->cputime_speedadj = 0; pcpu->cputime_speedadj_timestamp = pcpu->time_in_idle_timestamp; - expires = jiffies + usecs_to_jiffies(timer_rate); + expires = jiffies + usecs_to_jiffies(tunables->timer_rate); mod_timer_pinned(&pcpu->cpu_timer, expires); - if (timer_slack_val >= 0 && pcpu->target_freq > pcpu->policy->min) { - expires += usecs_to_jiffies(timer_slack_val); + if (tunables->timer_slack_val >= 0 && + pcpu->target_freq > pcpu->policy->min) { + expires += usecs_to_jiffies(tunables->timer_slack_val); mod_timer_pinned(&pcpu->cpu_slack_timer, expires); } @@ -142,58 +157,65 @@ static void cpufreq_interactive_timer_resched( * The cpu_timer and cpu_slack_timer must be deactivated when calling this * function. */ -static void cpufreq_interactive_timer_start(int cpu) +static void cpufreq_interactive_timer_start( + struct cpufreq_interactive_tunables *tunables, int cpu) { struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, cpu); - unsigned long expires = jiffies + usecs_to_jiffies(timer_rate); + unsigned long expires = jiffies + usecs_to_jiffies(tunables->timer_rate); unsigned long flags; pcpu->cpu_timer.expires = expires; add_timer_on(&pcpu->cpu_timer, cpu); - if (timer_slack_val >= 0 && pcpu->target_freq > pcpu->policy->min) { - expires += usecs_to_jiffies(timer_slack_val); + if (tunables->timer_slack_val >= 0 && + pcpu->target_freq > pcpu->policy->min) { + expires += usecs_to_jiffies(tunables->timer_slack_val); pcpu->cpu_slack_timer.expires = expires; add_timer_on(&pcpu->cpu_slack_timer, cpu); } spin_lock_irqsave(&pcpu->load_lock, flags); pcpu->time_in_idle = - get_cpu_idle_time(cpu, &pcpu->time_in_idle_timestamp); + get_cpu_idle_time(cpu, &pcpu->time_in_idle_timestamp, + tunables->io_is_busy); pcpu->cputime_speedadj = 0; pcpu->cputime_speedadj_timestamp = pcpu->time_in_idle_timestamp; spin_unlock_irqrestore(&pcpu->load_lock, flags); } -static unsigned int freq_to_above_hispeed_delay(unsigned int freq) +static unsigned int freq_to_above_hispeed_delay( + struct cpufreq_interactive_tunables *tunables, + unsigned int freq) { int i; unsigned int ret; unsigned long flags; - spin_lock_irqsave(&above_hispeed_delay_lock, flags); + spin_lock_irqsave(&tunables->above_hispeed_delay_lock, flags); - for (i = 0; i < nabove_hispeed_delay - 1 && - freq >= above_hispeed_delay[i+1]; i += 2) + for (i = 0; i < tunables->nabove_hispeed_delay - 1 && + freq >= tunables->above_hispeed_delay[i+1]; i += 2) ; - ret = above_hispeed_delay[i]; - spin_unlock_irqrestore(&above_hispeed_delay_lock, flags); + ret = tunables->above_hispeed_delay[i]; + spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags); return ret; } -static unsigned int freq_to_targetload(unsigned int freq) +static unsigned int freq_to_targetload(struct cpufreq_interactive_tunables + *tunables, unsigned int freq) { int i; unsigned int ret; unsigned long flags; - spin_lock_irqsave(&target_loads_lock, flags); + spin_lock_irqsave(&tunables->target_loads_lock, flags); - for (i = 0; i < ntarget_loads - 1 && freq >= target_loads[i+1]; i += 2) + for (i = 0; i < tunables->ntarget_loads - 1 && + freq >= tunables->target_loads[i+1]; i += 2) ; - ret = target_loads[i]; - spin_unlock_irqrestore(&target_loads_lock, flags); + ret = tunables->target_loads[i]; + spin_unlock_irqrestore(&tunables->target_loads_lock, flags); return ret; } @@ -202,9 +224,8 @@ static unsigned int freq_to_targetload(unsigned int freq) * choose_freq() will find the minimum frequency that does not exceed its * target load given the current load. */ - -static unsigned int choose_freq( - struct cpufreq_interactive_cpuinfo *pcpu, unsigned int loadadjfreq) +static unsigned int choose_freq(struct cpufreq_interactive_cpuinfo *pcpu, + unsigned int loadadjfreq) { unsigned int freq = pcpu->policy->cur; unsigned int prevfreq, freqmin, freqmax; @@ -216,7 +237,7 @@ static unsigned int choose_freq( do { prevfreq = freq; - tl = freq_to_targetload(freq); + tl = freq_to_targetload(pcpu->policy->governor_data, freq); /* * Find the lowest frequency where the computed load is less @@ -291,13 +312,15 @@ static unsigned int choose_freq( static u64 update_load(int cpu) { struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, cpu); + struct cpufreq_interactive_tunables *tunables = + pcpu->policy->governor_data; u64 now; u64 now_idle; unsigned int delta_idle; unsigned int delta_time; u64 active_time; - now_idle = get_cpu_idle_time(cpu, &now, io_is_busy); + now_idle = get_cpu_idle_time(cpu, &now, tunables->io_is_busy); delta_idle = (unsigned int)(now_idle - pcpu->time_in_idle); delta_time = (unsigned int)(now - pcpu->time_in_idle_timestamp); @@ -321,6 +344,8 @@ static void cpufreq_interactive_timer(unsigned long data) int cpu_load; struct cpufreq_interactive_cpuinfo *pcpu = &per_cpu(cpuinfo, data); + struct cpufreq_interactive_tunables *tunables = + pcpu->policy->governor_data; unsigned int new_freq; unsigned int loadadjfreq; unsigned int index; @@ -344,25 +369,25 @@ static void cpufreq_interactive_timer(unsigned long data) do_div(cputime_speedadj, delta_time); loadadjfreq = (unsigned int)cputime_speedadj * 100; cpu_load = loadadjfreq / pcpu->target_freq; - boosted = boost_val || now < boostpulse_endtime; + boosted = tunables->boost_val || now < tunables->boostpulse_endtime; - if (cpu_load >= go_hispeed_load || boosted) { - if (pcpu->target_freq < hispeed_freq) { - new_freq = hispeed_freq; + if (cpu_load >= tunables->go_hispeed_load || boosted) { + if (pcpu->target_freq < tunables->hispeed_freq) { + new_freq = tunables->hispeed_freq; } else { new_freq = choose_freq(pcpu, loadadjfreq); - if (new_freq < hispeed_freq) - new_freq = hispeed_freq; + if (new_freq < tunables->hispeed_freq) + new_freq = tunables->hispeed_freq; } } else { new_freq = choose_freq(pcpu, loadadjfreq); } - if (pcpu->target_freq >= hispeed_freq && + if (pcpu->target_freq >= tunables->hispeed_freq && new_freq > pcpu->target_freq && now - pcpu->hispeed_validate_time < - freq_to_above_hispeed_delay(pcpu->target_freq)) { + freq_to_above_hispeed_delay(tunables, pcpu->target_freq)) { trace_cpufreq_interactive_notyet( data, cpu_load, pcpu->target_freq, pcpu->policy->cur, new_freq); @@ -383,7 +408,8 @@ static void cpufreq_interactive_timer(unsigned long data) * floor frequency for the minimum sample time since last validated. */ if (new_freq < pcpu->floor_freq) { - if (now - pcpu->floor_validate_time < min_sample_time) { + if (now - pcpu->floor_validate_time < + tunables->min_sample_time) { trace_cpufreq_interactive_notyet( data, cpu_load, pcpu->target_freq, pcpu->policy->cur, new_freq); @@ -399,7 +425,7 @@ static void cpufreq_interactive_timer(unsigned long data) * (or the indefinite boost is turned off). */ - if (!boosted || new_freq > hispeed_freq) { + if (!boosted || new_freq > tunables->hispeed_freq) { pcpu->floor_freq = new_freq; pcpu->floor_validate_time = now; } @@ -560,14 +586,16 @@ static void cpufreq_interactive_boost(void) int anyboost = 0; unsigned long flags; struct cpufreq_interactive_cpuinfo *pcpu; + struct cpufreq_interactive_tunables *tunables; spin_lock_irqsave(&speedchange_cpumask_lock, flags); for_each_online_cpu(i) { pcpu = &per_cpu(cpuinfo, i); + tunables = pcpu->policy->governor_data; - if (pcpu->target_freq < hispeed_freq) { - pcpu->target_freq = hispeed_freq; + if (pcpu->target_freq < tunables->hispeed_freq) { + pcpu->target_freq = tunables->hispeed_freq; cpumask_set_cpu(i, &speedchange_cpumask); pcpu->hispeed_validate_time = ktime_to_us(ktime_get()); @@ -579,7 +607,7 @@ static void cpufreq_interactive_boost(void) * validated. */ - pcpu->floor_freq = hispeed_freq; + pcpu->floor_freq = tunables->hispeed_freq; pcpu->floor_validate_time = ktime_to_us(ktime_get()); } @@ -678,27 +706,26 @@ err: return ERR_PTR(err); } -static ssize_t show_target_loads( - struct kobject *kobj, struct attribute *attr, char *buf) +static ssize_t show_target_loads(struct cpufreq_interactive_tunables *tunables, + char *buf) { int i; ssize_t ret = 0; unsigned long flags; - spin_lock_irqsave(&target_loads_lock, flags); + spin_lock_irqsave(&tunables->target_loads_lock, flags); - for (i = 0; i < ntarget_loads; i++) - ret += sprintf(buf + ret, "%u%s", target_loads[i], + for (i = 0; i < tunables->ntarget_loads; i++) + ret += sprintf(buf + ret, "%u%s", tunables->target_loads[i], i & 0x1 ? ":" : " "); ret += sprintf(buf + ret, "\n"); - spin_unlock_irqrestore(&target_loads_lock, flags); + spin_unlock_irqrestore(&tunables->target_loads_lock, flags); return ret; } -static ssize_t store_target_loads( - struct kobject *kobj, struct attribute *attr, const char *buf, - size_t count) +static ssize_t store_target_loads(struct cpufreq_interactive_tunables *tunables, + const char *buf, size_t count) { int ntokens; unsigned int *new_target_loads = NULL; @@ -708,40 +735,36 @@ static ssize_t store_target_loads( if (IS_ERR(new_target_loads)) return PTR_RET(new_target_loads); - spin_lock_irqsave(&target_loads_lock, flags); - if (target_loads != default_target_loads) - kfree(target_loads); - target_loads = new_target_loads; - ntarget_loads = ntokens; - spin_unlock_irqrestore(&target_loads_lock, flags); + spin_lock_irqsave(&tunables->target_loads_lock, flags); + if (tunables->target_loads != default_target_loads) + kfree(tunables->target_loads); + tunables->target_loads = new_target_loads; + tunables->ntarget_loads = ntokens; + spin_unlock_irqrestore(&tunables->target_loads_lock, flags); return count; } -static struct global_attr target_loads_attr = - __ATTR(target_loads, S_IRUGO | S_IWUSR, - show_target_loads, store_target_loads); - -static ssize_t show_above_hispeed_delay( - struct kobject *kobj, struct attribute *attr, char *buf) +static ssize_t show_above_hispeed_delay(struct cpufreq_interactive_tunables + *tunables, char *buf) { int i; ssize_t ret = 0; unsigned long flags; - spin_lock_irqsave(&above_hispeed_delay_lock, flags); + spin_lock_irqsave(&tunables->above_hispeed_delay_lock, flags); - for (i = 0; i < nabove_hispeed_delay; i++) - ret += sprintf(buf + ret, "%u%s", above_hispeed_delay[i], + for (i = 0; i < tunables->nabove_hispeed_delay; i++) + ret += sprintf(buf + ret, "%u%s", + tunables->above_hispeed_delay[i], i & 0x1 ? ":" : " "); ret += sprintf(buf + ret, "\n"); - spin_unlock_irqrestore(&above_hispeed_delay_lock, flags); + spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags); return ret; } -static ssize_t store_above_hispeed_delay( - struct kobject *kobj, struct attribute *attr, const char *buf, - size_t count) +static ssize_t store_above_hispeed_delay(struct cpufreq_interactive_tunables + *tunables, const char *buf, size_t count) { int ntokens; unsigned int *new_above_hispeed_delay = NULL; @@ -751,29 +774,24 @@ static ssize_t store_above_hispeed_delay( if (IS_ERR(new_above_hispeed_delay)) return PTR_RET(new_above_hispeed_delay); - spin_lock_irqsave(&above_hispeed_delay_lock, flags); - if (above_hispeed_delay != default_above_hispeed_delay) - kfree(above_hispeed_delay); - above_hispeed_delay = new_above_hispeed_delay; - nabove_hispeed_delay = ntokens; - spin_unlock_irqrestore(&above_hispeed_delay_lock, flags); + spin_lock_irqsave(&tunables->above_hispeed_delay_lock, flags); + if (tunables->above_hispeed_delay != default_above_hispeed_delay) + kfree(tunables->above_hispeed_delay); + tunables->above_hispeed_delay = new_above_hispeed_delay; + tunables->nabove_hispeed_delay = ntokens; + spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags); return count; } -static struct global_attr above_hispeed_delay_attr = - __ATTR(above_hispeed_delay, S_IRUGO | S_IWUSR, - show_above_hispeed_delay, store_above_hispeed_delay); - -static ssize_t show_hispeed_freq(struct kobject *kobj, - struct attribute *attr, char *buf) +static ssize_t show_hispeed_freq(struct cpufreq_interactive_tunables *tunables, + char *buf) { - return sprintf(buf, "%u\n", hispeed_freq); + return sprintf(buf, "%u\n", tunables->hispeed_freq); } -static ssize_t store_hispeed_freq(struct kobject *kobj, - struct attribute *attr, const char *buf, - size_t count) +static ssize_t store_hispeed_freq(struct cpufreq_interactive_tunables *tunables, + const char *buf, size_t count) { int ret; long unsigned int val; @@ -781,22 +799,18 @@ static ssize_t store_hispeed_freq(struct kobject *kobj, ret = strict_strtoul(buf, 0, &val); if (ret < 0) return ret; - hispeed_freq = val; + tunables->hispeed_freq = val; return count; } -static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644, - show_hispeed_freq, store_hispeed_freq); - - -static ssize_t show_go_hispeed_load(struct kobject *kobj, - struct attribute *attr, char *buf) +static ssize_t show_go_hispeed_load(struct cpufreq_interactive_tunables + *tunables, char *buf) { - return sprintf(buf, "%lu\n", go_hispeed_load); + return sprintf(buf, "%lu\n", tunables->go_hispeed_load); } -static ssize_t store_go_hispeed_load(struct kobject *kobj, - struct attribute *attr, const char *buf, size_t count) +static ssize_t store_go_hispeed_load(struct cpufreq_interactive_tunables + *tunables, const char *buf, size_t count) { int ret; unsigned long val; @@ -804,21 +818,18 @@ static ssize_t store_go_hispeed_load(struct kobject *kobj, ret = strict_strtoul(buf, 0, &val); if (ret < 0) return ret; - go_hispeed_load = val; + tunables->go_hispeed_load = val; return count; } -static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644, - show_go_hispeed_load, store_go_hispeed_load); - -static ssize_t show_min_sample_time(struct kobject *kobj, - struct attribute *attr, char *buf) +static ssize_t show_min_sample_time(struct cpufreq_interactive_tunables + *tunables, char *buf) { - return sprintf(buf, "%lu\n", min_sample_time); + return sprintf(buf, "%lu\n", tunables->min_sample_time); } -static ssize_t store_min_sample_time(struct kobject *kobj, - struct attribute *attr, const char *buf, size_t count) +static ssize_t store_min_sample_time(struct cpufreq_interactive_tunables + *tunables, const char *buf, size_t count) { int ret; unsigned long val; @@ -826,21 +837,18 @@ static ssize_t store_min_sample_time(struct kobject *kobj, ret = strict_strtoul(buf, 0, &val); if (ret < 0) return ret; - min_sample_time = val; + tunables->min_sample_time = val; return count; } -static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644, - show_min_sample_time, store_min_sample_time); - -static ssize_t show_timer_rate(struct kobject *kobj, - struct attribute *attr, char *buf) +static ssize_t show_timer_rate(struct cpufreq_interactive_tunables *tunables, + char *buf) { - return sprintf(buf, "%lu\n", timer_rate); + return sprintf(buf, "%lu\n", tunables->timer_rate); } -static ssize_t store_timer_rate(struct kobject *kobj, - struct attribute *attr, const char *buf, size_t count) +static ssize_t store_timer_rate(struct cpufreq_interactive_tunables *tunables, + const char *buf, size_t count) { int ret; unsigned long val; @@ -848,22 +856,18 @@ static ssize_t store_timer_rate(struct kobject *kobj, ret = strict_strtoul(buf, 0, &val); if (ret < 0) return ret; - timer_rate = val; + tunables->timer_rate = val; return count; } -static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644, - show_timer_rate, store_timer_rate); - -static ssize_t show_timer_slack( - struct kobject *kobj, struct attribute *attr, char *buf) +static ssize_t show_timer_slack(struct cpufreq_interactive_tunables *tunables, + char *buf) { - return sprintf(buf, "%d\n", timer_slack_val); + return sprintf(buf, "%d\n", tunables->timer_slack_val); } -static ssize_t store_timer_slack( - struct kobject *kobj, struct attribute *attr, const char *buf, - size_t count) +static ssize_t store_timer_slack(struct cpufreq_interactive_tunables *tunables, + const char *buf, size_t count) { int ret; unsigned long val; @@ -872,19 +876,17 @@ static ssize_t store_timer_slack( if (ret < 0) return ret; - timer_slack_val = val; + tunables->timer_slack_val = val; return count; } -define_one_global_rw(timer_slack); - -static ssize_t show_boost(struct kobject *kobj, struct attribute *attr, +static ssize_t show_boost(struct cpufreq_interactive_tunables *tunables, char *buf) { - return sprintf(buf, "%d\n", boost_val); + return sprintf(buf, "%d\n", tunables->boost_val); } -static ssize_t store_boost(struct kobject *kobj, struct attribute *attr, +static ssize_t store_boost(struct cpufreq_interactive_tunables *tunables, const char *buf, size_t count) { int ret; @@ -894,9 +896,9 @@ static ssize_t store_boost(struct kobject *kobj, struct attribute *attr, if (ret < 0) return ret; - boost_val = val; + tunables->boost_val = val; - if (boost_val) { + if (tunables->boost_val) { trace_cpufreq_interactive_boost("on"); cpufreq_interactive_boost(); } else { @@ -906,9 +908,7 @@ static ssize_t store_boost(struct kobject *kobj, struct attribute *attr, return count; } -define_one_global_rw(boost); - -static ssize_t store_boostpulse(struct kobject *kobj, struct attribute *attr, +static ssize_t store_boostpulse(struct cpufreq_interactive_tunables *tunables, const char *buf, size_t count) { int ret; @@ -918,24 +918,21 @@ static ssize_t store_boostpulse(struct kobject *kobj, struct attribute *attr, if (ret < 0) return ret; - boostpulse_endtime = ktime_to_us(ktime_get()) + boostpulse_duration_val; + tunables->boostpulse_endtime = ktime_to_us(ktime_get()) + + tunables->boostpulse_duration_val; trace_cpufreq_interactive_boost("pulse"); cpufreq_interactive_boost(); return count; } -static struct global_attr boostpulse = - __ATTR(boostpulse, 0200, NULL, store_boostpulse); - -static ssize_t show_boostpulse_duration( - struct kobject *kobj, struct attribute *attr, char *buf) +static ssize_t show_boostpulse_duration(struct cpufreq_interactive_tunables + *tunables, char *buf) { - return sprintf(buf, "%d\n", boostpulse_duration_val); + return sprintf(buf, "%d\n", tunables->boostpulse_duration_val); } -static ssize_t store_boostpulse_duration( - struct kobject *kobj, struct attribute *attr, const char *buf, - size_t count) +static ssize_t store_boostpulse_duration(struct cpufreq_interactive_tunables + *tunables, const char *buf, size_t count) { int ret; unsigned long val; @@ -944,20 +941,18 @@ static ssize_t store_boostpulse_duration( if (ret < 0) return ret; - boostpulse_duration_val = val; + tunables->boostpulse_duration_val = val; return count; } -define_one_global_rw(boostpulse_duration); - -static ssize_t show_io_is_busy(struct kobject *kobj, - struct attribute *attr, char *buf) +static ssize_t show_io_is_busy(struct cpufreq_interactive_tunables *tunables, + char *buf) { - return sprintf(buf, "%u\n", io_is_busy); + return sprintf(buf, "%u\n", tunables->io_is_busy); } -static ssize_t store_io_is_busy(struct kobject *kobj, - struct attribute *attr, const char *buf, size_t count) +static ssize_t store_io_is_busy(struct cpufreq_interactive_tunables *tunables, + const char *buf, size_t count) { int ret; unsigned long val; @@ -965,33 +960,125 @@ static ssize_t store_io_is_busy(struct kobject *kobj, ret = kstrtoul(buf, 0, &val); if (ret < 0) return ret; - io_is_busy = val; + tunables->io_is_busy = val; return count; } -static struct global_attr io_is_busy_attr = __ATTR(io_is_busy, 0644, - show_io_is_busy, store_io_is_busy); - -static struct attribute *interactive_attributes[] = { - &target_loads_attr.attr, - &above_hispeed_delay_attr.attr, - &hispeed_freq_attr.attr, - &go_hispeed_load_attr.attr, - &min_sample_time_attr.attr, - &timer_rate_attr.attr, - &timer_slack.attr, - &boost.attr, - &boostpulse.attr, - &boostpulse_duration.attr, - &io_is_busy_attr.attr, +/* + * Create show/store routines + * - sys: One governor instance for complete SYSTEM + * - pol: One governor instance per struct cpufreq_policy + */ +#define show_gov_pol_sys(file_name) \ +static ssize_t show_##file_name##_gov_sys \ +(struct kobject *kobj, struct attribute *attr, char *buf) \ +{ \ + return show_##file_name(common_tunables, buf); \ +} \ + \ +static ssize_t show_##file_name##_gov_pol \ +(struct cpufreq_policy *policy, char *buf) \ +{ \ + return show_##file_name(policy->governor_data, buf); \ +} + +#define store_gov_pol_sys(file_name) \ +static ssize_t store_##file_name##_gov_sys \ +(struct kobject *kobj, struct attribute *attr, const char *buf, \ + size_t count) \ +{ \ + return store_##file_name(common_tunables, buf, count); \ +} \ + \ +static ssize_t store_##file_name##_gov_pol \ +(struct cpufreq_policy *policy, const char *buf, size_t count) \ +{ \ + return store_##file_name(policy->governor_data, buf, count); \ +} + +#define show_store_gov_pol_sys(file_name) \ +show_gov_pol_sys(file_name); \ +store_gov_pol_sys(file_name) + +show_store_gov_pol_sys(target_loads); +show_store_gov_pol_sys(above_hispeed_delay); +show_store_gov_pol_sys(hispeed_freq); +show_store_gov_pol_sys(go_hispeed_load); +show_store_gov_pol_sys(min_sample_time); +show_store_gov_pol_sys(timer_rate); +show_store_gov_pol_sys(timer_slack); +show_store_gov_pol_sys(boost); +store_gov_pol_sys(boostpulse); +show_store_gov_pol_sys(boostpulse_duration); +show_store_gov_pol_sys(io_is_busy); + +gov_sys_pol_attr_rw(target_loads); +gov_sys_pol_attr_rw(above_hispeed_delay); +gov_sys_pol_attr_rw(hispeed_freq); +gov_sys_pol_attr_rw(go_hispeed_load); +gov_sys_pol_attr_rw(min_sample_time); +gov_sys_pol_attr_rw(timer_rate); +gov_sys_pol_attr_rw(timer_slack); +gov_sys_pol_attr_rw(boost); +gov_sys_pol_attr_rw(boostpulse_duration); +gov_sys_pol_attr_rw(io_is_busy); + +static struct global_attr boostpulse_gov_sys = + __ATTR(boostpulse, 0200, NULL, store_boostpulse_gov_sys); + +static struct freq_attr boostpulse_gov_pol = + __ATTR(boostpulse, 0200, NULL, store_boostpulse_gov_pol); + +/* One Governor instance for entire system */ +static struct attribute *interactive_attributes_gov_sys[] = { + &target_loads_gov_sys.attr, + &above_hispeed_delay_gov_sys.attr, + &hispeed_freq_gov_sys.attr, + &go_hispeed_load_gov_sys.attr, + &min_sample_time_gov_sys.attr, + &timer_rate_gov_sys.attr, + &timer_slack_gov_sys.attr, + &boost_gov_sys.attr, + &boostpulse_gov_sys.attr, + &boostpulse_duration_gov_sys.attr, + &io_is_busy_gov_sys.attr, NULL, }; -static struct attribute_group interactive_attr_group = { - .attrs = interactive_attributes, +static struct attribute_group interactive_attr_group_gov_sys = { + .attrs = interactive_attributes_gov_sys, .name = "interactive", }; +/* Per policy governor instance */ +static struct attribute *interactive_attributes_gov_pol[] = { + &target_loads_gov_pol.attr, + &above_hispeed_delay_gov_pol.attr, + &hispeed_freq_gov_pol.attr, + &go_hispeed_load_gov_pol.attr, + &min_sample_time_gov_pol.attr, + &timer_rate_gov_pol.attr, + &timer_slack_gov_pol.attr, + &boost_gov_pol.attr, + &boostpulse_gov_pol.attr, + &boostpulse_duration_gov_pol.attr, + &io_is_busy_gov_pol.attr, + NULL, +}; + +static struct attribute_group interactive_attr_group_gov_pol = { + .attrs = interactive_attributes_gov_pol, + .name = "interactive", +}; + +static struct attribute_group *get_sysfs_attr(void) +{ + if (have_governor_per_policy()) + return &interactive_attr_group_gov_pol; + else + return &interactive_attr_group_gov_sys; +} + static int cpufreq_interactive_idle_notifier(struct notifier_block *nb, unsigned long val, void *data) @@ -1019,15 +1106,88 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy, unsigned int j; struct cpufreq_interactive_cpuinfo *pcpu; struct cpufreq_frequency_table *freq_table; + struct cpufreq_interactive_tunables *tunables; + + if (have_governor_per_policy()) + tunables = policy->governor_data; + else + tunables = common_tunables; + + WARN_ON(!tunables && (event != CPUFREQ_GOV_POLICY_INIT)); switch (event) { + case CPUFREQ_GOV_POLICY_INIT: + if (have_governor_per_policy()) { + WARN_ON(tunables); + } else if (tunables) { + tunables->usage_count++; + policy->governor_data = tunables; + return 0; + } + + tunables = kzalloc(sizeof(*tunables), GFP_KERNEL); + if (!tunables) { + pr_err("%s: POLICY_INIT: kzalloc failed\n", __func__); + return -ENOMEM; + } + + rc = sysfs_create_group(get_governor_parent_kobj(policy), + get_sysfs_attr()); + if (rc) { + kfree(tunables); + return rc; + } + + tunables->usage_count = 1; + tunables->above_hispeed_delay = default_above_hispeed_delay; + tunables->nabove_hispeed_delay = + ARRAY_SIZE(default_above_hispeed_delay); + tunables->go_hispeed_load = DEFAULT_GO_HISPEED_LOAD; + tunables->target_loads = default_target_loads; + tunables->ntarget_loads = ARRAY_SIZE(default_target_loads); + tunables->min_sample_time = DEFAULT_MIN_SAMPLE_TIME; + tunables->timer_rate = DEFAULT_TIMER_RATE; + tunables->boostpulse_duration_val = DEFAULT_MIN_SAMPLE_TIME; + tunables->timer_slack_val = DEFAULT_TIMER_SLACK; + + spin_lock_init(&tunables->target_loads_lock); + spin_lock_init(&tunables->above_hispeed_delay_lock); + + if (!policy->governor->initialized) { + idle_notifier_register(&cpufreq_interactive_idle_nb); + cpufreq_register_notifier(&cpufreq_notifier_block, + CPUFREQ_TRANSITION_NOTIFIER); + } + + policy->governor_data = tunables; + if (!have_governor_per_policy()) + common_tunables = tunables; + + break; + + case CPUFREQ_GOV_POLICY_EXIT: + if (!--tunables->usage_count) { + if (policy->governor->initialized == 1) { + cpufreq_unregister_notifier(&cpufreq_notifier_block, + CPUFREQ_TRANSITION_NOTIFIER); + idle_notifier_unregister(&cpufreq_interactive_idle_nb); + } + + sysfs_remove_group(get_governor_parent_kobj(policy), + get_sysfs_attr()); + kfree(tunables); + common_tunables = NULL; + } + + policy->governor_data = NULL; + break; + case CPUFREQ_GOV_START: mutex_lock(&gov_lock); - freq_table = - cpufreq_frequency_get_table(policy->cpu); - if (!hispeed_freq) - hispeed_freq = policy->max; + freq_table = cpufreq_frequency_get_table(policy->cpu); + if (!tunables->hispeed_freq) + tunables->hispeed_freq = policy->max; for_each_cpu(j, policy->cpus) { pcpu = &per_cpu(cpuinfo, j); @@ -1040,30 +1200,11 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy, pcpu->hispeed_validate_time = pcpu->floor_validate_time; down_write(&pcpu->enable_sem); - cpufreq_interactive_timer_start(j); + cpufreq_interactive_timer_start(tunables, j); pcpu->governor_enabled = 1; up_write(&pcpu->enable_sem); } - /* - * Do not register the idle hook and create sysfs - * entries if we have already done so. - */ - if (++active_count > 1) { - mutex_unlock(&gov_lock); - return 0; - } - - rc = sysfs_create_group(cpufreq_global_kobject, - &interactive_attr_group); - if (rc) { - mutex_unlock(&gov_lock); - return rc; - } - - idle_notifier_register(&cpufreq_interactive_idle_nb); - cpufreq_register_notifier( - &cpufreq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER); mutex_unlock(&gov_lock); break; @@ -1078,18 +1219,7 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy, up_write(&pcpu->enable_sem); } - if (--active_count > 0) { - mutex_unlock(&gov_lock); - return 0; - } - - cpufreq_unregister_notifier( - &cpufreq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER); - idle_notifier_unregister(&cpufreq_interactive_idle_nb); - sysfs_remove_group(cpufreq_global_kobject, - &interactive_attr_group); mutex_unlock(&gov_lock); - break; case CPUFREQ_GOV_LIMITS: @@ -1123,7 +1253,7 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy, */ del_timer_sync(&pcpu->cpu_timer); del_timer_sync(&pcpu->cpu_slack_timer); - cpufreq_interactive_timer_start(j); + cpufreq_interactive_timer_start(tunables, j); up_write(&pcpu->enable_sem); } break; @@ -1163,9 +1293,7 @@ static int __init cpufreq_interactive_init(void) init_rwsem(&pcpu->enable_sem); } - spin_lock_init(&target_loads_lock); spin_lock_init(&speedchange_cpumask_lock); - spin_lock_init(&above_hispeed_delay_lock); mutex_init(&gov_lock); speedchange_task = kthread_create(cpufreq_interactive_speedchange_task, NULL, -- cgit v1.2.3 From 1d7f81197e575c00bf5dfa988f4c7a12c45993b2 Mon Sep 17 00:00:00 2001 From: Chris Redpath Date: Mon, 17 Jun 2013 18:36:56 +0100 Subject: cpufreq: interactive governor drops bits in time calculation Keep time calculation in 64-bit throughout. If we have long times between idle calculations this can result in deltas > 32 bits which causes incorrect load percentage calculations and selecting the wrong frequencies if we truncate here. Signed-off-by: Chris Redpath --- drivers/cpufreq/cpufreq_interactive.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/cpufreq/cpufreq_interactive.c b/drivers/cpufreq/cpufreq_interactive.c index 7f1782ade53c..a494fbd7dbe8 100644 --- a/drivers/cpufreq/cpufreq_interactive.c +++ b/drivers/cpufreq/cpufreq_interactive.c @@ -316,13 +316,13 @@ static u64 update_load(int cpu) pcpu->policy->governor_data; u64 now; u64 now_idle; - unsigned int delta_idle; - unsigned int delta_time; + u64 delta_idle; + u64 delta_time; u64 active_time; now_idle = get_cpu_idle_time(cpu, &now, tunables->io_is_busy); - delta_idle = (unsigned int)(now_idle - pcpu->time_in_idle); - delta_time = (unsigned int)(now - pcpu->time_in_idle_timestamp); + delta_idle = (now_idle - pcpu->time_in_idle); + delta_time = (now - pcpu->time_in_idle_timestamp); if (delta_time <= delta_idle) active_time = 0; -- cgit v1.2.3