aboutsummaryrefslogtreecommitdiff
path: root/kernel/sched/cpufreq_sched.c
blob: 8c5ad4805e2953cfe7bfb7270ee26d01b8354e3b (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
/*
 *  Copyright (C)  2015 Michael Turquette <mturquette@linaro.org>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#include <linux/cpufreq.h>
#include <linux/module.h>
#include <linux/kthread.h>
#include <linux/percpu.h>
#include <linux/irq_work.h>

#include "sched.h"

#define THROTTLE_NSEC		50000000 /* 50ms default */

static DEFINE_PER_CPU(unsigned long, pcpu_capacity);
static DEFINE_PER_CPU(struct cpufreq_policy *, pcpu_policy);
static DEFINE_PER_CPU(int, governor_started);

/**
 * gov_data - per-policy data internal to the governor
 * @throttle: next throttling period expiry. Derived from throttle_nsec
 * @throttle_nsec: throttle period length in nanoseconds
 * @task: worker thread for dvfs transition that may block/sleep
 * @irq_work: callback used to wake up worker thread
 * @freq: new frequency stored in *_sched_update_cpu and used in *_sched_thread
 *
 * struct gov_data is the per-policy cpufreq_sched-specific data structure. A
 * per-policy instance of it is created when the cpufreq_sched governor receives
 * the CPUFREQ_GOV_START condition and a pointer to it exists in the gov_data
 * member of struct cpufreq_policy.
 *
 * Readers of this data must call down_read(policy->rwsem). Writers must
 * call down_write(policy->rwsem).
 */
struct gov_data {
	ktime_t throttle;
	unsigned int throttle_nsec;
	struct task_struct *task;
	struct irq_work irq_work;
	struct cpufreq_policy *policy;
	unsigned int freq;
};

static void cpufreq_sched_try_driver_target(struct cpufreq_policy *policy, unsigned int freq)
{
	struct gov_data *gd = policy->governor_data;

	/* avoid race with cpufreq_sched_stop */
	if (!down_write_trylock(&policy->rwsem))
		return;

	__cpufreq_driver_target(policy, freq, CPUFREQ_RELATION_L);

	gd->throttle = ktime_add_ns(ktime_get(), gd->throttle_nsec);
	up_write(&policy->rwsem);
}

/*
 * we pass in struct cpufreq_policy. This is safe because changing out the
 * policy requires a call to __cpufreq_governor(policy, CPUFREQ_GOV_STOP),
 * which tears down all of the data structures and __cpufreq_governor(policy,
 * CPUFREQ_GOV_START) will do a full rebuild, including this kthread with the
 * new policy pointer
 */
static int cpufreq_sched_thread(void *data)
{
	struct sched_param param;
	struct cpufreq_policy *policy;
	struct gov_data *gd;
	int ret;

	policy = (struct cpufreq_policy *) data;
	if (!policy) {
		pr_warn("%s: missing policy\n", __func__);
		do_exit(-EINVAL);
	}

	gd = policy->governor_data;
	if (!gd) {
		pr_warn("%s: missing governor data\n", __func__);
		do_exit(-EINVAL);
	}

	param.sched_priority = 50;
	ret = sched_setscheduler_nocheck(gd->task, SCHED_FIFO, &param);
	if (ret) {
		pr_warn("%s: failed to set SCHED_FIFO\n", __func__);
		do_exit(-EINVAL);
	} else {
		pr_debug("%s: kthread (%d) set to SCHED_FIFO\n",
				__func__, gd->task->pid);
	}

	ret = set_cpus_allowed_ptr(gd->task, policy->related_cpus);
	if (ret) {
		pr_warn("%s: failed to set allowed ptr\n", __func__);
		do_exit(-EINVAL);
	}

	/* main loop of the per-policy kthread */
	do {
		set_current_state(TASK_INTERRUPTIBLE);
		schedule();
		if (kthread_should_stop())
			break;

		cpufreq_sched_try_driver_target(policy, gd->freq);
	} while (!kthread_should_stop());

	do_exit(0);
}

static void cpufreq_sched_irq_work(struct irq_work *irq_work)
{
	struct gov_data *gd;

	gd = container_of(irq_work, struct gov_data, irq_work);
	if (!gd) {
		return;
	}

	wake_up_process(gd->task);
}

/**
 * cpufreq_sched_set_capacity - interface to scheduler for changing capacity values
 * @cpu: cpu whose capacity utilization has recently changed
 * @capacity: the new capacity requested by cpu
 *
 * cpufreq_sched_sched_capacity is an interface exposed to the scheduler so
 * that the scheduler may inform the governor of updates to capacity
 * utilization and make changes to cpu frequency. Currently this interface is
 * designed around PELT values in CFS. It can be expanded to other scheduling
 * classes in the future if needed.
 *
 * cpufreq_sched_set_capacity raises an IPI. The irq_work handler for that IPI
 * wakes up the thread that does the actual work, cpufreq_sched_thread.
 *
 * This functions bails out early if either condition is true:
 * 1) this cpu did not the new maximum capacity for its frequency domain
 * 2) no change in cpu frequency is necessary to meet the new capacity request
 */
void cpufreq_sched_set_cap(int cpu, unsigned long capacity)
{
	unsigned int freq_new, cpu_tmp;
	struct cpufreq_policy *policy;
	struct gov_data *gd;
	unsigned long capacity_max = 0;

	if (!per_cpu(governor_started, cpu))
		return;

	/* update per-cpu capacity request */
	per_cpu(pcpu_capacity, cpu) = capacity;

	policy = cpufreq_cpu_get(cpu);
	if (IS_ERR_OR_NULL(policy)) {
		return;
	}

	if (!policy->governor_data)
		goto out;

	gd = policy->governor_data;

	/* bail early if we are throttled */
	if (ktime_before(ktime_get(), gd->throttle))
		goto out;

	/* find max capacity requested by cpus in this policy */
	for_each_cpu(cpu_tmp, policy->cpus)
		capacity_max = max(capacity_max, per_cpu(pcpu_capacity, cpu_tmp));

	/*
	 * We only change frequency if this cpu's capacity request represents a
	 * new max. If another cpu has requested a capacity greater than the
	 * previous max then we rely on that cpu to hit this code path and make
	 * the change. IOW, the cpu with the new max capacity is responsible
	 * for setting the new capacity/frequency.
	 *
	 * If this cpu is not the new maximum then bail
	 */
	if (capacity_max > capacity)
		goto out;

	/* Convert the new maximum capacity request into a cpu frequency */
	freq_new = (capacity * policy->max) / capacity_orig_of(cpu);

	/* No change in frequency? Bail and return current capacity. */
	if (freq_new == policy->cur)
		goto out;

	/* store the new frequency and perform the transition */
	gd->freq = freq_new;

	if (cpufreq_driver_might_sleep())
		irq_work_queue_on(&gd->irq_work, cpu);
	else
		cpufreq_sched_try_driver_target(policy, freq_new);

out:
	cpufreq_cpu_put(policy);
	return;
}

/**
 * cpufreq_sched_reset_capacity - interface to scheduler for resetting capacity
 *                                requests
 * @cpu: cpu whose capacity request has to be reset
 *
 * This _wont trigger_ any capacity update.
 */
void cpufreq_sched_reset_cap(int cpu)
{
	per_cpu(pcpu_capacity, cpu) = 0;
}

static inline void set_sched_energy_freq(void)
{
	static_key_slow_inc(&__sched_energy_freq);
}

static inline void clear_sched_energy_freq(void)
{
	static_key_slow_dec(&__sched_energy_freq);
}

static int cpufreq_sched_start(struct cpufreq_policy *policy)
{
	struct gov_data *gd;
	int cpu;

	/* prepare per-policy private data */
	gd = kzalloc(sizeof(*gd), GFP_KERNEL);
	if (!gd) {
		pr_debug("%s: failed to allocate private data\n", __func__);
		return -ENOMEM;
	}

	/* initialize per-cpu data */
	for_each_cpu(cpu, policy->cpus) {
		per_cpu(pcpu_capacity, cpu) = 0;
		per_cpu(pcpu_policy, cpu) = policy;
	}

	/*
	 * Don't ask for freq changes at an higher rate than what
	 * the driver advertises as transition latency.
	 */
	gd->throttle_nsec = policy->cpuinfo.transition_latency ?
			    policy->cpuinfo.transition_latency :
			    THROTTLE_NSEC;
	pr_debug("%s: throttle threshold = %u [ns]\n",
		  __func__, gd->throttle_nsec);

	if (cpufreq_driver_might_sleep()) {
		/* init per-policy kthread */
		gd->task = kthread_run(cpufreq_sched_thread, policy, "kcpufreq_sched_task");
		if (IS_ERR_OR_NULL(gd->task)) {
			pr_err("%s: failed to create kcpufreq_sched_task thread\n", __func__);
			goto err;
		}
		init_irq_work(&gd->irq_work, cpufreq_sched_irq_work);
	}

	policy->governor_data = gd;
	gd->policy = policy;
	set_sched_energy_freq();

	for_each_cpu(cpu, policy->cpus)
		per_cpu(governor_started, cpu) = 1;

	return 0;

err:
	kfree(gd);
	return -ENOMEM;
}

static int cpufreq_sched_stop(struct cpufreq_policy *policy)
{
	struct gov_data *gd = policy->governor_data;
	int cpu;

	for_each_cpu(cpu, policy->cpus)
		per_cpu(governor_started, cpu) = 0;

	clear_sched_energy_freq();
	if (cpufreq_driver_might_sleep()) {
		kthread_stop(gd->task);
	}

	policy->governor_data = NULL;

	/* FIXME replace with devm counterparts? */
	kfree(gd);
	return 0;
}

static int cpufreq_sched_setup(struct cpufreq_policy *policy, unsigned int event)
{
	switch (event) {
		case CPUFREQ_GOV_START:
			/* Start managing the frequency */
			return cpufreq_sched_start(policy);

		case CPUFREQ_GOV_STOP:
			return cpufreq_sched_stop(policy);

		case CPUFREQ_GOV_LIMITS:	/* unused */
		case CPUFREQ_GOV_POLICY_INIT:	/* unused */
		case CPUFREQ_GOV_POLICY_EXIT:	/* unused */
			break;
	}
	return 0;
}

#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_SCHED
static
#endif
struct cpufreq_governor cpufreq_gov_sched = {
	.name			= "sched",
	.governor		= cpufreq_sched_setup,
	.owner			= THIS_MODULE,
};

static int __init cpufreq_sched_init(void)
{
	int cpu;

	for_each_cpu(cpu, cpu_possible_mask) {
		per_cpu(governor_started, cpu) = 0;
	}
	return cpufreq_register_governor(&cpufreq_gov_sched);
}

static void __exit cpufreq_sched_exit(void)
{
	cpufreq_unregister_governor(&cpufreq_gov_sched);
}

/* Try to make this the default governor */
fs_initcall(cpufreq_sched_init);

MODULE_LICENSE("GPL v2");