blob: 99890831d55714a14e32ec6ab8876b93a0a2fa74 [file] [log] [blame]
Andrew Morton3fcfab12006-10-19 23:28:16 -07001
2#include <linux/wait.h>
3#include <linux/backing-dev.h>
Jens Axboe03ba3782009-09-09 09:08:54 +02004#include <linux/kthread.h>
5#include <linux/freezer.h>
Andrew Morton3fcfab12006-10-19 23:28:16 -07006#include <linux/fs.h>
Jens Axboe26160152009-03-17 09:35:06 +01007#include <linux/pagemap.h>
Jens Axboe03ba3782009-09-09 09:08:54 +02008#include <linux/mm.h>
Andrew Morton3fcfab12006-10-19 23:28:16 -07009#include <linux/sched.h>
10#include <linux/module.h>
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -070011#include <linux/writeback.h>
12#include <linux/device.h>
Dave Chinner455b2862010-07-07 13:24:06 +100013#include <trace/events/writeback.h>
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -070014
Jens Axboec3c53202010-04-22 11:37:01 +020015static atomic_long_t bdi_seq = ATOMIC_LONG_INIT(0);
16
Jens Axboe26160152009-03-17 09:35:06 +010017void default_unplug_io_fn(struct backing_dev_info *bdi, struct page *page)
18{
19}
20EXPORT_SYMBOL(default_unplug_io_fn);
21
22struct backing_dev_info default_backing_dev_info = {
Jens Axboed9938312009-06-12 14:45:52 +020023 .name = "default",
Jens Axboe26160152009-03-17 09:35:06 +010024 .ra_pages = VM_MAX_READAHEAD * 1024 / PAGE_CACHE_SIZE,
25 .state = 0,
26 .capabilities = BDI_CAP_MAP_COPY,
27 .unplug_io_fn = default_unplug_io_fn,
28};
29EXPORT_SYMBOL_GPL(default_backing_dev_info);
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -070030
Jörn Engel5129a462010-04-25 08:54:42 +020031struct backing_dev_info noop_backing_dev_info = {
32 .name = "noop",
33};
34EXPORT_SYMBOL_GPL(noop_backing_dev_info);
35
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -070036static struct class *bdi_class;
Jens Axboecfc4ba52009-09-14 13:12:40 +020037
38/*
39 * bdi_lock protects updates to bdi_list and bdi_pending_list, as well as
40 * reader side protection for bdi_pending_list. bdi_list has RCU reader side
41 * locking.
42 */
Jens Axboe03ba3782009-09-09 09:08:54 +020043DEFINE_SPINLOCK(bdi_lock);
Jens Axboe66f3b8e2009-09-02 09:19:46 +020044LIST_HEAD(bdi_list);
Jens Axboe03ba3782009-09-09 09:08:54 +020045LIST_HEAD(bdi_pending_list);
46
47static struct task_struct *sync_supers_tsk;
48static struct timer_list sync_supers_timer;
49
50static int bdi_sync_supers(void *);
51static void sync_supers_timer_fn(unsigned long);
Jens Axboe03ba3782009-09-09 09:08:54 +020052
Miklos Szeredi76f14182008-04-30 00:54:36 -070053#ifdef CONFIG_DEBUG_FS
54#include <linux/debugfs.h>
55#include <linux/seq_file.h>
56
57static struct dentry *bdi_debug_root;
58
59static void bdi_debug_init(void)
60{
61 bdi_debug_root = debugfs_create_dir("bdi", NULL);
62}
63
64static int bdi_debug_stats_show(struct seq_file *m, void *v)
65{
66 struct backing_dev_info *bdi = m->private;
Christoph Hellwigc1955ce2010-06-19 23:08:06 +020067 struct bdi_writeback *wb = &bdi->wb;
David Rientjes364aeb22009-01-06 14:39:29 -080068 unsigned long background_thresh;
69 unsigned long dirty_thresh;
70 unsigned long bdi_thresh;
Jens Axboef09b00d2009-05-25 09:08:21 +020071 unsigned long nr_dirty, nr_io, nr_more_io, nr_wb;
72 struct inode *inode;
73
Jens Axboef09b00d2009-05-25 09:08:21 +020074 nr_wb = nr_dirty = nr_io = nr_more_io = 0;
75 spin_lock(&inode_lock);
Christoph Hellwigc1955ce2010-06-19 23:08:06 +020076 list_for_each_entry(inode, &wb->b_dirty, i_list)
77 nr_dirty++;
78 list_for_each_entry(inode, &wb->b_io, i_list)
79 nr_io++;
80 list_for_each_entry(inode, &wb->b_more_io, i_list)
81 nr_more_io++;
Jens Axboef09b00d2009-05-25 09:08:21 +020082 spin_unlock(&inode_lock);
Miklos Szeredi76f14182008-04-30 00:54:36 -070083
84 get_dirty_limits(&background_thresh, &dirty_thresh, &bdi_thresh, bdi);
85
86#define K(x) ((x) << (PAGE_SHIFT - 10))
87 seq_printf(m,
88 "BdiWriteback: %8lu kB\n"
89 "BdiReclaimable: %8lu kB\n"
90 "BdiDirtyThresh: %8lu kB\n"
91 "DirtyThresh: %8lu kB\n"
Jens Axboef09b00d2009-05-25 09:08:21 +020092 "BackgroundThresh: %8lu kB\n"
Jens Axboef09b00d2009-05-25 09:08:21 +020093 "b_dirty: %8lu\n"
94 "b_io: %8lu\n"
95 "b_more_io: %8lu\n"
96 "bdi_list: %8u\n"
Christoph Hellwigc1955ce2010-06-19 23:08:06 +020097 "state: %8lx\n",
Miklos Szeredi76f14182008-04-30 00:54:36 -070098 (unsigned long) K(bdi_stat(bdi, BDI_WRITEBACK)),
99 (unsigned long) K(bdi_stat(bdi, BDI_RECLAIMABLE)),
Jens Axboef09b00d2009-05-25 09:08:21 +0200100 K(bdi_thresh), K(dirty_thresh),
Christoph Hellwigc1955ce2010-06-19 23:08:06 +0200101 K(background_thresh), nr_dirty, nr_io, nr_more_io,
102 !list_empty(&bdi->bdi_list), bdi->state);
Miklos Szeredi76f14182008-04-30 00:54:36 -0700103#undef K
104
105 return 0;
106}
107
108static int bdi_debug_stats_open(struct inode *inode, struct file *file)
109{
110 return single_open(file, bdi_debug_stats_show, inode->i_private);
111}
112
113static const struct file_operations bdi_debug_stats_fops = {
114 .open = bdi_debug_stats_open,
115 .read = seq_read,
116 .llseek = seq_lseek,
117 .release = single_release,
118};
119
120static void bdi_debug_register(struct backing_dev_info *bdi, const char *name)
121{
122 bdi->debug_dir = debugfs_create_dir(name, bdi_debug_root);
123 bdi->debug_stats = debugfs_create_file("stats", 0444, bdi->debug_dir,
124 bdi, &bdi_debug_stats_fops);
125}
126
127static void bdi_debug_unregister(struct backing_dev_info *bdi)
128{
129 debugfs_remove(bdi->debug_stats);
130 debugfs_remove(bdi->debug_dir);
131}
132#else
133static inline void bdi_debug_init(void)
134{
135}
136static inline void bdi_debug_register(struct backing_dev_info *bdi,
137 const char *name)
138{
139}
140static inline void bdi_debug_unregister(struct backing_dev_info *bdi)
141{
142}
143#endif
144
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700145static ssize_t read_ahead_kb_store(struct device *dev,
146 struct device_attribute *attr,
147 const char *buf, size_t count)
148{
149 struct backing_dev_info *bdi = dev_get_drvdata(dev);
150 char *end;
151 unsigned long read_ahead_kb;
152 ssize_t ret = -EINVAL;
153
154 read_ahead_kb = simple_strtoul(buf, &end, 10);
155 if (*buf && (end[0] == '\0' || (end[0] == '\n' && end[1] == '\0'))) {
156 bdi->ra_pages = read_ahead_kb >> (PAGE_SHIFT - 10);
157 ret = count;
158 }
159 return ret;
160}
161
162#define K(pages) ((pages) << (PAGE_SHIFT - 10))
163
164#define BDI_SHOW(name, expr) \
165static ssize_t name##_show(struct device *dev, \
166 struct device_attribute *attr, char *page) \
167{ \
168 struct backing_dev_info *bdi = dev_get_drvdata(dev); \
169 \
170 return snprintf(page, PAGE_SIZE-1, "%lld\n", (long long)expr); \
171}
172
173BDI_SHOW(read_ahead_kb, K(bdi->ra_pages))
174
Peter Zijlstra189d3c42008-04-30 00:54:35 -0700175static ssize_t min_ratio_store(struct device *dev,
176 struct device_attribute *attr, const char *buf, size_t count)
177{
178 struct backing_dev_info *bdi = dev_get_drvdata(dev);
179 char *end;
180 unsigned int ratio;
181 ssize_t ret = -EINVAL;
182
183 ratio = simple_strtoul(buf, &end, 10);
184 if (*buf && (end[0] == '\0' || (end[0] == '\n' && end[1] == '\0'))) {
185 ret = bdi_set_min_ratio(bdi, ratio);
186 if (!ret)
187 ret = count;
188 }
189 return ret;
190}
191BDI_SHOW(min_ratio, bdi->min_ratio)
192
Peter Zijlstraa42dde02008-04-30 00:54:36 -0700193static ssize_t max_ratio_store(struct device *dev,
194 struct device_attribute *attr, const char *buf, size_t count)
195{
196 struct backing_dev_info *bdi = dev_get_drvdata(dev);
197 char *end;
198 unsigned int ratio;
199 ssize_t ret = -EINVAL;
200
201 ratio = simple_strtoul(buf, &end, 10);
202 if (*buf && (end[0] == '\0' || (end[0] == '\n' && end[1] == '\0'))) {
203 ret = bdi_set_max_ratio(bdi, ratio);
204 if (!ret)
205 ret = count;
206 }
207 return ret;
208}
209BDI_SHOW(max_ratio, bdi->max_ratio)
210
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700211#define __ATTR_RW(attr) __ATTR(attr, 0644, attr##_show, attr##_store)
212
213static struct device_attribute bdi_dev_attrs[] = {
214 __ATTR_RW(read_ahead_kb),
Peter Zijlstra189d3c42008-04-30 00:54:35 -0700215 __ATTR_RW(min_ratio),
Peter Zijlstraa42dde02008-04-30 00:54:36 -0700216 __ATTR_RW(max_ratio),
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700217 __ATTR_NULL,
218};
219
220static __init int bdi_class_init(void)
221{
222 bdi_class = class_create(THIS_MODULE, "bdi");
Anton Blanchard14421452010-04-02 09:46:55 +0200223 if (IS_ERR(bdi_class))
224 return PTR_ERR(bdi_class);
225
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700226 bdi_class->dev_attrs = bdi_dev_attrs;
Miklos Szeredi76f14182008-04-30 00:54:36 -0700227 bdi_debug_init();
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700228 return 0;
229}
Miklos Szeredi76f14182008-04-30 00:54:36 -0700230postcore_initcall(bdi_class_init);
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700231
Jens Axboe26160152009-03-17 09:35:06 +0100232static int __init default_bdi_init(void)
233{
234 int err;
235
Jens Axboe03ba3782009-09-09 09:08:54 +0200236 sync_supers_tsk = kthread_run(bdi_sync_supers, NULL, "sync_supers");
237 BUG_ON(IS_ERR(sync_supers_tsk));
238
Jens Axboe03ba3782009-09-09 09:08:54 +0200239 setup_timer(&sync_supers_timer, sync_supers_timer_fn, 0);
Jens Axboe64231042010-05-21 20:00:35 +0200240 bdi_arm_supers_timer();
Jens Axboe03ba3782009-09-09 09:08:54 +0200241
Jens Axboe26160152009-03-17 09:35:06 +0100242 err = bdi_init(&default_backing_dev_info);
243 if (!err)
244 bdi_register(&default_backing_dev_info, NULL, "default");
245
246 return err;
247}
248subsys_initcall(default_bdi_init);
249
Jens Axboe03ba3782009-09-09 09:08:54 +0200250int bdi_has_dirty_io(struct backing_dev_info *bdi)
251{
252 return wb_has_dirty_io(&bdi->wb);
253}
254
255static void bdi_flush_io(struct backing_dev_info *bdi)
256{
257 struct writeback_control wbc = {
Jens Axboe03ba3782009-09-09 09:08:54 +0200258 .sync_mode = WB_SYNC_NONE,
259 .older_than_this = NULL,
260 .range_cyclic = 1,
261 .nr_to_write = 1024,
262 };
263
Christoph Hellwig9c3a8ee2010-06-10 12:07:27 +0200264 writeback_inodes_wb(&bdi->wb, &wbc);
Jens Axboe03ba3782009-09-09 09:08:54 +0200265}
266
267/*
Artem Bityutskiy6f904ff2010-07-25 14:29:11 +0300268 * kupdated() used to do this. We cannot do it from the bdi_forker_thread()
Jens Axboe03ba3782009-09-09 09:08:54 +0200269 * or we risk deadlocking on ->s_umount. The longer term solution would be
270 * to implement sync_supers_bdi() or similar and simply do it from the
Artem Bityutskiy6f904ff2010-07-25 14:29:11 +0300271 * bdi writeback thread individually.
Jens Axboe03ba3782009-09-09 09:08:54 +0200272 */
273static int bdi_sync_supers(void *unused)
274{
275 set_user_nice(current, 0);
276
277 while (!kthread_should_stop()) {
278 set_current_state(TASK_INTERRUPTIBLE);
279 schedule();
280
281 /*
282 * Do this periodically, like kupdated() did before.
283 */
284 sync_supers();
285 }
286
287 return 0;
288}
289
Jens Axboe64231042010-05-21 20:00:35 +0200290void bdi_arm_supers_timer(void)
Jens Axboe03ba3782009-09-09 09:08:54 +0200291{
292 unsigned long next;
293
Jens Axboe64231042010-05-21 20:00:35 +0200294 if (!dirty_writeback_interval)
295 return;
296
Jens Axboe03ba3782009-09-09 09:08:54 +0200297 next = msecs_to_jiffies(dirty_writeback_interval * 10) + jiffies;
298 mod_timer(&sync_supers_timer, round_jiffies_up(next));
299}
300
301static void sync_supers_timer_fn(unsigned long unused)
302{
303 wake_up_process(sync_supers_tsk);
Jens Axboe64231042010-05-21 20:00:35 +0200304 bdi_arm_supers_timer();
Jens Axboe03ba3782009-09-09 09:08:54 +0200305}
306
Artem Bityutskiy64677162010-07-25 14:29:22 +0300307static void wakeup_timer_fn(unsigned long data)
308{
309 struct backing_dev_info *bdi = (struct backing_dev_info *)data;
310
311 spin_lock_bh(&bdi->wb_lock);
312 if (bdi->wb.task) {
313 wake_up_process(bdi->wb.task);
314 } else {
315 /*
316 * When bdi tasks are inactive for long time, they are killed.
317 * In this case we have to wake-up the forker thread which
318 * should create and run the bdi thread.
319 */
320 wake_up_process(default_backing_dev_info.wb.task);
321 }
322 spin_unlock_bh(&bdi->wb_lock);
323}
324
325/*
326 * This function is used when the first inode for this bdi is marked dirty. It
327 * wakes-up the corresponding bdi thread which should then take care of the
328 * periodic background write-out of dirty inodes. Since the write-out would
329 * starts only 'dirty_writeback_interval' centisecs from now anyway, we just
330 * set up a timer which wakes the bdi thread up later.
331 *
332 * Note, we wouldn't bother setting up the timer, but this function is on the
333 * fast-path (used by '__mark_inode_dirty()'), so we save few context switches
334 * by delaying the wake-up.
335 */
336void bdi_wakeup_thread_delayed(struct backing_dev_info *bdi)
337{
338 unsigned long timeout;
339
340 timeout = msecs_to_jiffies(dirty_writeback_interval * 10);
341 mod_timer(&bdi->wb.wakeup_timer, jiffies + timeout);
342}
343
Artem Bityutskiyfff5b852010-07-25 14:29:20 +0300344/*
345 * Calculate the longest interval (jiffies) bdi threads are allowed to be
346 * inactive.
347 */
348static unsigned long bdi_longest_inactive(void)
349{
350 unsigned long interval;
351
352 interval = msecs_to_jiffies(dirty_writeback_interval * 10);
353 return max(5UL * 60 * HZ, interval);
354}
355
Artem Bityutskiy6f904ff2010-07-25 14:29:11 +0300356static int bdi_forker_thread(void *ptr)
Jens Axboe03ba3782009-09-09 09:08:54 +0200357{
358 struct bdi_writeback *me = ptr;
359
Christoph Hellwigc1955ce2010-06-19 23:08:06 +0200360 current->flags |= PF_FLUSHER | PF_SWAPWRITE;
361 set_freezable();
362
363 /*
364 * Our parent may run at a different priority, just set us to normal
365 */
366 set_user_nice(current, 0);
Jens Axboe03ba3782009-09-09 09:08:54 +0200367
368 for (;;) {
Artem Bityutskiyfff5b852010-07-25 14:29:20 +0300369 struct task_struct *task = NULL;
Artem Bityutskiy78c40cb2010-07-25 14:29:17 +0300370 struct backing_dev_info *bdi;
Artem Bityutskiyadf39242010-07-25 14:29:19 +0300371 enum {
372 NO_ACTION, /* Nothing to do */
373 FORK_THREAD, /* Fork bdi thread */
Artem Bityutskiyfff5b852010-07-25 14:29:20 +0300374 KILL_THREAD, /* Kill inactive bdi thread */
Artem Bityutskiyadf39242010-07-25 14:29:19 +0300375 } action = NO_ACTION;
Jens Axboe03ba3782009-09-09 09:08:54 +0200376
377 /*
378 * Temporary measure, we want to make sure we don't see
379 * dirty data on the default backing_dev_info
380 */
Artem Bityutskiy64677162010-07-25 14:29:22 +0300381 if (wb_has_dirty_io(me) || !list_empty(&me->bdi->work_list)) {
382 del_timer(&me->wakeup_timer);
Jens Axboe03ba3782009-09-09 09:08:54 +0200383 wb_do_writeback(me, 0);
Artem Bityutskiy64677162010-07-25 14:29:22 +0300384 }
Jens Axboe03ba3782009-09-09 09:08:54 +0200385
Jens Axboecfc4ba52009-09-14 13:12:40 +0200386 spin_lock_bh(&bdi_lock);
Artem Bityutskiyc5f7ad22010-07-25 14:29:13 +0300387 set_current_state(TASK_INTERRUPTIBLE);
Jens Axboe03ba3782009-09-09 09:08:54 +0200388
Artem Bityutskiy78c40cb2010-07-25 14:29:17 +0300389 list_for_each_entry(bdi, &bdi_list, bdi_list) {
Artem Bityutskiyadf39242010-07-25 14:29:19 +0300390 bool have_dirty_io;
391
392 if (!bdi_cap_writeback_dirty(bdi) ||
393 bdi_cap_flush_forker(bdi))
Jens Axboe03ba3782009-09-09 09:08:54 +0200394 continue;
395
Artem Bityutskiy080dcec2010-07-25 14:29:16 +0300396 WARN(!test_bit(BDI_registered, &bdi->state),
397 "bdi %p/%s is not registered!\n", bdi, bdi->name);
398
Artem Bityutskiyadf39242010-07-25 14:29:19 +0300399 have_dirty_io = !list_empty(&bdi->work_list) ||
400 wb_has_dirty_io(&bdi->wb);
Artem Bityutskiy78c40cb2010-07-25 14:29:17 +0300401
402 /*
Artem Bityutskiyadf39242010-07-25 14:29:19 +0300403 * If the bdi has work to do, but the thread does not
404 * exist - create it.
Artem Bityutskiy78c40cb2010-07-25 14:29:17 +0300405 */
Artem Bityutskiyadf39242010-07-25 14:29:19 +0300406 if (!bdi->wb.task && have_dirty_io) {
407 /*
408 * Set the pending bit - if someone will try to
409 * unregister this bdi - it'll wait on this bit.
410 */
411 set_bit(BDI_pending, &bdi->state);
412 action = FORK_THREAD;
413 break;
414 }
Artem Bityutskiyfff5b852010-07-25 14:29:20 +0300415
Artem Bityutskiy64677162010-07-25 14:29:22 +0300416 spin_lock_bh(&bdi->wb_lock);
Artem Bityutskiyfff5b852010-07-25 14:29:20 +0300417 /*
418 * If there is no work to do and the bdi thread was
419 * inactive long enough - kill it. The wb_lock is taken
420 * to make sure no-one adds more work to this bdi and
421 * wakes the bdi thread up.
422 */
423 if (bdi->wb.task && !have_dirty_io &&
424 time_after(jiffies, bdi->wb.last_active +
425 bdi_longest_inactive())) {
426 task = bdi->wb.task;
427 bdi->wb.task = NULL;
428 spin_unlock(&bdi->wb_lock);
429 set_bit(BDI_pending, &bdi->state);
430 action = KILL_THREAD;
431 break;
432 }
Artem Bityutskiy64677162010-07-25 14:29:22 +0300433 spin_unlock_bh(&bdi->wb_lock);
Jens Axboe03ba3782009-09-09 09:08:54 +0200434 }
Artem Bityutskiy080dcec2010-07-25 14:29:16 +0300435 spin_unlock_bh(&bdi_lock);
Jens Axboe03ba3782009-09-09 09:08:54 +0200436
Artem Bityutskiyc4ec7902010-07-25 14:29:14 +0300437 /* Keep working if default bdi still has things to do */
438 if (!list_empty(&me->bdi->work_list))
439 __set_current_state(TASK_RUNNING);
440
Artem Bityutskiyadf39242010-07-25 14:29:19 +0300441 switch (action) {
442 case FORK_THREAD:
443 __set_current_state(TASK_RUNNING);
444 task = kthread_run(bdi_writeback_thread, &bdi->wb, "flush-%s",
445 dev_name(bdi->dev));
446 if (IS_ERR(task)) {
447 /*
448 * If thread creation fails, force writeout of
449 * the bdi from the thread.
450 */
451 bdi_flush_io(bdi);
Artem Bityutskiyfff5b852010-07-25 14:29:20 +0300452 } else {
453 /*
454 * The spinlock makes sure we do not lose
455 * wake-ups when racing with 'bdi_queue_work()'.
456 */
Artem Bityutskiy64677162010-07-25 14:29:22 +0300457 spin_lock_bh(&bdi->wb_lock);
Artem Bityutskiyadf39242010-07-25 14:29:19 +0300458 bdi->wb.task = task;
Artem Bityutskiy64677162010-07-25 14:29:22 +0300459 spin_unlock_bh(&bdi->wb_lock);
Artem Bityutskiyfff5b852010-07-25 14:29:20 +0300460 }
461 break;
462
463 case KILL_THREAD:
464 __set_current_state(TASK_RUNNING);
465 kthread_stop(task);
Artem Bityutskiyadf39242010-07-25 14:29:19 +0300466 break;
Jens Axboe03ba3782009-09-09 09:08:54 +0200467
Artem Bityutskiyadf39242010-07-25 14:29:19 +0300468 case NO_ACTION:
Artem Bityutskiy253c34e2010-07-25 14:29:21 +0300469 if (!wb_has_dirty_io(me) || !dirty_writeback_interval)
470 /*
471 * There are no dirty data. The only thing we
472 * should now care about is checking for
473 * inactive bdi threads and killing them. Thus,
474 * let's sleep for longer time, save energy and
475 * be friendly for battery-driven devices.
476 */
477 schedule_timeout(bdi_longest_inactive());
Jens Axboe64231042010-05-21 20:00:35 +0200478 else
Artem Bityutskiy253c34e2010-07-25 14:29:21 +0300479 schedule_timeout(msecs_to_jiffies(dirty_writeback_interval * 10));
Jens Axboe03ba3782009-09-09 09:08:54 +0200480 try_to_freeze();
Artem Bityutskiyadf39242010-07-25 14:29:19 +0300481 /* Back to the main loop */
Jens Axboe03ba3782009-09-09 09:08:54 +0200482 continue;
483 }
Artem Bityutskiyfff5b852010-07-25 14:29:20 +0300484
485 /*
486 * Clear pending bit and wakeup anybody waiting to tear us down.
487 */
488 clear_bit(BDI_pending, &bdi->state);
489 smp_mb__after_clear_bit();
490 wake_up_bit(&bdi->state, BDI_pending);
Jens Axboe03ba3782009-09-09 09:08:54 +0200491 }
492
493 return 0;
494}
495
Jens Axboecfc4ba52009-09-14 13:12:40 +0200496/*
497 * Remove bdi from bdi_list, and ensure that it is no longer visible
498 */
499static void bdi_remove_from_list(struct backing_dev_info *bdi)
500{
501 spin_lock_bh(&bdi_lock);
502 list_del_rcu(&bdi->bdi_list);
503 spin_unlock_bh(&bdi_lock);
504
505 synchronize_rcu();
506}
507
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700508int bdi_register(struct backing_dev_info *bdi, struct device *parent,
509 const char *fmt, ...)
510{
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700511 va_list args;
512 int ret = 0;
513 struct device *dev;
514
Andrew Morton69fc2082008-12-09 13:14:06 -0800515 if (bdi->dev) /* The driver needs to use separate queues per device */
Kay Sieversf1d0b062008-12-02 10:31:50 -0800516 goto exit;
517
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700518 va_start(args, fmt);
Greg Kroah-Hartman19051c52008-05-15 13:44:08 -0700519 dev = device_create_vargs(bdi_class, parent, MKDEV(0, 0), bdi, fmt, args);
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700520 va_end(args);
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700521 if (IS_ERR(dev)) {
522 ret = PTR_ERR(dev);
523 goto exit;
524 }
525
Jens Axboecfc4ba52009-09-14 13:12:40 +0200526 spin_lock_bh(&bdi_lock);
527 list_add_tail_rcu(&bdi->bdi_list, &bdi_list);
528 spin_unlock_bh(&bdi_lock);
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200529
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700530 bdi->dev = dev;
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700531
Jens Axboe03ba3782009-09-09 09:08:54 +0200532 /*
533 * Just start the forker thread for our default backing_dev_info,
534 * and add other bdi's to the list. They will get a thread created
535 * on-demand when they need it.
536 */
537 if (bdi_cap_flush_forker(bdi)) {
538 struct bdi_writeback *wb = &bdi->wb;
539
Artem Bityutskiy6f904ff2010-07-25 14:29:11 +0300540 wb->task = kthread_run(bdi_forker_thread, wb, "bdi-%s",
Jens Axboe03ba3782009-09-09 09:08:54 +0200541 dev_name(dev));
542 if (IS_ERR(wb->task)) {
543 wb->task = NULL;
544 ret = -ENOMEM;
545
Jens Axboecfc4ba52009-09-14 13:12:40 +0200546 bdi_remove_from_list(bdi);
Jens Axboe03ba3782009-09-09 09:08:54 +0200547 goto exit;
548 }
549 }
550
551 bdi_debug_register(bdi, dev_name(dev));
Jens Axboe500b0672009-09-09 09:10:25 +0200552 set_bit(BDI_registered, &bdi->state);
Dave Chinner455b2862010-07-07 13:24:06 +1000553 trace_writeback_bdi_register(bdi);
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700554exit:
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700555 return ret;
556}
557EXPORT_SYMBOL(bdi_register);
558
559int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev)
560{
561 return bdi_register(bdi, NULL, "%u:%u", MAJOR(dev), MINOR(dev));
562}
563EXPORT_SYMBOL(bdi_register_dev);
564
Jens Axboe03ba3782009-09-09 09:08:54 +0200565/*
566 * Remove bdi from the global list and shutdown any threads we have running
567 */
568static void bdi_wb_shutdown(struct backing_dev_info *bdi)
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200569{
Jens Axboe03ba3782009-09-09 09:08:54 +0200570 if (!bdi_cap_writeback_dirty(bdi))
571 return;
572
573 /*
Artem Bityutskiyfff5b852010-07-25 14:29:20 +0300574 * Make sure nobody finds us on the bdi_list anymore
575 */
576 bdi_remove_from_list(bdi);
577
578 /*
Jens Axboe03ba3782009-09-09 09:08:54 +0200579 * If setup is pending, wait for that to complete first
580 */
581 wait_on_bit(&bdi->state, BDI_pending, bdi_sched_wait,
582 TASK_UNINTERRUPTIBLE);
583
584 /*
Christoph Hellwigc1955ce2010-06-19 23:08:06 +0200585 * Finally, kill the kernel thread. We don't need to be RCU
Romit Dasguptac62b17a2009-11-12 13:08:11 +0100586 * safe anymore, since the bdi is gone from visibility. Force
587 * unfreeze of the thread before calling kthread_stop(), otherwise
588 * it would never exet if it is currently stuck in the refrigerator.
Jens Axboe03ba3782009-09-09 09:08:54 +0200589 */
Christoph Hellwigc1955ce2010-06-19 23:08:06 +0200590 if (bdi->wb.task) {
591 thaw_process(bdi->wb.task);
592 kthread_stop(bdi->wb.task);
Romit Dasguptac62b17a2009-11-12 13:08:11 +0100593 }
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200594}
595
Jens Axboe592b09a2009-10-29 11:46:12 +0100596/*
597 * This bdi is going away now, make sure that no super_blocks point to it
598 */
599static void bdi_prune_sb(struct backing_dev_info *bdi)
600{
601 struct super_block *sb;
602
603 spin_lock(&sb_lock);
604 list_for_each_entry(sb, &super_blocks, s_list) {
605 if (sb->s_bdi == bdi)
606 sb->s_bdi = NULL;
607 }
608 spin_unlock(&sb_lock);
609}
610
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700611void bdi_unregister(struct backing_dev_info *bdi)
612{
613 if (bdi->dev) {
Dave Chinner455b2862010-07-07 13:24:06 +1000614 trace_writeback_bdi_unregister(bdi);
Jens Axboe8c4db332009-11-03 20:18:44 +0100615 bdi_prune_sb(bdi);
Artem Bityutskiy64677162010-07-25 14:29:22 +0300616 del_timer_sync(&bdi->wb.wakeup_timer);
Jens Axboe8c4db332009-11-03 20:18:44 +0100617
Jens Axboe03ba3782009-09-09 09:08:54 +0200618 if (!bdi_cap_flush_forker(bdi))
619 bdi_wb_shutdown(bdi);
Miklos Szeredi76f14182008-04-30 00:54:36 -0700620 bdi_debug_unregister(bdi);
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700621 device_unregister(bdi->dev);
622 bdi->dev = NULL;
623 }
624}
625EXPORT_SYMBOL(bdi_unregister);
Andrew Morton3fcfab12006-10-19 23:28:16 -0700626
Artem Bityutskiy64677162010-07-25 14:29:22 +0300627static void bdi_wb_init(struct bdi_writeback *wb, struct backing_dev_info *bdi)
628{
629 memset(wb, 0, sizeof(*wb));
630
631 wb->bdi = bdi;
632 wb->last_old_flush = jiffies;
633 INIT_LIST_HEAD(&wb->b_dirty);
634 INIT_LIST_HEAD(&wb->b_io);
635 INIT_LIST_HEAD(&wb->b_more_io);
636 setup_timer(&wb->wakeup_timer, wakeup_timer_fn, (unsigned long)bdi);
637}
638
Peter Zijlstrab2e8fb62007-10-16 23:25:47 -0700639int bdi_init(struct backing_dev_info *bdi)
640{
Jens Axboe03ba3782009-09-09 09:08:54 +0200641 int i, err;
Peter Zijlstrab2e8fb62007-10-16 23:25:47 -0700642
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700643 bdi->dev = NULL;
644
Peter Zijlstra189d3c42008-04-30 00:54:35 -0700645 bdi->min_ratio = 0;
Peter Zijlstraa42dde02008-04-30 00:54:36 -0700646 bdi->max_ratio = 100;
647 bdi->max_prop_frac = PROP_FRAC_BASE;
Jens Axboe03ba3782009-09-09 09:08:54 +0200648 spin_lock_init(&bdi->wb_lock);
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200649 INIT_LIST_HEAD(&bdi->bdi_list);
Jens Axboe03ba3782009-09-09 09:08:54 +0200650 INIT_LIST_HEAD(&bdi->work_list);
651
652 bdi_wb_init(&bdi->wb, bdi);
653
Peter Zijlstrab2e8fb62007-10-16 23:25:47 -0700654 for (i = 0; i < NR_BDI_STAT_ITEMS; i++) {
Peter Zijlstraea319512008-12-26 15:08:55 +0100655 err = percpu_counter_init(&bdi->bdi_stat[i], 0);
Peter Zijlstra04fbfdc2007-10-16 23:25:50 -0700656 if (err)
657 goto err;
658 }
659
660 bdi->dirty_exceeded = 0;
661 err = prop_local_init_percpu(&bdi->completions);
662
663 if (err) {
664err:
Denis Cheng4b01a0b2007-12-04 23:45:07 -0800665 while (i--)
Peter Zijlstra04fbfdc2007-10-16 23:25:50 -0700666 percpu_counter_destroy(&bdi->bdi_stat[i]);
Peter Zijlstrab2e8fb62007-10-16 23:25:47 -0700667 }
668
669 return err;
670}
671EXPORT_SYMBOL(bdi_init);
672
673void bdi_destroy(struct backing_dev_info *bdi)
674{
675 int i;
676
Jens Axboece5f8e72009-09-14 12:57:56 +0200677 /*
678 * Splice our entries to the default_backing_dev_info, if this
679 * bdi disappears
680 */
681 if (bdi_has_dirty_io(bdi)) {
682 struct bdi_writeback *dst = &default_backing_dev_info.wb;
683
684 spin_lock(&inode_lock);
685 list_splice(&bdi->wb.b_dirty, &dst->b_dirty);
686 list_splice(&bdi->wb.b_io, &dst->b_io);
687 list_splice(&bdi->wb.b_more_io, &dst->b_more_io);
688 spin_unlock(&inode_lock);
689 }
Jens Axboe66f3b8e2009-09-02 09:19:46 +0200690
Peter Zijlstracf0ca9f2008-04-30 00:54:32 -0700691 bdi_unregister(bdi);
692
Peter Zijlstrab2e8fb62007-10-16 23:25:47 -0700693 for (i = 0; i < NR_BDI_STAT_ITEMS; i++)
694 percpu_counter_destroy(&bdi->bdi_stat[i]);
Peter Zijlstra04fbfdc2007-10-16 23:25:50 -0700695
696 prop_local_destroy_percpu(&bdi->completions);
Peter Zijlstrab2e8fb62007-10-16 23:25:47 -0700697}
698EXPORT_SYMBOL(bdi_destroy);
699
Jens Axboec3c53202010-04-22 11:37:01 +0200700/*
701 * For use from filesystems to quickly init and register a bdi associated
702 * with dirty writeback
703 */
704int bdi_setup_and_register(struct backing_dev_info *bdi, char *name,
705 unsigned int cap)
706{
707 char tmp[32];
708 int err;
709
710 bdi->name = name;
711 bdi->capabilities = cap;
712 err = bdi_init(bdi);
713 if (err)
714 return err;
715
716 sprintf(tmp, "%.28s%s", name, "-%d");
717 err = bdi_register(bdi, NULL, tmp, atomic_long_inc_return(&bdi_seq));
718 if (err) {
719 bdi_destroy(bdi);
720 return err;
721 }
722
723 return 0;
724}
725EXPORT_SYMBOL(bdi_setup_and_register);
726
Andrew Morton3fcfab12006-10-19 23:28:16 -0700727static wait_queue_head_t congestion_wqh[2] = {
728 __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[0]),
729 __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[1])
730 };
731
Jens Axboe1faa16d2009-04-06 14:48:01 +0200732void clear_bdi_congested(struct backing_dev_info *bdi, int sync)
Andrew Morton3fcfab12006-10-19 23:28:16 -0700733{
734 enum bdi_state bit;
Jens Axboe1faa16d2009-04-06 14:48:01 +0200735 wait_queue_head_t *wqh = &congestion_wqh[sync];
Andrew Morton3fcfab12006-10-19 23:28:16 -0700736
Jens Axboe1faa16d2009-04-06 14:48:01 +0200737 bit = sync ? BDI_sync_congested : BDI_async_congested;
Andrew Morton3fcfab12006-10-19 23:28:16 -0700738 clear_bit(bit, &bdi->state);
739 smp_mb__after_clear_bit();
740 if (waitqueue_active(wqh))
741 wake_up(wqh);
742}
743EXPORT_SYMBOL(clear_bdi_congested);
744
Jens Axboe1faa16d2009-04-06 14:48:01 +0200745void set_bdi_congested(struct backing_dev_info *bdi, int sync)
Andrew Morton3fcfab12006-10-19 23:28:16 -0700746{
747 enum bdi_state bit;
748
Jens Axboe1faa16d2009-04-06 14:48:01 +0200749 bit = sync ? BDI_sync_congested : BDI_async_congested;
Andrew Morton3fcfab12006-10-19 23:28:16 -0700750 set_bit(bit, &bdi->state);
751}
752EXPORT_SYMBOL(set_bdi_congested);
753
754/**
755 * congestion_wait - wait for a backing_dev to become uncongested
Jens Axboe8aa7e842009-07-09 14:52:32 +0200756 * @sync: SYNC or ASYNC IO
Andrew Morton3fcfab12006-10-19 23:28:16 -0700757 * @timeout: timeout in jiffies
758 *
759 * Waits for up to @timeout jiffies for a backing_dev (any backing_dev) to exit
760 * write congestion. If no backing_devs are congested then just wait for the
761 * next write to be completed.
762 */
Jens Axboe8aa7e842009-07-09 14:52:32 +0200763long congestion_wait(int sync, long timeout)
Andrew Morton3fcfab12006-10-19 23:28:16 -0700764{
765 long ret;
766 DEFINE_WAIT(wait);
Jens Axboe8aa7e842009-07-09 14:52:32 +0200767 wait_queue_head_t *wqh = &congestion_wqh[sync];
Andrew Morton3fcfab12006-10-19 23:28:16 -0700768
769 prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
770 ret = io_schedule_timeout(timeout);
771 finish_wait(wqh, &wait);
772 return ret;
773}
774EXPORT_SYMBOL(congestion_wait);
Peter Zijlstra04fbfdc2007-10-16 23:25:50 -0700775