blob: 9a08f621b27d49a497cbe2b62d678d3d212d122f [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * raid10.c : Multiple Devices driver for Linux
3 *
4 * Copyright (C) 2000-2004 Neil Brown
5 *
6 * RAID-10 support for md.
7 *
Lucas De Marchi25985ed2011-03-30 22:57:33 -03008 * Base on code in raid1.c. See raid1.c for further copyright information.
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 *
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2, or (at your option)
14 * any later version.
15 *
16 * You should have received a copy of the GNU General Public License
17 * (for example /usr/src/linux/COPYING); if not, write to the Free
18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 */
20
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090021#include <linux/slab.h>
Stephen Rothwell25570722008-10-15 09:09:21 +110022#include <linux/delay.h>
NeilBrownbff61972009-03-31 14:33:13 +110023#include <linux/blkdev.h>
Paul Gortmaker056075c2011-07-03 13:58:33 -040024#include <linux/module.h>
NeilBrownbff61972009-03-31 14:33:13 +110025#include <linux/seq_file.h>
Christian Dietrich8bda4702011-07-27 11:00:36 +100026#include <linux/ratelimit.h>
NeilBrown3ea7daa2012-05-22 13:53:47 +100027#include <linux/kthread.h>
NeilBrown43b2e5d2009-03-31 14:33:13 +110028#include "md.h"
Christoph Hellwigef740c32009-03-31 14:27:03 +110029#include "raid10.h"
Trela, Maciejdab8b292010-03-08 16:02:45 +110030#include "raid0.h"
Christoph Hellwigef740c32009-03-31 14:27:03 +110031#include "bitmap.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070032
33/*
34 * RAID10 provides a combination of RAID0 and RAID1 functionality.
35 * The layout of data is defined by
36 * chunk_size
37 * raid_disks
38 * near_copies (stored in low byte of layout)
39 * far_copies (stored in second byte of layout)
NeilBrownc93983b2006-06-26 00:27:41 -070040 * far_offset (stored in bit 16 of layout )
Linus Torvalds1da177e2005-04-16 15:20:36 -070041 *
42 * The data to be stored is divided into chunks using chunksize.
43 * Each device is divided into far_copies sections.
44 * In each section, chunks are laid out in a style similar to raid0, but
45 * near_copies copies of each chunk is stored (each on a different drive).
46 * The starting device for each section is offset near_copies from the starting
47 * device of the previous section.
NeilBrownc93983b2006-06-26 00:27:41 -070048 * Thus they are (near_copies*far_copies) of each chunk, and each is on a different
Linus Torvalds1da177e2005-04-16 15:20:36 -070049 * drive.
50 * near_copies and far_copies must be at least one, and their product is at most
51 * raid_disks.
NeilBrownc93983b2006-06-26 00:27:41 -070052 *
53 * If far_offset is true, then the far_copies are handled a bit differently.
54 * The copies are still in different stripes, but instead of be very far apart
55 * on disk, there are adjacent stripes.
Linus Torvalds1da177e2005-04-16 15:20:36 -070056 */
57
58/*
59 * Number of guaranteed r10bios in case of extreme VM load:
60 */
61#define NR_RAID10_BIOS 256
62
Jonathan Brassow473e87c2012-07-31 10:03:52 +100063/* when we get a read error on a read-only array, we redirect to another
64 * device without failing the first device, or trying to over-write to
65 * correct the read error. To keep track of bad blocks on a per-bio
66 * level, we store IO_BLOCKED in the appropriate 'bios' pointer
67 */
68#define IO_BLOCKED ((struct bio *)1)
69/* When we successfully write to a known bad-block, we need to remove the
70 * bad-block marking which must be done from process context. So we record
71 * the success by setting devs[n].bio to IO_MADE_GOOD
72 */
73#define IO_MADE_GOOD ((struct bio *)2)
74
75#define BIO_SPECIAL(bio) ((unsigned long)bio <= 2)
76
77/* When there are this many requests queued to be written by
NeilBrown34db0cd2011-10-11 16:50:01 +110078 * the raid10 thread, we become 'congested' to provide back-pressure
79 * for writeback.
80 */
81static int max_queued_requests = 1024;
82
NeilBrowne879a872011-10-11 16:49:02 +110083static void allow_barrier(struct r10conf *conf);
84static void lower_barrier(struct r10conf *conf);
NeilBrownfae8cc5e2012-02-14 11:10:10 +110085static int enough(struct r10conf *conf, int ignore);
NeilBrown3ea7daa2012-05-22 13:53:47 +100086static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
87 int *skipped);
88static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio);
89static void end_reshape_write(struct bio *bio, int error);
90static void end_reshape(struct r10conf *conf);
NeilBrown0a27ec92006-01-06 00:20:13 -080091
Al Virodd0fc662005-10-07 07:46:04 +010092static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -070093{
NeilBrowne879a872011-10-11 16:49:02 +110094 struct r10conf *conf = data;
NeilBrown9f2c9d12011-10-11 16:48:43 +110095 int size = offsetof(struct r10bio, devs[conf->copies]);
Linus Torvalds1da177e2005-04-16 15:20:36 -070096
NeilBrown69335ef2011-12-23 10:17:54 +110097 /* allocate a r10bio with room for raid_disks entries in the
98 * bios array */
Jens Axboe7eaceac2011-03-10 08:52:07 +010099 return kzalloc(size, gfp_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100}
101
102static void r10bio_pool_free(void *r10_bio, void *data)
103{
104 kfree(r10_bio);
105}
106
NeilBrown0310fa22008-08-05 15:54:14 +1000107/* Maximum size of each resync request */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108#define RESYNC_BLOCK_SIZE (64*1024)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109#define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
NeilBrown0310fa22008-08-05 15:54:14 +1000110/* amount of memory to reserve for resync requests */
111#define RESYNC_WINDOW (1024*1024)
112/* maximum number of concurrent requests, memory permitting */
113#define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114
115/*
116 * When performing a resync, we need to read and compare, so
117 * we need as many pages are there are copies.
118 * When performing a recovery, we need 2 bios, one for read,
119 * one for write (we recover only one drive per r10buf)
120 *
121 */
Al Virodd0fc662005-10-07 07:46:04 +0100122static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700123{
NeilBrowne879a872011-10-11 16:49:02 +1100124 struct r10conf *conf = data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700125 struct page *page;
NeilBrown9f2c9d12011-10-11 16:48:43 +1100126 struct r10bio *r10_bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127 struct bio *bio;
128 int i, j;
129 int nalloc;
130
131 r10_bio = r10bio_pool_alloc(gfp_flags, conf);
Jens Axboe7eaceac2011-03-10 08:52:07 +0100132 if (!r10_bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134
NeilBrown3ea7daa2012-05-22 13:53:47 +1000135 if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||
136 test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137 nalloc = conf->copies; /* resync */
138 else
139 nalloc = 2; /* recovery */
140
141 /*
142 * Allocate bios.
143 */
144 for (j = nalloc ; j-- ; ) {
NeilBrown67465572010-10-26 17:33:54 +1100145 bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146 if (!bio)
147 goto out_free_bio;
148 r10_bio->devs[j].bio = bio;
NeilBrown69335ef2011-12-23 10:17:54 +1100149 if (!conf->have_replacement)
150 continue;
151 bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
152 if (!bio)
153 goto out_free_bio;
154 r10_bio->devs[j].repl_bio = bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155 }
156 /*
157 * Allocate RESYNC_PAGES data pages and attach them
158 * where needed.
159 */
160 for (j = 0 ; j < nalloc; j++) {
NeilBrown69335ef2011-12-23 10:17:54 +1100161 struct bio *rbio = r10_bio->devs[j].repl_bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162 bio = r10_bio->devs[j].bio;
163 for (i = 0; i < RESYNC_PAGES; i++) {
NeilBrown3ea7daa2012-05-22 13:53:47 +1000164 if (j > 0 && !test_bit(MD_RECOVERY_SYNC,
165 &conf->mddev->recovery)) {
166 /* we can share bv_page's during recovery
167 * and reshape */
Namhyung Kimc65060a2011-07-18 17:38:49 +1000168 struct bio *rbio = r10_bio->devs[0].bio;
169 page = rbio->bi_io_vec[i].bv_page;
170 get_page(page);
171 } else
172 page = alloc_page(gfp_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173 if (unlikely(!page))
174 goto out_free_pages;
175
176 bio->bi_io_vec[i].bv_page = page;
NeilBrown69335ef2011-12-23 10:17:54 +1100177 if (rbio)
178 rbio->bi_io_vec[i].bv_page = page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179 }
180 }
181
182 return r10_bio;
183
184out_free_pages:
185 for ( ; i > 0 ; i--)
NeilBrown1345b1d2006-01-06 00:20:40 -0800186 safe_put_page(bio->bi_io_vec[i-1].bv_page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187 while (j--)
188 for (i = 0; i < RESYNC_PAGES ; i++)
NeilBrown1345b1d2006-01-06 00:20:40 -0800189 safe_put_page(r10_bio->devs[j].bio->bi_io_vec[i].bv_page);
majianpeng5fdd2cf2012-05-22 13:55:03 +1000190 j = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191out_free_bio:
majianpeng5fdd2cf2012-05-22 13:55:03 +1000192 for ( ; j < nalloc; j++) {
193 if (r10_bio->devs[j].bio)
194 bio_put(r10_bio->devs[j].bio);
NeilBrown69335ef2011-12-23 10:17:54 +1100195 if (r10_bio->devs[j].repl_bio)
196 bio_put(r10_bio->devs[j].repl_bio);
197 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198 r10bio_pool_free(r10_bio, conf);
199 return NULL;
200}
201
202static void r10buf_pool_free(void *__r10_bio, void *data)
203{
204 int i;
NeilBrowne879a872011-10-11 16:49:02 +1100205 struct r10conf *conf = data;
NeilBrown9f2c9d12011-10-11 16:48:43 +1100206 struct r10bio *r10bio = __r10_bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207 int j;
208
209 for (j=0; j < conf->copies; j++) {
210 struct bio *bio = r10bio->devs[j].bio;
211 if (bio) {
212 for (i = 0; i < RESYNC_PAGES; i++) {
NeilBrown1345b1d2006-01-06 00:20:40 -0800213 safe_put_page(bio->bi_io_vec[i].bv_page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214 bio->bi_io_vec[i].bv_page = NULL;
215 }
216 bio_put(bio);
217 }
NeilBrown69335ef2011-12-23 10:17:54 +1100218 bio = r10bio->devs[j].repl_bio;
219 if (bio)
220 bio_put(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221 }
222 r10bio_pool_free(r10bio, conf);
223}
224
NeilBrowne879a872011-10-11 16:49:02 +1100225static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700226{
227 int i;
228
229 for (i = 0; i < conf->copies; i++) {
230 struct bio **bio = & r10_bio->devs[i].bio;
NeilBrown749c55e2011-07-28 11:39:24 +1000231 if (!BIO_SPECIAL(*bio))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232 bio_put(*bio);
233 *bio = NULL;
NeilBrown69335ef2011-12-23 10:17:54 +1100234 bio = &r10_bio->devs[i].repl_bio;
235 if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio))
236 bio_put(*bio);
237 *bio = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238 }
239}
240
NeilBrown9f2c9d12011-10-11 16:48:43 +1100241static void free_r10bio(struct r10bio *r10_bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242{
NeilBrowne879a872011-10-11 16:49:02 +1100243 struct r10conf *conf = r10_bio->mddev->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245 put_all_bios(conf, r10_bio);
246 mempool_free(r10_bio, conf->r10bio_pool);
247}
248
NeilBrown9f2c9d12011-10-11 16:48:43 +1100249static void put_buf(struct r10bio *r10_bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250{
NeilBrowne879a872011-10-11 16:49:02 +1100251 struct r10conf *conf = r10_bio->mddev->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252
253 mempool_free(r10_bio, conf->r10buf_pool);
254
NeilBrown0a27ec92006-01-06 00:20:13 -0800255 lower_barrier(conf);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256}
257
NeilBrown9f2c9d12011-10-11 16:48:43 +1100258static void reschedule_retry(struct r10bio *r10_bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259{
260 unsigned long flags;
NeilBrownfd01b882011-10-11 16:47:53 +1100261 struct mddev *mddev = r10_bio->mddev;
NeilBrowne879a872011-10-11 16:49:02 +1100262 struct r10conf *conf = mddev->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263
264 spin_lock_irqsave(&conf->device_lock, flags);
265 list_add(&r10_bio->retry_list, &conf->retry_list);
NeilBrown4443ae12006-01-06 00:20:28 -0800266 conf->nr_queued ++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267 spin_unlock_irqrestore(&conf->device_lock, flags);
268
Arthur Jones388667b2008-07-25 12:03:38 -0700269 /* wake up frozen array... */
270 wake_up(&conf->wait_barrier);
271
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272 md_wakeup_thread(mddev->thread);
273}
274
275/*
276 * raid_end_bio_io() is called when we have finished servicing a mirrored
277 * operation and are ready to return a success/failure code to the buffer
278 * cache layer.
279 */
NeilBrown9f2c9d12011-10-11 16:48:43 +1100280static void raid_end_bio_io(struct r10bio *r10_bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700281{
282 struct bio *bio = r10_bio->master_bio;
NeilBrown856e08e2011-07-28 11:39:23 +1000283 int done;
NeilBrowne879a872011-10-11 16:49:02 +1100284 struct r10conf *conf = r10_bio->mddev->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285
NeilBrown856e08e2011-07-28 11:39:23 +1000286 if (bio->bi_phys_segments) {
287 unsigned long flags;
288 spin_lock_irqsave(&conf->device_lock, flags);
289 bio->bi_phys_segments--;
290 done = (bio->bi_phys_segments == 0);
291 spin_unlock_irqrestore(&conf->device_lock, flags);
292 } else
293 done = 1;
294 if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
295 clear_bit(BIO_UPTODATE, &bio->bi_flags);
296 if (done) {
297 bio_endio(bio, 0);
298 /*
299 * Wake up any possible resync thread that waits for the device
300 * to go idle.
301 */
302 allow_barrier(conf);
303 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304 free_r10bio(r10_bio);
305}
306
307/*
308 * Update disk head position estimator based on IRQ completion info.
309 */
NeilBrown9f2c9d12011-10-11 16:48:43 +1100310static inline void update_head_pos(int slot, struct r10bio *r10_bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311{
NeilBrowne879a872011-10-11 16:49:02 +1100312 struct r10conf *conf = r10_bio->mddev->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313
314 conf->mirrors[r10_bio->devs[slot].devnum].head_position =
315 r10_bio->devs[slot].addr + (r10_bio->sectors);
316}
317
Namhyung Kim778ca012011-07-18 17:38:47 +1000318/*
319 * Find the disk number which triggered given bio
320 */
NeilBrowne879a872011-10-11 16:49:02 +1100321static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio,
NeilBrown69335ef2011-12-23 10:17:54 +1100322 struct bio *bio, int *slotp, int *replp)
Namhyung Kim778ca012011-07-18 17:38:47 +1000323{
324 int slot;
NeilBrown69335ef2011-12-23 10:17:54 +1100325 int repl = 0;
Namhyung Kim778ca012011-07-18 17:38:47 +1000326
NeilBrown69335ef2011-12-23 10:17:54 +1100327 for (slot = 0; slot < conf->copies; slot++) {
Namhyung Kim778ca012011-07-18 17:38:47 +1000328 if (r10_bio->devs[slot].bio == bio)
329 break;
NeilBrown69335ef2011-12-23 10:17:54 +1100330 if (r10_bio->devs[slot].repl_bio == bio) {
331 repl = 1;
332 break;
333 }
334 }
Namhyung Kim778ca012011-07-18 17:38:47 +1000335
336 BUG_ON(slot == conf->copies);
337 update_head_pos(slot, r10_bio);
338
NeilBrown749c55e2011-07-28 11:39:24 +1000339 if (slotp)
340 *slotp = slot;
NeilBrown69335ef2011-12-23 10:17:54 +1100341 if (replp)
342 *replp = repl;
Namhyung Kim778ca012011-07-18 17:38:47 +1000343 return r10_bio->devs[slot].devnum;
344}
345
NeilBrown6712ecf2007-09-27 12:47:43 +0200346static void raid10_end_read_request(struct bio *bio, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347{
348 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
NeilBrown9f2c9d12011-10-11 16:48:43 +1100349 struct r10bio *r10_bio = bio->bi_private;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350 int slot, dev;
NeilBrownabbf0982011-12-23 10:17:54 +1100351 struct md_rdev *rdev;
NeilBrowne879a872011-10-11 16:49:02 +1100352 struct r10conf *conf = r10_bio->mddev->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353
Linus Torvalds1da177e2005-04-16 15:20:36 -0700354
355 slot = r10_bio->read_slot;
356 dev = r10_bio->devs[slot].devnum;
NeilBrownabbf0982011-12-23 10:17:54 +1100357 rdev = r10_bio->devs[slot].rdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358 /*
359 * this branch is our 'one mirror IO has finished' event handler:
360 */
NeilBrown4443ae12006-01-06 00:20:28 -0800361 update_head_pos(slot, r10_bio);
362
363 if (uptodate) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364 /*
365 * Set R10BIO_Uptodate in our master bio, so that
366 * we will return a good error code to the higher
367 * levels even if IO on some other mirrored buffer fails.
368 *
369 * The 'master' represents the composite IO operation to
370 * user-side. So if something waits for IO, then it will
371 * wait for the 'master' bio.
372 */
373 set_bit(R10BIO_Uptodate, &r10_bio->state);
NeilBrownfae8cc5e2012-02-14 11:10:10 +1100374 } else {
375 /* If all other devices that store this block have
376 * failed, we want to return the error upwards rather
377 * than fail the last device. Here we redefine
378 * "uptodate" to mean "Don't want to retry"
379 */
380 unsigned long flags;
381 spin_lock_irqsave(&conf->device_lock, flags);
382 if (!enough(conf, rdev->raid_disk))
383 uptodate = 1;
384 spin_unlock_irqrestore(&conf->device_lock, flags);
385 }
386 if (uptodate) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700387 raid_end_bio_io(r10_bio);
NeilBrownabbf0982011-12-23 10:17:54 +1100388 rdev_dec_pending(rdev, conf->mddev);
NeilBrown4443ae12006-01-06 00:20:28 -0800389 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390 /*
NeilBrown7c4e06f2011-05-11 14:53:17 +1000391 * oops, read error - keep the refcount on the rdev
Linus Torvalds1da177e2005-04-16 15:20:36 -0700392 */
393 char b[BDEVNAME_SIZE];
Christian Dietrich8bda4702011-07-27 11:00:36 +1000394 printk_ratelimited(KERN_ERR
395 "md/raid10:%s: %s: rescheduling sector %llu\n",
396 mdname(conf->mddev),
NeilBrownabbf0982011-12-23 10:17:54 +1100397 bdevname(rdev->bdev, b),
Christian Dietrich8bda4702011-07-27 11:00:36 +1000398 (unsigned long long)r10_bio->sector);
NeilBrown856e08e2011-07-28 11:39:23 +1000399 set_bit(R10BIO_ReadError, &r10_bio->state);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700400 reschedule_retry(r10_bio);
401 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700402}
403
NeilBrown9f2c9d12011-10-11 16:48:43 +1100404static void close_write(struct r10bio *r10_bio)
NeilBrownbd870a12011-07-28 11:39:24 +1000405{
406 /* clear the bitmap if all writes complete successfully */
407 bitmap_endwrite(r10_bio->mddev->bitmap, r10_bio->sector,
408 r10_bio->sectors,
409 !test_bit(R10BIO_Degraded, &r10_bio->state),
410 0);
411 md_write_end(r10_bio->mddev);
412}
413
NeilBrown9f2c9d12011-10-11 16:48:43 +1100414static void one_write_done(struct r10bio *r10_bio)
NeilBrown19d5f832011-09-10 17:21:17 +1000415{
416 if (atomic_dec_and_test(&r10_bio->remaining)) {
417 if (test_bit(R10BIO_WriteError, &r10_bio->state))
418 reschedule_retry(r10_bio);
419 else {
420 close_write(r10_bio);
421 if (test_bit(R10BIO_MadeGood, &r10_bio->state))
422 reschedule_retry(r10_bio);
423 else
424 raid_end_bio_io(r10_bio);
425 }
426 }
427}
428
NeilBrown6712ecf2007-09-27 12:47:43 +0200429static void raid10_end_write_request(struct bio *bio, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430{
431 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
NeilBrown9f2c9d12011-10-11 16:48:43 +1100432 struct r10bio *r10_bio = bio->bi_private;
Namhyung Kim778ca012011-07-18 17:38:47 +1000433 int dev;
NeilBrown749c55e2011-07-28 11:39:24 +1000434 int dec_rdev = 1;
NeilBrowne879a872011-10-11 16:49:02 +1100435 struct r10conf *conf = r10_bio->mddev->private;
NeilBrown475b0322011-12-23 10:17:55 +1100436 int slot, repl;
NeilBrown4ca40c22011-12-23 10:17:55 +1100437 struct md_rdev *rdev = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700438
NeilBrown475b0322011-12-23 10:17:55 +1100439 dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440
NeilBrown475b0322011-12-23 10:17:55 +1100441 if (repl)
442 rdev = conf->mirrors[dev].replacement;
NeilBrown4ca40c22011-12-23 10:17:55 +1100443 if (!rdev) {
444 smp_rmb();
445 repl = 0;
NeilBrown475b0322011-12-23 10:17:55 +1100446 rdev = conf->mirrors[dev].rdev;
NeilBrown4ca40c22011-12-23 10:17:55 +1100447 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448 /*
449 * this branch is our 'one mirror IO has finished' event handler:
450 */
NeilBrown6cce3b232006-01-06 00:20:16 -0800451 if (!uptodate) {
NeilBrown475b0322011-12-23 10:17:55 +1100452 if (repl)
453 /* Never record new bad blocks to replacement,
454 * just fail it.
455 */
456 md_error(rdev->mddev, rdev);
457 else {
458 set_bit(WriteErrorSeen, &rdev->flags);
NeilBrownb7044d42011-12-23 10:17:56 +1100459 if (!test_and_set_bit(WantReplacement, &rdev->flags))
460 set_bit(MD_RECOVERY_NEEDED,
461 &rdev->mddev->recovery);
NeilBrown475b0322011-12-23 10:17:55 +1100462 set_bit(R10BIO_WriteError, &r10_bio->state);
463 dec_rdev = 0;
464 }
NeilBrown749c55e2011-07-28 11:39:24 +1000465 } else {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700466 /*
467 * Set R10BIO_Uptodate in our master bio, so that
468 * we will return a good error code for to the higher
469 * levels even if IO on some other mirrored buffer fails.
470 *
471 * The 'master' represents the composite IO operation to
472 * user-side. So if something waits for IO, then it will
473 * wait for the 'master' bio.
474 */
NeilBrown749c55e2011-07-28 11:39:24 +1000475 sector_t first_bad;
476 int bad_sectors;
477
Linus Torvalds1da177e2005-04-16 15:20:36 -0700478 set_bit(R10BIO_Uptodate, &r10_bio->state);
479
NeilBrown749c55e2011-07-28 11:39:24 +1000480 /* Maybe we can clear some bad blocks. */
NeilBrown475b0322011-12-23 10:17:55 +1100481 if (is_badblock(rdev,
NeilBrown749c55e2011-07-28 11:39:24 +1000482 r10_bio->devs[slot].addr,
483 r10_bio->sectors,
484 &first_bad, &bad_sectors)) {
485 bio_put(bio);
NeilBrown475b0322011-12-23 10:17:55 +1100486 if (repl)
487 r10_bio->devs[slot].repl_bio = IO_MADE_GOOD;
488 else
489 r10_bio->devs[slot].bio = IO_MADE_GOOD;
NeilBrown749c55e2011-07-28 11:39:24 +1000490 dec_rdev = 0;
491 set_bit(R10BIO_MadeGood, &r10_bio->state);
492 }
493 }
494
Linus Torvalds1da177e2005-04-16 15:20:36 -0700495 /*
496 *
497 * Let's see if all mirrored write operations have finished
498 * already.
499 */
NeilBrown19d5f832011-09-10 17:21:17 +1000500 one_write_done(r10_bio);
NeilBrown749c55e2011-07-28 11:39:24 +1000501 if (dec_rdev)
502 rdev_dec_pending(conf->mirrors[dev].rdev, conf->mddev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503}
504
Linus Torvalds1da177e2005-04-16 15:20:36 -0700505/*
506 * RAID10 layout manager
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300507 * As well as the chunksize and raid_disks count, there are two
Linus Torvalds1da177e2005-04-16 15:20:36 -0700508 * parameters: near_copies and far_copies.
509 * near_copies * far_copies must be <= raid_disks.
510 * Normally one of these will be 1.
511 * If both are 1, we get raid0.
512 * If near_copies == raid_disks, we get raid1.
513 *
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300514 * Chunks are laid out in raid0 style with near_copies copies of the
Linus Torvalds1da177e2005-04-16 15:20:36 -0700515 * first chunk, followed by near_copies copies of the next chunk and
516 * so on.
517 * If far_copies > 1, then after 1/far_copies of the array has been assigned
518 * as described above, we start again with a device offset of near_copies.
519 * So we effectively have another copy of the whole array further down all
520 * the drives, but with blocks on different drives.
521 * With this layout, and block is never stored twice on the one device.
522 *
523 * raid10_find_phys finds the sector offset of a given virtual sector
NeilBrownc93983b2006-06-26 00:27:41 -0700524 * on each device that it is on.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525 *
526 * raid10_find_virt does the reverse mapping, from a device and a
527 * sector offset to a virtual address
528 */
529
NeilBrownf8c9e742012-05-21 09:28:33 +1000530static void __raid10_find_phys(struct geom *geo, struct r10bio *r10bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531{
532 int n,f;
533 sector_t sector;
534 sector_t chunk;
535 sector_t stripe;
536 int dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537 int slot = 0;
538
539 /* now calculate first sector/dev */
NeilBrown5cf00fc2012-05-21 09:28:20 +1000540 chunk = r10bio->sector >> geo->chunk_shift;
541 sector = r10bio->sector & geo->chunk_mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542
NeilBrown5cf00fc2012-05-21 09:28:20 +1000543 chunk *= geo->near_copies;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544 stripe = chunk;
NeilBrown5cf00fc2012-05-21 09:28:20 +1000545 dev = sector_div(stripe, geo->raid_disks);
546 if (geo->far_offset)
547 stripe *= geo->far_copies;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700548
NeilBrown5cf00fc2012-05-21 09:28:20 +1000549 sector += stripe << geo->chunk_shift;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550
551 /* and calculate all the others */
NeilBrown5cf00fc2012-05-21 09:28:20 +1000552 for (n = 0; n < geo->near_copies; n++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553 int d = dev;
554 sector_t s = sector;
555 r10bio->devs[slot].addr = sector;
556 r10bio->devs[slot].devnum = d;
557 slot++;
558
NeilBrown5cf00fc2012-05-21 09:28:20 +1000559 for (f = 1; f < geo->far_copies; f++) {
560 d += geo->near_copies;
561 if (d >= geo->raid_disks)
562 d -= geo->raid_disks;
563 s += geo->stride;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700564 r10bio->devs[slot].devnum = d;
565 r10bio->devs[slot].addr = s;
566 slot++;
567 }
568 dev++;
NeilBrown5cf00fc2012-05-21 09:28:20 +1000569 if (dev >= geo->raid_disks) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570 dev = 0;
NeilBrown5cf00fc2012-05-21 09:28:20 +1000571 sector += (geo->chunk_mask + 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572 }
573 }
NeilBrownf8c9e742012-05-21 09:28:33 +1000574}
575
576static void raid10_find_phys(struct r10conf *conf, struct r10bio *r10bio)
577{
578 struct geom *geo = &conf->geo;
579
580 if (conf->reshape_progress != MaxSector &&
581 ((r10bio->sector >= conf->reshape_progress) !=
582 conf->mddev->reshape_backwards)) {
583 set_bit(R10BIO_Previous, &r10bio->state);
584 geo = &conf->prev;
585 } else
586 clear_bit(R10BIO_Previous, &r10bio->state);
587
588 __raid10_find_phys(geo, r10bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589}
590
NeilBrowne879a872011-10-11 16:49:02 +1100591static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592{
593 sector_t offset, chunk, vchunk;
NeilBrownf8c9e742012-05-21 09:28:33 +1000594 /* Never use conf->prev as this is only called during resync
595 * or recovery, so reshape isn't happening
596 */
NeilBrown5cf00fc2012-05-21 09:28:20 +1000597 struct geom *geo = &conf->geo;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598
NeilBrown5cf00fc2012-05-21 09:28:20 +1000599 offset = sector & geo->chunk_mask;
600 if (geo->far_offset) {
NeilBrownc93983b2006-06-26 00:27:41 -0700601 int fc;
NeilBrown5cf00fc2012-05-21 09:28:20 +1000602 chunk = sector >> geo->chunk_shift;
603 fc = sector_div(chunk, geo->far_copies);
604 dev -= fc * geo->near_copies;
NeilBrownc93983b2006-06-26 00:27:41 -0700605 if (dev < 0)
NeilBrown5cf00fc2012-05-21 09:28:20 +1000606 dev += geo->raid_disks;
NeilBrownc93983b2006-06-26 00:27:41 -0700607 } else {
NeilBrown5cf00fc2012-05-21 09:28:20 +1000608 while (sector >= geo->stride) {
609 sector -= geo->stride;
610 if (dev < geo->near_copies)
611 dev += geo->raid_disks - geo->near_copies;
NeilBrownc93983b2006-06-26 00:27:41 -0700612 else
NeilBrown5cf00fc2012-05-21 09:28:20 +1000613 dev -= geo->near_copies;
NeilBrownc93983b2006-06-26 00:27:41 -0700614 }
NeilBrown5cf00fc2012-05-21 09:28:20 +1000615 chunk = sector >> geo->chunk_shift;
NeilBrownc93983b2006-06-26 00:27:41 -0700616 }
NeilBrown5cf00fc2012-05-21 09:28:20 +1000617 vchunk = chunk * geo->raid_disks + dev;
618 sector_div(vchunk, geo->near_copies);
619 return (vchunk << geo->chunk_shift) + offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620}
621
622/**
623 * raid10_mergeable_bvec -- tell bio layer if a two requests can be merged
624 * @q: request queue
Alasdair G Kergoncc371e62008-07-03 09:53:43 +0200625 * @bvm: properties of new bio
Linus Torvalds1da177e2005-04-16 15:20:36 -0700626 * @biovec: the request that could be merged to it.
627 *
628 * Return amount of bytes we can accept at this offset
NeilBrown050b6612012-03-19 12:46:39 +1100629 * This requires checking for end-of-chunk if near_copies != raid_disks,
630 * and for subordinate merge_bvec_fns if merge_check_needed.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631 */
Alasdair G Kergoncc371e62008-07-03 09:53:43 +0200632static int raid10_mergeable_bvec(struct request_queue *q,
633 struct bvec_merge_data *bvm,
634 struct bio_vec *biovec)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635{
NeilBrownfd01b882011-10-11 16:47:53 +1100636 struct mddev *mddev = q->queuedata;
NeilBrown050b6612012-03-19 12:46:39 +1100637 struct r10conf *conf = mddev->private;
Alasdair G Kergoncc371e62008-07-03 09:53:43 +0200638 sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700639 int max;
NeilBrown3ea7daa2012-05-22 13:53:47 +1000640 unsigned int chunk_sectors;
Alasdair G Kergoncc371e62008-07-03 09:53:43 +0200641 unsigned int bio_sectors = bvm->bi_size >> 9;
NeilBrown5cf00fc2012-05-21 09:28:20 +1000642 struct geom *geo = &conf->geo;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700643
NeilBrown3ea7daa2012-05-22 13:53:47 +1000644 chunk_sectors = (conf->geo.chunk_mask & conf->prev.chunk_mask) + 1;
NeilBrownf8c9e742012-05-21 09:28:33 +1000645 if (conf->reshape_progress != MaxSector &&
646 ((sector >= conf->reshape_progress) !=
647 conf->mddev->reshape_backwards))
648 geo = &conf->prev;
649
NeilBrown5cf00fc2012-05-21 09:28:20 +1000650 if (geo->near_copies < geo->raid_disks) {
NeilBrown050b6612012-03-19 12:46:39 +1100651 max = (chunk_sectors - ((sector & (chunk_sectors - 1))
652 + bio_sectors)) << 9;
653 if (max < 0)
654 /* bio_add cannot handle a negative return */
655 max = 0;
656 if (max <= biovec->bv_len && bio_sectors == 0)
657 return biovec->bv_len;
658 } else
659 max = biovec->bv_len;
660
661 if (mddev->merge_check_needed) {
NeilBrowne0ee7782012-08-18 09:51:42 +1000662 struct {
663 struct r10bio r10_bio;
664 struct r10dev devs[conf->copies];
665 } on_stack;
666 struct r10bio *r10_bio = &on_stack.r10_bio;
NeilBrown050b6612012-03-19 12:46:39 +1100667 int s;
NeilBrownf8c9e742012-05-21 09:28:33 +1000668 if (conf->reshape_progress != MaxSector) {
669 /* Cannot give any guidance during reshape */
670 if (max <= biovec->bv_len && bio_sectors == 0)
671 return biovec->bv_len;
672 return 0;
673 }
NeilBrowne0ee7782012-08-18 09:51:42 +1000674 r10_bio->sector = sector;
675 raid10_find_phys(conf, r10_bio);
NeilBrown050b6612012-03-19 12:46:39 +1100676 rcu_read_lock();
677 for (s = 0; s < conf->copies; s++) {
NeilBrowne0ee7782012-08-18 09:51:42 +1000678 int disk = r10_bio->devs[s].devnum;
NeilBrown050b6612012-03-19 12:46:39 +1100679 struct md_rdev *rdev = rcu_dereference(
680 conf->mirrors[disk].rdev);
681 if (rdev && !test_bit(Faulty, &rdev->flags)) {
682 struct request_queue *q =
683 bdev_get_queue(rdev->bdev);
684 if (q->merge_bvec_fn) {
NeilBrowne0ee7782012-08-18 09:51:42 +1000685 bvm->bi_sector = r10_bio->devs[s].addr
NeilBrown050b6612012-03-19 12:46:39 +1100686 + rdev->data_offset;
687 bvm->bi_bdev = rdev->bdev;
688 max = min(max, q->merge_bvec_fn(
689 q, bvm, biovec));
690 }
691 }
692 rdev = rcu_dereference(conf->mirrors[disk].replacement);
693 if (rdev && !test_bit(Faulty, &rdev->flags)) {
694 struct request_queue *q =
695 bdev_get_queue(rdev->bdev);
696 if (q->merge_bvec_fn) {
NeilBrowne0ee7782012-08-18 09:51:42 +1000697 bvm->bi_sector = r10_bio->devs[s].addr
NeilBrown050b6612012-03-19 12:46:39 +1100698 + rdev->data_offset;
699 bvm->bi_bdev = rdev->bdev;
700 max = min(max, q->merge_bvec_fn(
701 q, bvm, biovec));
702 }
703 }
704 }
705 rcu_read_unlock();
706 }
707 return max;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700708}
709
710/*
711 * This routine returns the disk from which the requested read should
712 * be done. There is a per-array 'next expected sequential IO' sector
713 * number - if this matches on the next IO then we use the last disk.
714 * There is also a per-disk 'last know head position' sector that is
715 * maintained from IRQ contexts, both the normal and the resync IO
716 * completion handlers update this position correctly. If there is no
717 * perfect sequential match then we pick the disk whose head is closest.
718 *
719 * If there are 2 mirrors in the same 2 devices, performance degrades
720 * because position is mirror, not device based.
721 *
722 * The rdev for the device selected will have nr_pending incremented.
723 */
724
725/*
726 * FIXME: possibly should rethink readbalancing and do it differently
727 * depending on near_copies / far_copies geometry.
728 */
NeilBrown96c3fd12011-12-23 10:17:54 +1100729static struct md_rdev *read_balance(struct r10conf *conf,
730 struct r10bio *r10_bio,
731 int *max_sectors)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700732{
NeilBrownaf3a2cd2010-05-08 08:20:17 +1000733 const sector_t this_sector = r10_bio->sector;
NeilBrown56d99122011-05-11 14:27:03 +1000734 int disk, slot;
NeilBrown856e08e2011-07-28 11:39:23 +1000735 int sectors = r10_bio->sectors;
736 int best_good_sectors;
NeilBrown56d99122011-05-11 14:27:03 +1000737 sector_t new_distance, best_dist;
Jonathan Brassow3bbae042012-07-31 10:03:52 +1000738 struct md_rdev *best_rdev, *rdev = NULL;
NeilBrown56d99122011-05-11 14:27:03 +1000739 int do_balance;
740 int best_slot;
NeilBrown5cf00fc2012-05-21 09:28:20 +1000741 struct geom *geo = &conf->geo;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742
743 raid10_find_phys(conf, r10_bio);
744 rcu_read_lock();
NeilBrown56d99122011-05-11 14:27:03 +1000745retry:
NeilBrown856e08e2011-07-28 11:39:23 +1000746 sectors = r10_bio->sectors;
NeilBrown56d99122011-05-11 14:27:03 +1000747 best_slot = -1;
NeilBrownabbf0982011-12-23 10:17:54 +1100748 best_rdev = NULL;
NeilBrown56d99122011-05-11 14:27:03 +1000749 best_dist = MaxSector;
NeilBrown856e08e2011-07-28 11:39:23 +1000750 best_good_sectors = 0;
NeilBrown56d99122011-05-11 14:27:03 +1000751 do_balance = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700752 /*
753 * Check if we can balance. We can balance on the whole
NeilBrown6cce3b232006-01-06 00:20:16 -0800754 * device if no resync is going on (recovery is ok), or below
755 * the resync window. We take the first readable disk when
756 * above the resync window.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700757 */
758 if (conf->mddev->recovery_cp < MaxSector
NeilBrown56d99122011-05-11 14:27:03 +1000759 && (this_sector + sectors >= conf->next_resync))
760 do_balance = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761
NeilBrown56d99122011-05-11 14:27:03 +1000762 for (slot = 0; slot < conf->copies ; slot++) {
NeilBrown856e08e2011-07-28 11:39:23 +1000763 sector_t first_bad;
764 int bad_sectors;
765 sector_t dev_sector;
766
NeilBrown56d99122011-05-11 14:27:03 +1000767 if (r10_bio->devs[slot].bio == IO_BLOCKED)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700768 continue;
NeilBrown56d99122011-05-11 14:27:03 +1000769 disk = r10_bio->devs[slot].devnum;
NeilBrownabbf0982011-12-23 10:17:54 +1100770 rdev = rcu_dereference(conf->mirrors[disk].replacement);
771 if (rdev == NULL || test_bit(Faulty, &rdev->flags) ||
NeilBrown050b6612012-03-19 12:46:39 +1100772 test_bit(Unmerged, &rdev->flags) ||
NeilBrownabbf0982011-12-23 10:17:54 +1100773 r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
774 rdev = rcu_dereference(conf->mirrors[disk].rdev);
NeilBrown050b6612012-03-19 12:46:39 +1100775 if (rdev == NULL ||
776 test_bit(Faulty, &rdev->flags) ||
777 test_bit(Unmerged, &rdev->flags))
NeilBrownabbf0982011-12-23 10:17:54 +1100778 continue;
779 if (!test_bit(In_sync, &rdev->flags) &&
780 r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
NeilBrown56d99122011-05-11 14:27:03 +1000781 continue;
782
NeilBrown856e08e2011-07-28 11:39:23 +1000783 dev_sector = r10_bio->devs[slot].addr;
784 if (is_badblock(rdev, dev_sector, sectors,
785 &first_bad, &bad_sectors)) {
786 if (best_dist < MaxSector)
787 /* Already have a better slot */
788 continue;
789 if (first_bad <= dev_sector) {
790 /* Cannot read here. If this is the
791 * 'primary' device, then we must not read
792 * beyond 'bad_sectors' from another device.
793 */
794 bad_sectors -= (dev_sector - first_bad);
795 if (!do_balance && sectors > bad_sectors)
796 sectors = bad_sectors;
797 if (best_good_sectors > sectors)
798 best_good_sectors = sectors;
799 } else {
800 sector_t good_sectors =
801 first_bad - dev_sector;
802 if (good_sectors > best_good_sectors) {
803 best_good_sectors = good_sectors;
804 best_slot = slot;
NeilBrownabbf0982011-12-23 10:17:54 +1100805 best_rdev = rdev;
NeilBrown856e08e2011-07-28 11:39:23 +1000806 }
807 if (!do_balance)
808 /* Must read from here */
809 break;
810 }
811 continue;
812 } else
813 best_good_sectors = sectors;
814
NeilBrown56d99122011-05-11 14:27:03 +1000815 if (!do_balance)
816 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700817
NeilBrown22dfdf52005-11-28 13:44:09 -0800818 /* This optimisation is debatable, and completely destroys
819 * sequential read speed for 'far copies' arrays. So only
820 * keep it for 'near' arrays, and review those later.
821 */
NeilBrown5cf00fc2012-05-21 09:28:20 +1000822 if (geo->near_copies > 1 && !atomic_read(&rdev->nr_pending))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823 break;
Keld Simonsen8ed3a192008-03-04 14:29:34 -0800824
825 /* for far > 1 always use the lowest address */
NeilBrown5cf00fc2012-05-21 09:28:20 +1000826 if (geo->far_copies > 1)
NeilBrown56d99122011-05-11 14:27:03 +1000827 new_distance = r10_bio->devs[slot].addr;
Keld Simonsen8ed3a192008-03-04 14:29:34 -0800828 else
NeilBrown56d99122011-05-11 14:27:03 +1000829 new_distance = abs(r10_bio->devs[slot].addr -
830 conf->mirrors[disk].head_position);
831 if (new_distance < best_dist) {
832 best_dist = new_distance;
833 best_slot = slot;
NeilBrownabbf0982011-12-23 10:17:54 +1100834 best_rdev = rdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700835 }
836 }
NeilBrownabbf0982011-12-23 10:17:54 +1100837 if (slot >= conf->copies) {
NeilBrown56d99122011-05-11 14:27:03 +1000838 slot = best_slot;
NeilBrownabbf0982011-12-23 10:17:54 +1100839 rdev = best_rdev;
840 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841
NeilBrown56d99122011-05-11 14:27:03 +1000842 if (slot >= 0) {
NeilBrown56d99122011-05-11 14:27:03 +1000843 atomic_inc(&rdev->nr_pending);
844 if (test_bit(Faulty, &rdev->flags)) {
845 /* Cannot risk returning a device that failed
846 * before we inc'ed nr_pending
847 */
848 rdev_dec_pending(rdev, conf->mddev);
849 goto retry;
850 }
851 r10_bio->read_slot = slot;
852 } else
NeilBrown96c3fd12011-12-23 10:17:54 +1100853 rdev = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700854 rcu_read_unlock();
NeilBrown856e08e2011-07-28 11:39:23 +1000855 *max_sectors = best_good_sectors;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700856
NeilBrown96c3fd12011-12-23 10:17:54 +1100857 return rdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700858}
859
Jonathan Brassowcc4d1ef2012-07-31 10:03:53 +1000860int md_raid10_congested(struct mddev *mddev, int bits)
NeilBrown0d129222006-10-03 01:15:54 -0700861{
NeilBrowne879a872011-10-11 16:49:02 +1100862 struct r10conf *conf = mddev->private;
NeilBrown0d129222006-10-03 01:15:54 -0700863 int i, ret = 0;
864
NeilBrown34db0cd2011-10-11 16:50:01 +1100865 if ((bits & (1 << BDI_async_congested)) &&
866 conf->pending_count >= max_queued_requests)
867 return 1;
868
NeilBrown0d129222006-10-03 01:15:54 -0700869 rcu_read_lock();
NeilBrownf8c9e742012-05-21 09:28:33 +1000870 for (i = 0;
871 (i < conf->geo.raid_disks || i < conf->prev.raid_disks)
872 && ret == 0;
873 i++) {
NeilBrown3cb03002011-10-11 16:45:26 +1100874 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
NeilBrown0d129222006-10-03 01:15:54 -0700875 if (rdev && !test_bit(Faulty, &rdev->flags)) {
Jens Axboe165125e2007-07-24 09:28:11 +0200876 struct request_queue *q = bdev_get_queue(rdev->bdev);
NeilBrown0d129222006-10-03 01:15:54 -0700877
878 ret |= bdi_congested(&q->backing_dev_info, bits);
879 }
880 }
881 rcu_read_unlock();
882 return ret;
883}
Jonathan Brassowcc4d1ef2012-07-31 10:03:53 +1000884EXPORT_SYMBOL_GPL(md_raid10_congested);
885
886static int raid10_congested(void *data, int bits)
887{
888 struct mddev *mddev = data;
889
890 return mddev_congested(mddev, bits) ||
891 md_raid10_congested(mddev, bits);
892}
NeilBrown0d129222006-10-03 01:15:54 -0700893
NeilBrowne879a872011-10-11 16:49:02 +1100894static void flush_pending_writes(struct r10conf *conf)
NeilBrowna35e63e2008-03-04 14:29:29 -0800895{
896 /* Any writes that have been queued but are awaiting
897 * bitmap updates get flushed here.
NeilBrowna35e63e2008-03-04 14:29:29 -0800898 */
NeilBrowna35e63e2008-03-04 14:29:29 -0800899 spin_lock_irq(&conf->device_lock);
900
901 if (conf->pending_bio_list.head) {
902 struct bio *bio;
903 bio = bio_list_get(&conf->pending_bio_list);
NeilBrown34db0cd2011-10-11 16:50:01 +1100904 conf->pending_count = 0;
NeilBrowna35e63e2008-03-04 14:29:29 -0800905 spin_unlock_irq(&conf->device_lock);
906 /* flush any pending bitmap writes to disk
907 * before proceeding w/ I/O */
908 bitmap_unplug(conf->mddev->bitmap);
NeilBrown34db0cd2011-10-11 16:50:01 +1100909 wake_up(&conf->wait_barrier);
NeilBrowna35e63e2008-03-04 14:29:29 -0800910
911 while (bio) { /* submit pending writes */
912 struct bio *next = bio->bi_next;
913 bio->bi_next = NULL;
Shaohua Li532a2a32012-10-11 13:30:52 +1100914 if (unlikely((bio->bi_rw & REQ_DISCARD) &&
915 !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
916 /* Just ignore it */
917 bio_endio(bio, 0);
918 else
919 generic_make_request(bio);
NeilBrowna35e63e2008-03-04 14:29:29 -0800920 bio = next;
921 }
NeilBrowna35e63e2008-03-04 14:29:29 -0800922 } else
923 spin_unlock_irq(&conf->device_lock);
NeilBrowna35e63e2008-03-04 14:29:29 -0800924}
Jens Axboe7eaceac2011-03-10 08:52:07 +0100925
NeilBrown0a27ec92006-01-06 00:20:13 -0800926/* Barriers....
927 * Sometimes we need to suspend IO while we do something else,
928 * either some resync/recovery, or reconfigure the array.
929 * To do this we raise a 'barrier'.
930 * The 'barrier' is a counter that can be raised multiple times
931 * to count how many activities are happening which preclude
932 * normal IO.
933 * We can only raise the barrier if there is no pending IO.
934 * i.e. if nr_pending == 0.
935 * We choose only to raise the barrier if no-one is waiting for the
936 * barrier to go down. This means that as soon as an IO request
937 * is ready, no other operations which require a barrier will start
938 * until the IO request has had a chance.
939 *
940 * So: regular IO calls 'wait_barrier'. When that returns there
941 * is no backgroup IO happening, It must arrange to call
942 * allow_barrier when it has finished its IO.
943 * backgroup IO calls must call raise_barrier. Once that returns
944 * there is no normal IO happeing. It must arrange to call
945 * lower_barrier when the particular background IO completes.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700947
NeilBrowne879a872011-10-11 16:49:02 +1100948static void raise_barrier(struct r10conf *conf, int force)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700949{
NeilBrown6cce3b232006-01-06 00:20:16 -0800950 BUG_ON(force && !conf->barrier);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700951 spin_lock_irq(&conf->resync_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700952
NeilBrown6cce3b232006-01-06 00:20:16 -0800953 /* Wait until no block IO is waiting (unless 'force') */
954 wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting,
Lukas Czernereed8c022012-11-30 11:42:40 +0100955 conf->resync_lock);
NeilBrown0a27ec92006-01-06 00:20:13 -0800956
957 /* block any new IO from starting */
958 conf->barrier++;
959
NeilBrownc3b328a2011-04-18 18:25:43 +1000960 /* Now wait for all pending IO to complete */
NeilBrown0a27ec92006-01-06 00:20:13 -0800961 wait_event_lock_irq(conf->wait_barrier,
962 !conf->nr_pending && conf->barrier < RESYNC_DEPTH,
Lukas Czernereed8c022012-11-30 11:42:40 +0100963 conf->resync_lock);
NeilBrown0a27ec92006-01-06 00:20:13 -0800964
Linus Torvalds1da177e2005-04-16 15:20:36 -0700965 spin_unlock_irq(&conf->resync_lock);
966}
967
NeilBrowne879a872011-10-11 16:49:02 +1100968static void lower_barrier(struct r10conf *conf)
NeilBrown0a27ec92006-01-06 00:20:13 -0800969{
970 unsigned long flags;
971 spin_lock_irqsave(&conf->resync_lock, flags);
972 conf->barrier--;
973 spin_unlock_irqrestore(&conf->resync_lock, flags);
974 wake_up(&conf->wait_barrier);
975}
976
NeilBrowne879a872011-10-11 16:49:02 +1100977static void wait_barrier(struct r10conf *conf)
NeilBrown0a27ec92006-01-06 00:20:13 -0800978{
979 spin_lock_irq(&conf->resync_lock);
980 if (conf->barrier) {
981 conf->nr_waiting++;
NeilBrownd6b42dc2012-03-19 12:46:38 +1100982 /* Wait for the barrier to drop.
983 * However if there are already pending
984 * requests (preventing the barrier from
985 * rising completely), and the
986 * pre-process bio queue isn't empty,
987 * then don't wait, as we need to empty
988 * that queue to get the nr_pending
989 * count down.
990 */
991 wait_event_lock_irq(conf->wait_barrier,
992 !conf->barrier ||
993 (conf->nr_pending &&
994 current->bio_list &&
995 !bio_list_empty(current->bio_list)),
Lukas Czernereed8c022012-11-30 11:42:40 +0100996 conf->resync_lock);
NeilBrown0a27ec92006-01-06 00:20:13 -0800997 conf->nr_waiting--;
998 }
999 conf->nr_pending++;
1000 spin_unlock_irq(&conf->resync_lock);
1001}
1002
NeilBrowne879a872011-10-11 16:49:02 +11001003static void allow_barrier(struct r10conf *conf)
NeilBrown0a27ec92006-01-06 00:20:13 -08001004{
1005 unsigned long flags;
1006 spin_lock_irqsave(&conf->resync_lock, flags);
1007 conf->nr_pending--;
1008 spin_unlock_irqrestore(&conf->resync_lock, flags);
1009 wake_up(&conf->wait_barrier);
1010}
1011
NeilBrowne879a872011-10-11 16:49:02 +11001012static void freeze_array(struct r10conf *conf)
NeilBrown4443ae12006-01-06 00:20:28 -08001013{
1014 /* stop syncio and normal IO and wait for everything to
NeilBrownf1885932006-01-06 00:20:42 -08001015 * go quiet.
NeilBrown4443ae12006-01-06 00:20:28 -08001016 * We increment barrier and nr_waiting, and then
NeilBrown1c830532008-03-04 14:29:35 -08001017 * wait until nr_pending match nr_queued+1
1018 * This is called in the context of one normal IO request
1019 * that has failed. Thus any sync request that might be pending
1020 * will be blocked by nr_pending, and we need to wait for
1021 * pending IO requests to complete or be queued for re-try.
1022 * Thus the number queued (nr_queued) plus this request (1)
1023 * must match the number of pending IOs (nr_pending) before
1024 * we continue.
NeilBrown4443ae12006-01-06 00:20:28 -08001025 */
1026 spin_lock_irq(&conf->resync_lock);
1027 conf->barrier++;
1028 conf->nr_waiting++;
Lukas Czernereed8c022012-11-30 11:42:40 +01001029 wait_event_lock_irq_cmd(conf->wait_barrier,
1030 conf->nr_pending == conf->nr_queued+1,
1031 conf->resync_lock,
1032 flush_pending_writes(conf));
NeilBrownc3b328a2011-04-18 18:25:43 +10001033
NeilBrown4443ae12006-01-06 00:20:28 -08001034 spin_unlock_irq(&conf->resync_lock);
1035}
1036
NeilBrowne879a872011-10-11 16:49:02 +11001037static void unfreeze_array(struct r10conf *conf)
NeilBrown4443ae12006-01-06 00:20:28 -08001038{
1039 /* reverse the effect of the freeze */
1040 spin_lock_irq(&conf->resync_lock);
1041 conf->barrier--;
1042 conf->nr_waiting--;
1043 wake_up(&conf->wait_barrier);
1044 spin_unlock_irq(&conf->resync_lock);
1045}
1046
NeilBrownf8c9e742012-05-21 09:28:33 +10001047static sector_t choose_data_offset(struct r10bio *r10_bio,
1048 struct md_rdev *rdev)
1049{
1050 if (!test_bit(MD_RECOVERY_RESHAPE, &rdev->mddev->recovery) ||
1051 test_bit(R10BIO_Previous, &r10_bio->state))
1052 return rdev->data_offset;
1053 else
1054 return rdev->new_data_offset;
1055}
1056
NeilBrown57c67df2012-10-11 13:32:13 +11001057struct raid10_plug_cb {
1058 struct blk_plug_cb cb;
1059 struct bio_list pending;
1060 int pending_cnt;
1061};
1062
1063static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule)
1064{
1065 struct raid10_plug_cb *plug = container_of(cb, struct raid10_plug_cb,
1066 cb);
1067 struct mddev *mddev = plug->cb.data;
1068 struct r10conf *conf = mddev->private;
1069 struct bio *bio;
1070
1071 if (from_schedule) {
1072 spin_lock_irq(&conf->device_lock);
1073 bio_list_merge(&conf->pending_bio_list, &plug->pending);
1074 conf->pending_count += plug->pending_cnt;
1075 spin_unlock_irq(&conf->device_lock);
1076 md_wakeup_thread(mddev->thread);
1077 kfree(plug);
1078 return;
1079 }
1080
1081 /* we aren't scheduling, so we can do the write-out directly. */
1082 bio = bio_list_get(&plug->pending);
1083 bitmap_unplug(mddev->bitmap);
1084 wake_up(&conf->wait_barrier);
1085
1086 while (bio) { /* submit pending writes */
1087 struct bio *next = bio->bi_next;
1088 bio->bi_next = NULL;
1089 generic_make_request(bio);
1090 bio = next;
1091 }
1092 kfree(plug);
1093}
1094
Linus Torvaldsb4fdcb02011-11-04 17:06:58 -07001095static void make_request(struct mddev *mddev, struct bio * bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001096{
NeilBrowne879a872011-10-11 16:49:02 +11001097 struct r10conf *conf = mddev->private;
NeilBrown9f2c9d12011-10-11 16:48:43 +11001098 struct r10bio *r10_bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099 struct bio *read_bio;
1100 int i;
NeilBrownf8c9e742012-05-21 09:28:33 +10001101 sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask);
NeilBrown5cf00fc2012-05-21 09:28:20 +10001102 int chunk_sects = chunk_mask + 1;
Jens Axboea3623572005-11-01 09:26:16 +01001103 const int rw = bio_data_dir(bio);
NeilBrown2c7d46e2010-08-18 16:16:05 +10001104 const unsigned long do_sync = (bio->bi_rw & REQ_SYNC);
Tejun Heoe9c74692010-09-03 11:56:18 +02001105 const unsigned long do_fua = (bio->bi_rw & REQ_FUA);
Shaohua Li532a2a32012-10-11 13:30:52 +11001106 const unsigned long do_discard = (bio->bi_rw
1107 & (REQ_DISCARD | REQ_SECURE));
NeilBrown6cce3b232006-01-06 00:20:16 -08001108 unsigned long flags;
NeilBrown3cb03002011-10-11 16:45:26 +11001109 struct md_rdev *blocked_rdev;
NeilBrown57c67df2012-10-11 13:32:13 +11001110 struct blk_plug_cb *cb;
1111 struct raid10_plug_cb *plug = NULL;
NeilBrownd4432c22011-07-28 11:39:24 +10001112 int sectors_handled;
1113 int max_sectors;
NeilBrown3ea7daa2012-05-22 13:53:47 +10001114 int sectors;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001115
Tejun Heoe9c74692010-09-03 11:56:18 +02001116 if (unlikely(bio->bi_rw & REQ_FLUSH)) {
1117 md_flush_request(mddev, bio);
Christoph Hellwig5a7bbad2011-09-12 12:12:01 +02001118 return;
NeilBrowne5dcdd82005-09-09 16:23:41 -07001119 }
1120
Linus Torvalds1da177e2005-04-16 15:20:36 -07001121 /* If this request crosses a chunk boundary, we need to
1122 * split it. This will only happen for 1 PAGE (or less) requests.
1123 */
NeilBrown5cf00fc2012-05-21 09:28:20 +10001124 if (unlikely((bio->bi_sector & chunk_mask) + (bio->bi_size >> 9)
1125 > chunk_sects
NeilBrownf8c9e742012-05-21 09:28:33 +10001126 && (conf->geo.near_copies < conf->geo.raid_disks
1127 || conf->prev.near_copies < conf->prev.raid_disks))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001128 struct bio_pair *bp;
1129 /* Sanity check -- queue functions should prevent this happening */
Shaohua Li532a2a32012-10-11 13:30:52 +11001130 if ((bio->bi_vcnt != 1 && bio->bi_vcnt != 0) ||
Linus Torvalds1da177e2005-04-16 15:20:36 -07001131 bio->bi_idx != 0)
1132 goto bad_map;
1133 /* This is a one page bio that upper layers
1134 * refuse to split for us, so we need to split it.
1135 */
Denis ChengRq6feef532008-10-09 08:57:05 +02001136 bp = bio_split(bio,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001137 chunk_sects - (bio->bi_sector & (chunk_sects - 1)) );
NeilBrown51e9ac72010-08-07 21:17:00 +10001138
1139 /* Each of these 'make_request' calls will call 'wait_barrier'.
1140 * If the first succeeds but the second blocks due to the resync
1141 * thread raising the barrier, we will deadlock because the
1142 * IO to the underlying device will be queued in generic_make_request
1143 * and will never complete, so will never reduce nr_pending.
1144 * So increment nr_waiting here so no new raise_barriers will
1145 * succeed, and so the second wait_barrier cannot block.
1146 */
1147 spin_lock_irq(&conf->resync_lock);
1148 conf->nr_waiting++;
1149 spin_unlock_irq(&conf->resync_lock);
1150
Christoph Hellwig5a7bbad2011-09-12 12:12:01 +02001151 make_request(mddev, &bp->bio1);
1152 make_request(mddev, &bp->bio2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001153
NeilBrown51e9ac72010-08-07 21:17:00 +10001154 spin_lock_irq(&conf->resync_lock);
1155 conf->nr_waiting--;
1156 wake_up(&conf->wait_barrier);
1157 spin_unlock_irq(&conf->resync_lock);
1158
Linus Torvalds1da177e2005-04-16 15:20:36 -07001159 bio_pair_release(bp);
Christoph Hellwig5a7bbad2011-09-12 12:12:01 +02001160 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001161 bad_map:
NeilBrown128595e2010-05-03 14:47:14 +10001162 printk("md/raid10:%s: make_request bug: can't convert block across chunks"
1163 " or bigger than %dk %llu %d\n", mdname(mddev), chunk_sects/2,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001164 (unsigned long long)bio->bi_sector, bio->bi_size >> 10);
1165
NeilBrown6712ecf2007-09-27 12:47:43 +02001166 bio_io_error(bio);
Christoph Hellwig5a7bbad2011-09-12 12:12:01 +02001167 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001168 }
1169
NeilBrown3d310eb2005-06-21 17:17:26 -07001170 md_write_start(mddev, bio);
NeilBrown06d91a52005-06-21 17:17:12 -07001171
Linus Torvalds1da177e2005-04-16 15:20:36 -07001172 /*
1173 * Register the new request and wait if the reconstruction
1174 * thread has put up a bar for new requests.
1175 * Continue immediately if no resync is active currently.
1176 */
NeilBrown0a27ec92006-01-06 00:20:13 -08001177 wait_barrier(conf);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001178
NeilBrown3ea7daa2012-05-22 13:53:47 +10001179 sectors = bio->bi_size >> 9;
1180 while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1181 bio->bi_sector < conf->reshape_progress &&
1182 bio->bi_sector + sectors > conf->reshape_progress) {
1183 /* IO spans the reshape position. Need to wait for
1184 * reshape to pass
1185 */
1186 allow_barrier(conf);
1187 wait_event(conf->wait_barrier,
1188 conf->reshape_progress <= bio->bi_sector ||
1189 conf->reshape_progress >= bio->bi_sector + sectors);
1190 wait_barrier(conf);
1191 }
1192 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1193 bio_data_dir(bio) == WRITE &&
1194 (mddev->reshape_backwards
1195 ? (bio->bi_sector < conf->reshape_safe &&
1196 bio->bi_sector + sectors > conf->reshape_progress)
1197 : (bio->bi_sector + sectors > conf->reshape_safe &&
1198 bio->bi_sector < conf->reshape_progress))) {
1199 /* Need to update reshape_position in metadata */
1200 mddev->reshape_position = conf->reshape_progress;
1201 set_bit(MD_CHANGE_DEVS, &mddev->flags);
1202 set_bit(MD_CHANGE_PENDING, &mddev->flags);
1203 md_wakeup_thread(mddev->thread);
1204 wait_event(mddev->sb_wait,
1205 !test_bit(MD_CHANGE_PENDING, &mddev->flags));
1206
1207 conf->reshape_safe = mddev->reshape_position;
1208 }
1209
Linus Torvalds1da177e2005-04-16 15:20:36 -07001210 r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
1211
1212 r10_bio->master_bio = bio;
NeilBrown3ea7daa2012-05-22 13:53:47 +10001213 r10_bio->sectors = sectors;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001214
1215 r10_bio->mddev = mddev;
1216 r10_bio->sector = bio->bi_sector;
NeilBrown6cce3b232006-01-06 00:20:16 -08001217 r10_bio->state = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001218
NeilBrown856e08e2011-07-28 11:39:23 +10001219 /* We might need to issue multiple reads to different
1220 * devices if there are bad blocks around, so we keep
1221 * track of the number of reads in bio->bi_phys_segments.
1222 * If this is 0, there is only one r10_bio and no locking
1223 * will be needed when the request completes. If it is
1224 * non-zero, then it is the number of not-completed requests.
1225 */
1226 bio->bi_phys_segments = 0;
1227 clear_bit(BIO_SEG_VALID, &bio->bi_flags);
1228
Jens Axboea3623572005-11-01 09:26:16 +01001229 if (rw == READ) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230 /*
1231 * read balancing logic:
1232 */
NeilBrown96c3fd12011-12-23 10:17:54 +11001233 struct md_rdev *rdev;
NeilBrown856e08e2011-07-28 11:39:23 +10001234 int slot;
1235
1236read_again:
NeilBrown96c3fd12011-12-23 10:17:54 +11001237 rdev = read_balance(conf, r10_bio, &max_sectors);
1238 if (!rdev) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001239 raid_end_bio_io(r10_bio);
Christoph Hellwig5a7bbad2011-09-12 12:12:01 +02001240 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001241 }
NeilBrown96c3fd12011-12-23 10:17:54 +11001242 slot = r10_bio->read_slot;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001243
NeilBrowna167f662010-10-26 18:31:13 +11001244 read_bio = bio_clone_mddev(bio, GFP_NOIO, mddev);
NeilBrown856e08e2011-07-28 11:39:23 +10001245 md_trim_bio(read_bio, r10_bio->sector - bio->bi_sector,
1246 max_sectors);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001247
1248 r10_bio->devs[slot].bio = read_bio;
NeilBrownabbf0982011-12-23 10:17:54 +11001249 r10_bio->devs[slot].rdev = rdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001250
1251 read_bio->bi_sector = r10_bio->devs[slot].addr +
NeilBrownf8c9e742012-05-21 09:28:33 +10001252 choose_data_offset(r10_bio, rdev);
NeilBrown96c3fd12011-12-23 10:17:54 +11001253 read_bio->bi_bdev = rdev->bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001254 read_bio->bi_end_io = raid10_end_read_request;
Christoph Hellwig7b6d91d2010-08-07 18:20:39 +02001255 read_bio->bi_rw = READ | do_sync;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001256 read_bio->bi_private = r10_bio;
1257
NeilBrown856e08e2011-07-28 11:39:23 +10001258 if (max_sectors < r10_bio->sectors) {
1259 /* Could not read all from this device, so we will
1260 * need another r10_bio.
1261 */
NeilBrown856e08e2011-07-28 11:39:23 +10001262 sectors_handled = (r10_bio->sectors + max_sectors
1263 - bio->bi_sector);
1264 r10_bio->sectors = max_sectors;
1265 spin_lock_irq(&conf->device_lock);
1266 if (bio->bi_phys_segments == 0)
1267 bio->bi_phys_segments = 2;
1268 else
1269 bio->bi_phys_segments++;
1270 spin_unlock(&conf->device_lock);
1271 /* Cannot call generic_make_request directly
1272 * as that will be queued in __generic_make_request
1273 * and subsequent mempool_alloc might block
1274 * waiting for it. so hand bio over to raid10d.
1275 */
1276 reschedule_retry(r10_bio);
1277
1278 r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
1279
1280 r10_bio->master_bio = bio;
1281 r10_bio->sectors = ((bio->bi_size >> 9)
1282 - sectors_handled);
1283 r10_bio->state = 0;
1284 r10_bio->mddev = mddev;
1285 r10_bio->sector = bio->bi_sector + sectors_handled;
1286 goto read_again;
1287 } else
1288 generic_make_request(read_bio);
Christoph Hellwig5a7bbad2011-09-12 12:12:01 +02001289 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001290 }
1291
1292 /*
1293 * WRITE:
1294 */
NeilBrown34db0cd2011-10-11 16:50:01 +11001295 if (conf->pending_count >= max_queued_requests) {
1296 md_wakeup_thread(mddev->thread);
1297 wait_event(conf->wait_barrier,
1298 conf->pending_count < max_queued_requests);
1299 }
Dan Williams6bfe0b42008-04-30 00:52:32 -07001300 /* first select target devices under rcu_lock and
Linus Torvalds1da177e2005-04-16 15:20:36 -07001301 * inc refcount on their rdev. Record them by setting
1302 * bios[x] to bio
NeilBrownd4432c22011-07-28 11:39:24 +10001303 * If there are known/acknowledged bad blocks on any device
1304 * on which we have seen a write error, we want to avoid
1305 * writing to those blocks. This potentially requires several
1306 * writes to write around the bad blocks. Each set of writes
1307 * gets its own r10_bio with a set of bios attached. The number
1308 * of r10_bios is recored in bio->bi_phys_segments just as with
1309 * the read case.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001310 */
NeilBrownc3b328a2011-04-18 18:25:43 +10001311
NeilBrown69335ef2011-12-23 10:17:54 +11001312 r10_bio->read_slot = -1; /* make sure repl_bio gets freed */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001313 raid10_find_phys(conf, r10_bio);
NeilBrownd4432c22011-07-28 11:39:24 +10001314retry_write:
Harvey Harrisoncb6969e2008-05-06 20:42:32 -07001315 blocked_rdev = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001316 rcu_read_lock();
NeilBrownd4432c22011-07-28 11:39:24 +10001317 max_sectors = r10_bio->sectors;
1318
Linus Torvalds1da177e2005-04-16 15:20:36 -07001319 for (i = 0; i < conf->copies; i++) {
1320 int d = r10_bio->devs[i].devnum;
NeilBrown3cb03002011-10-11 16:45:26 +11001321 struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev);
NeilBrown475b0322011-12-23 10:17:55 +11001322 struct md_rdev *rrdev = rcu_dereference(
1323 conf->mirrors[d].replacement);
NeilBrown4ca40c22011-12-23 10:17:55 +11001324 if (rdev == rrdev)
1325 rrdev = NULL;
Dan Williams6bfe0b42008-04-30 00:52:32 -07001326 if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
1327 atomic_inc(&rdev->nr_pending);
1328 blocked_rdev = rdev;
1329 break;
1330 }
NeilBrown475b0322011-12-23 10:17:55 +11001331 if (rrdev && unlikely(test_bit(Blocked, &rrdev->flags))) {
1332 atomic_inc(&rrdev->nr_pending);
1333 blocked_rdev = rrdev;
1334 break;
1335 }
NeilBrown050b6612012-03-19 12:46:39 +11001336 if (rrdev && (test_bit(Faulty, &rrdev->flags)
1337 || test_bit(Unmerged, &rrdev->flags)))
NeilBrown475b0322011-12-23 10:17:55 +11001338 rrdev = NULL;
1339
NeilBrownd4432c22011-07-28 11:39:24 +10001340 r10_bio->devs[i].bio = NULL;
NeilBrown475b0322011-12-23 10:17:55 +11001341 r10_bio->devs[i].repl_bio = NULL;
NeilBrown050b6612012-03-19 12:46:39 +11001342 if (!rdev || test_bit(Faulty, &rdev->flags) ||
1343 test_bit(Unmerged, &rdev->flags)) {
NeilBrown6cce3b232006-01-06 00:20:16 -08001344 set_bit(R10BIO_Degraded, &r10_bio->state);
NeilBrownd4432c22011-07-28 11:39:24 +10001345 continue;
NeilBrown6cce3b232006-01-06 00:20:16 -08001346 }
NeilBrownd4432c22011-07-28 11:39:24 +10001347 if (test_bit(WriteErrorSeen, &rdev->flags)) {
1348 sector_t first_bad;
1349 sector_t dev_sector = r10_bio->devs[i].addr;
1350 int bad_sectors;
1351 int is_bad;
1352
1353 is_bad = is_badblock(rdev, dev_sector,
1354 max_sectors,
1355 &first_bad, &bad_sectors);
1356 if (is_bad < 0) {
1357 /* Mustn't write here until the bad block
1358 * is acknowledged
1359 */
1360 atomic_inc(&rdev->nr_pending);
1361 set_bit(BlockedBadBlocks, &rdev->flags);
1362 blocked_rdev = rdev;
1363 break;
1364 }
1365 if (is_bad && first_bad <= dev_sector) {
1366 /* Cannot write here at all */
1367 bad_sectors -= (dev_sector - first_bad);
1368 if (bad_sectors < max_sectors)
1369 /* Mustn't write more than bad_sectors
1370 * to other devices yet
1371 */
1372 max_sectors = bad_sectors;
1373 /* We don't set R10BIO_Degraded as that
1374 * only applies if the disk is missing,
1375 * so it might be re-added, and we want to
1376 * know to recover this chunk.
1377 * In this case the device is here, and the
1378 * fact that this chunk is not in-sync is
1379 * recorded in the bad block log.
1380 */
1381 continue;
1382 }
1383 if (is_bad) {
1384 int good_sectors = first_bad - dev_sector;
1385 if (good_sectors < max_sectors)
1386 max_sectors = good_sectors;
1387 }
1388 }
1389 r10_bio->devs[i].bio = bio;
1390 atomic_inc(&rdev->nr_pending);
NeilBrown475b0322011-12-23 10:17:55 +11001391 if (rrdev) {
1392 r10_bio->devs[i].repl_bio = bio;
1393 atomic_inc(&rrdev->nr_pending);
1394 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001395 }
1396 rcu_read_unlock();
1397
Dan Williams6bfe0b42008-04-30 00:52:32 -07001398 if (unlikely(blocked_rdev)) {
1399 /* Have to wait for this device to get unblocked, then retry */
1400 int j;
1401 int d;
1402
NeilBrown475b0322011-12-23 10:17:55 +11001403 for (j = 0; j < i; j++) {
Dan Williams6bfe0b42008-04-30 00:52:32 -07001404 if (r10_bio->devs[j].bio) {
1405 d = r10_bio->devs[j].devnum;
1406 rdev_dec_pending(conf->mirrors[d].rdev, mddev);
1407 }
NeilBrown475b0322011-12-23 10:17:55 +11001408 if (r10_bio->devs[j].repl_bio) {
NeilBrown4ca40c22011-12-23 10:17:55 +11001409 struct md_rdev *rdev;
NeilBrown475b0322011-12-23 10:17:55 +11001410 d = r10_bio->devs[j].devnum;
NeilBrown4ca40c22011-12-23 10:17:55 +11001411 rdev = conf->mirrors[d].replacement;
1412 if (!rdev) {
1413 /* Race with remove_disk */
1414 smp_mb();
1415 rdev = conf->mirrors[d].rdev;
1416 }
1417 rdev_dec_pending(rdev, mddev);
NeilBrown475b0322011-12-23 10:17:55 +11001418 }
1419 }
Dan Williams6bfe0b42008-04-30 00:52:32 -07001420 allow_barrier(conf);
1421 md_wait_for_blocked_rdev(blocked_rdev, mddev);
1422 wait_barrier(conf);
1423 goto retry_write;
1424 }
1425
NeilBrownd4432c22011-07-28 11:39:24 +10001426 if (max_sectors < r10_bio->sectors) {
1427 /* We are splitting this into multiple parts, so
1428 * we need to prepare for allocating another r10_bio.
1429 */
1430 r10_bio->sectors = max_sectors;
1431 spin_lock_irq(&conf->device_lock);
1432 if (bio->bi_phys_segments == 0)
1433 bio->bi_phys_segments = 2;
1434 else
1435 bio->bi_phys_segments++;
1436 spin_unlock_irq(&conf->device_lock);
1437 }
1438 sectors_handled = r10_bio->sector + max_sectors - bio->bi_sector;
1439
NeilBrown4e780642010-10-19 12:54:01 +11001440 atomic_set(&r10_bio->remaining, 1);
NeilBrownd4432c22011-07-28 11:39:24 +10001441 bitmap_startwrite(mddev->bitmap, r10_bio->sector, r10_bio->sectors, 0);
NeilBrown06d91a52005-06-21 17:17:12 -07001442
Linus Torvalds1da177e2005-04-16 15:20:36 -07001443 for (i = 0; i < conf->copies; i++) {
1444 struct bio *mbio;
1445 int d = r10_bio->devs[i].devnum;
1446 if (!r10_bio->devs[i].bio)
1447 continue;
1448
NeilBrowna167f662010-10-26 18:31:13 +11001449 mbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
NeilBrownd4432c22011-07-28 11:39:24 +10001450 md_trim_bio(mbio, r10_bio->sector - bio->bi_sector,
1451 max_sectors);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001452 r10_bio->devs[i].bio = mbio;
1453
NeilBrownd4432c22011-07-28 11:39:24 +10001454 mbio->bi_sector = (r10_bio->devs[i].addr+
NeilBrownf8c9e742012-05-21 09:28:33 +10001455 choose_data_offset(r10_bio,
1456 conf->mirrors[d].rdev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001457 mbio->bi_bdev = conf->mirrors[d].rdev->bdev;
1458 mbio->bi_end_io = raid10_end_write_request;
Shaohua Li532a2a32012-10-11 13:30:52 +11001459 mbio->bi_rw = WRITE | do_sync | do_fua | do_discard;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001460 mbio->bi_private = r10_bio;
1461
1462 atomic_inc(&r10_bio->remaining);
NeilBrown57c67df2012-10-11 13:32:13 +11001463
1464 cb = blk_check_plugged(raid10_unplug, mddev, sizeof(*plug));
1465 if (cb)
1466 plug = container_of(cb, struct raid10_plug_cb, cb);
1467 else
1468 plug = NULL;
NeilBrown4e780642010-10-19 12:54:01 +11001469 spin_lock_irqsave(&conf->device_lock, flags);
NeilBrown57c67df2012-10-11 13:32:13 +11001470 if (plug) {
1471 bio_list_add(&plug->pending, mbio);
1472 plug->pending_cnt++;
1473 } else {
1474 bio_list_add(&conf->pending_bio_list, mbio);
1475 conf->pending_count++;
1476 }
NeilBrown4e780642010-10-19 12:54:01 +11001477 spin_unlock_irqrestore(&conf->device_lock, flags);
NeilBrown57c67df2012-10-11 13:32:13 +11001478 if (!plug)
NeilBrownb357f042012-07-03 17:45:31 +10001479 md_wakeup_thread(mddev->thread);
NeilBrown475b0322011-12-23 10:17:55 +11001480
1481 if (!r10_bio->devs[i].repl_bio)
1482 continue;
1483
1484 mbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1485 md_trim_bio(mbio, r10_bio->sector - bio->bi_sector,
1486 max_sectors);
1487 r10_bio->devs[i].repl_bio = mbio;
1488
NeilBrown4ca40c22011-12-23 10:17:55 +11001489 /* We are actively writing to the original device
1490 * so it cannot disappear, so the replacement cannot
1491 * become NULL here
1492 */
NeilBrownf8c9e742012-05-21 09:28:33 +10001493 mbio->bi_sector = (r10_bio->devs[i].addr +
1494 choose_data_offset(
1495 r10_bio,
1496 conf->mirrors[d].replacement));
NeilBrown475b0322011-12-23 10:17:55 +11001497 mbio->bi_bdev = conf->mirrors[d].replacement->bdev;
1498 mbio->bi_end_io = raid10_end_write_request;
Shaohua Li532a2a32012-10-11 13:30:52 +11001499 mbio->bi_rw = WRITE | do_sync | do_fua | do_discard;
NeilBrown475b0322011-12-23 10:17:55 +11001500 mbio->bi_private = r10_bio;
1501
1502 atomic_inc(&r10_bio->remaining);
1503 spin_lock_irqsave(&conf->device_lock, flags);
1504 bio_list_add(&conf->pending_bio_list, mbio);
1505 conf->pending_count++;
1506 spin_unlock_irqrestore(&conf->device_lock, flags);
NeilBrownb357f042012-07-03 17:45:31 +10001507 if (!mddev_check_plugged(mddev))
1508 md_wakeup_thread(mddev->thread);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001509 }
1510
NeilBrown079fa162011-09-10 17:21:23 +10001511 /* Don't remove the bias on 'remaining' (one_write_done) until
1512 * after checking if we need to go around again.
1513 */
NeilBrowna35e63e2008-03-04 14:29:29 -08001514
NeilBrownd4432c22011-07-28 11:39:24 +10001515 if (sectors_handled < (bio->bi_size >> 9)) {
NeilBrown079fa162011-09-10 17:21:23 +10001516 one_write_done(r10_bio);
NeilBrown5e570282011-07-28 11:39:25 +10001517 /* We need another r10_bio. It has already been counted
NeilBrownd4432c22011-07-28 11:39:24 +10001518 * in bio->bi_phys_segments.
1519 */
1520 r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
1521
1522 r10_bio->master_bio = bio;
1523 r10_bio->sectors = (bio->bi_size >> 9) - sectors_handled;
1524
1525 r10_bio->mddev = mddev;
1526 r10_bio->sector = bio->bi_sector + sectors_handled;
1527 r10_bio->state = 0;
1528 goto retry_write;
1529 }
NeilBrown079fa162011-09-10 17:21:23 +10001530 one_write_done(r10_bio);
1531
1532 /* In case raid10d snuck in to freeze_array */
1533 wake_up(&conf->wait_barrier);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001534}
1535
NeilBrownfd01b882011-10-11 16:47:53 +11001536static void status(struct seq_file *seq, struct mddev *mddev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001537{
NeilBrowne879a872011-10-11 16:49:02 +11001538 struct r10conf *conf = mddev->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001539 int i;
1540
NeilBrown5cf00fc2012-05-21 09:28:20 +10001541 if (conf->geo.near_copies < conf->geo.raid_disks)
Andre Noll9d8f0362009-06-18 08:45:01 +10001542 seq_printf(seq, " %dK chunks", mddev->chunk_sectors / 2);
NeilBrown5cf00fc2012-05-21 09:28:20 +10001543 if (conf->geo.near_copies > 1)
1544 seq_printf(seq, " %d near-copies", conf->geo.near_copies);
1545 if (conf->geo.far_copies > 1) {
1546 if (conf->geo.far_offset)
1547 seq_printf(seq, " %d offset-copies", conf->geo.far_copies);
NeilBrownc93983b2006-06-26 00:27:41 -07001548 else
NeilBrown5cf00fc2012-05-21 09:28:20 +10001549 seq_printf(seq, " %d far-copies", conf->geo.far_copies);
NeilBrownc93983b2006-06-26 00:27:41 -07001550 }
NeilBrown5cf00fc2012-05-21 09:28:20 +10001551 seq_printf(seq, " [%d/%d] [", conf->geo.raid_disks,
1552 conf->geo.raid_disks - mddev->degraded);
1553 for (i = 0; i < conf->geo.raid_disks; i++)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001554 seq_printf(seq, "%s",
1555 conf->mirrors[i].rdev &&
NeilBrownb2d444d2005-11-08 21:39:31 -08001556 test_bit(In_sync, &conf->mirrors[i].rdev->flags) ? "U" : "_");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001557 seq_printf(seq, "]");
1558}
1559
NeilBrown700c7212011-07-27 11:00:36 +10001560/* check if there are enough drives for
1561 * every block to appear on atleast one.
1562 * Don't consider the device numbered 'ignore'
1563 * as we might be about to remove it.
1564 */
NeilBrownf8c9e742012-05-21 09:28:33 +10001565static int _enough(struct r10conf *conf, struct geom *geo, int ignore)
NeilBrown700c7212011-07-27 11:00:36 +10001566{
1567 int first = 0;
1568
1569 do {
1570 int n = conf->copies;
1571 int cnt = 0;
NeilBrown80b48122012-09-27 12:35:21 +10001572 int this = first;
NeilBrown700c7212011-07-27 11:00:36 +10001573 while (n--) {
NeilBrown80b48122012-09-27 12:35:21 +10001574 if (conf->mirrors[this].rdev &&
1575 this != ignore)
NeilBrown700c7212011-07-27 11:00:36 +10001576 cnt++;
NeilBrown80b48122012-09-27 12:35:21 +10001577 this = (this+1) % geo->raid_disks;
NeilBrown700c7212011-07-27 11:00:36 +10001578 }
1579 if (cnt == 0)
1580 return 0;
NeilBrown80b48122012-09-27 12:35:21 +10001581 first = (first + geo->near_copies) % geo->raid_disks;
NeilBrown700c7212011-07-27 11:00:36 +10001582 } while (first != 0);
1583 return 1;
1584}
1585
NeilBrownf8c9e742012-05-21 09:28:33 +10001586static int enough(struct r10conf *conf, int ignore)
1587{
1588 return _enough(conf, &conf->geo, ignore) &&
1589 _enough(conf, &conf->prev, ignore);
1590}
1591
NeilBrownfd01b882011-10-11 16:47:53 +11001592static void error(struct mddev *mddev, struct md_rdev *rdev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001593{
1594 char b[BDEVNAME_SIZE];
NeilBrowne879a872011-10-11 16:49:02 +11001595 struct r10conf *conf = mddev->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001596
1597 /*
1598 * If it is not operational, then we have already marked it as dead
1599 * else if it is the last working disks, ignore the error, let the
1600 * next level up know.
1601 * else mark the drive as failed
1602 */
NeilBrownb2d444d2005-11-08 21:39:31 -08001603 if (test_bit(In_sync, &rdev->flags)
NeilBrown700c7212011-07-27 11:00:36 +10001604 && !enough(conf, rdev->raid_disk))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001605 /*
1606 * Don't fail the drive, just return an IO error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001607 */
1608 return;
NeilBrownc04be0a2006-10-03 01:15:53 -07001609 if (test_and_clear_bit(In_sync, &rdev->flags)) {
1610 unsigned long flags;
1611 spin_lock_irqsave(&conf->device_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001612 mddev->degraded++;
NeilBrownc04be0a2006-10-03 01:15:53 -07001613 spin_unlock_irqrestore(&conf->device_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001614 /*
1615 * if recovery is running, make sure it aborts.
1616 */
NeilBrowndfc70642008-05-23 13:04:39 -07001617 set_bit(MD_RECOVERY_INTR, &mddev->recovery);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001618 }
NeilBrownde393cd2011-07-28 11:31:48 +10001619 set_bit(Blocked, &rdev->flags);
NeilBrownb2d444d2005-11-08 21:39:31 -08001620 set_bit(Faulty, &rdev->flags);
NeilBrown850b2b42006-10-03 01:15:46 -07001621 set_bit(MD_CHANGE_DEVS, &mddev->flags);
Joe Perches067032b2011-01-14 09:14:33 +11001622 printk(KERN_ALERT
1623 "md/raid10:%s: Disk failure on %s, disabling device.\n"
1624 "md/raid10:%s: Operation continuing on %d devices.\n",
NeilBrown128595e2010-05-03 14:47:14 +10001625 mdname(mddev), bdevname(rdev->bdev, b),
NeilBrown5cf00fc2012-05-21 09:28:20 +10001626 mdname(mddev), conf->geo.raid_disks - mddev->degraded);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001627}
1628
NeilBrowne879a872011-10-11 16:49:02 +11001629static void print_conf(struct r10conf *conf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001630{
1631 int i;
Jonathan Brassowdc280d982012-07-31 10:03:52 +10001632 struct raid10_info *tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001633
NeilBrown128595e2010-05-03 14:47:14 +10001634 printk(KERN_DEBUG "RAID10 conf printout:\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001635 if (!conf) {
NeilBrown128595e2010-05-03 14:47:14 +10001636 printk(KERN_DEBUG "(!conf)\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001637 return;
1638 }
NeilBrown5cf00fc2012-05-21 09:28:20 +10001639 printk(KERN_DEBUG " --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded,
1640 conf->geo.raid_disks);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001641
NeilBrown5cf00fc2012-05-21 09:28:20 +10001642 for (i = 0; i < conf->geo.raid_disks; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001643 char b[BDEVNAME_SIZE];
1644 tmp = conf->mirrors + i;
1645 if (tmp->rdev)
NeilBrown128595e2010-05-03 14:47:14 +10001646 printk(KERN_DEBUG " disk %d, wo:%d, o:%d, dev:%s\n",
NeilBrownb2d444d2005-11-08 21:39:31 -08001647 i, !test_bit(In_sync, &tmp->rdev->flags),
1648 !test_bit(Faulty, &tmp->rdev->flags),
Linus Torvalds1da177e2005-04-16 15:20:36 -07001649 bdevname(tmp->rdev->bdev,b));
1650 }
1651}
1652
NeilBrowne879a872011-10-11 16:49:02 +11001653static void close_sync(struct r10conf *conf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001654{
NeilBrown0a27ec92006-01-06 00:20:13 -08001655 wait_barrier(conf);
1656 allow_barrier(conf);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001657
1658 mempool_destroy(conf->r10buf_pool);
1659 conf->r10buf_pool = NULL;
1660}
1661
NeilBrownfd01b882011-10-11 16:47:53 +11001662static int raid10_spare_active(struct mddev *mddev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001663{
1664 int i;
NeilBrowne879a872011-10-11 16:49:02 +11001665 struct r10conf *conf = mddev->private;
Jonathan Brassowdc280d982012-07-31 10:03:52 +10001666 struct raid10_info *tmp;
NeilBrown6b965622010-08-18 11:56:59 +10001667 int count = 0;
1668 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001669
1670 /*
1671 * Find all non-in_sync disks within the RAID10 configuration
1672 * and mark them in_sync
1673 */
NeilBrown5cf00fc2012-05-21 09:28:20 +10001674 for (i = 0; i < conf->geo.raid_disks; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001675 tmp = conf->mirrors + i;
NeilBrown4ca40c22011-12-23 10:17:55 +11001676 if (tmp->replacement
1677 && tmp->replacement->recovery_offset == MaxSector
1678 && !test_bit(Faulty, &tmp->replacement->flags)
1679 && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
1680 /* Replacement has just become active */
1681 if (!tmp->rdev
1682 || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
1683 count++;
1684 if (tmp->rdev) {
1685 /* Replaced device not technically faulty,
1686 * but we need to be sure it gets removed
1687 * and never re-added.
1688 */
1689 set_bit(Faulty, &tmp->rdev->flags);
1690 sysfs_notify_dirent_safe(
1691 tmp->rdev->sysfs_state);
1692 }
1693 sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
1694 } else if (tmp->rdev
1695 && !test_bit(Faulty, &tmp->rdev->flags)
1696 && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
NeilBrown6b965622010-08-18 11:56:59 +10001697 count++;
Jonathan Brassow2863b9e2012-10-11 13:38:58 +11001698 sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001699 }
1700 }
NeilBrown6b965622010-08-18 11:56:59 +10001701 spin_lock_irqsave(&conf->device_lock, flags);
1702 mddev->degraded -= count;
1703 spin_unlock_irqrestore(&conf->device_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001704
1705 print_conf(conf);
NeilBrown6b965622010-08-18 11:56:59 +10001706 return count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001707}
1708
1709
NeilBrownfd01b882011-10-11 16:47:53 +11001710static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001711{
NeilBrowne879a872011-10-11 16:49:02 +11001712 struct r10conf *conf = mddev->private;
Neil Brown199050e2008-06-28 08:31:33 +10001713 int err = -EEXIST;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001714 int mirror;
Neil Brown6c2fce22008-06-28 08:31:31 +10001715 int first = 0;
NeilBrown5cf00fc2012-05-21 09:28:20 +10001716 int last = conf->geo.raid_disks - 1;
NeilBrown050b6612012-03-19 12:46:39 +11001717 struct request_queue *q = bdev_get_queue(rdev->bdev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001718
1719 if (mddev->recovery_cp < MaxSector)
1720 /* only hot-add to in-sync arrays, as recovery is
1721 * very different from resync
1722 */
Neil Brown199050e2008-06-28 08:31:33 +10001723 return -EBUSY;
NeilBrownf8c9e742012-05-21 09:28:33 +10001724 if (rdev->saved_raid_disk < 0 && !_enough(conf, &conf->prev, -1))
Neil Brown199050e2008-06-28 08:31:33 +10001725 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001726
NeilBrowna53a6c82008-11-06 17:28:20 +11001727 if (rdev->raid_disk >= 0)
Neil Brown6c2fce22008-06-28 08:31:31 +10001728 first = last = rdev->raid_disk;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001729
NeilBrown050b6612012-03-19 12:46:39 +11001730 if (q->merge_bvec_fn) {
1731 set_bit(Unmerged, &rdev->flags);
1732 mddev->merge_check_needed = 1;
1733 }
1734
Namhyung Kim2c4193d2011-07-18 17:38:43 +10001735 if (rdev->saved_raid_disk >= first &&
NeilBrown6cce3b232006-01-06 00:20:16 -08001736 conf->mirrors[rdev->saved_raid_disk].rdev == NULL)
1737 mirror = rdev->saved_raid_disk;
1738 else
Neil Brown6c2fce22008-06-28 08:31:31 +10001739 mirror = first;
NeilBrown2bb77732011-07-27 11:00:36 +10001740 for ( ; mirror <= last ; mirror++) {
Jonathan Brassowdc280d982012-07-31 10:03:52 +10001741 struct raid10_info *p = &conf->mirrors[mirror];
NeilBrown2bb77732011-07-27 11:00:36 +10001742 if (p->recovery_disabled == mddev->recovery_disabled)
1743 continue;
NeilBrownb7044d42011-12-23 10:17:56 +11001744 if (p->rdev) {
1745 if (!test_bit(WantReplacement, &p->rdev->flags) ||
1746 p->replacement != NULL)
1747 continue;
1748 clear_bit(In_sync, &rdev->flags);
1749 set_bit(Replacement, &rdev->flags);
1750 rdev->raid_disk = mirror;
1751 err = 0;
1752 disk_stack_limits(mddev->gendisk, rdev->bdev,
1753 rdev->data_offset << 9);
NeilBrownb7044d42011-12-23 10:17:56 +11001754 conf->fullsync = 1;
1755 rcu_assign_pointer(p->replacement, rdev);
1756 break;
1757 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001758
NeilBrown2bb77732011-07-27 11:00:36 +10001759 disk_stack_limits(mddev->gendisk, rdev->bdev,
1760 rdev->data_offset << 9);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001761
NeilBrown2bb77732011-07-27 11:00:36 +10001762 p->head_position = 0;
NeilBrownd890fa22011-10-26 11:54:39 +11001763 p->recovery_disabled = mddev->recovery_disabled - 1;
NeilBrown2bb77732011-07-27 11:00:36 +10001764 rdev->raid_disk = mirror;
1765 err = 0;
1766 if (rdev->saved_raid_disk != mirror)
1767 conf->fullsync = 1;
1768 rcu_assign_pointer(p->rdev, rdev);
1769 break;
1770 }
NeilBrown050b6612012-03-19 12:46:39 +11001771 if (err == 0 && test_bit(Unmerged, &rdev->flags)) {
1772 /* Some requests might not have seen this new
1773 * merge_bvec_fn. We must wait for them to complete
1774 * before merging the device fully.
1775 * First we make sure any code which has tested
1776 * our function has submitted the request, then
1777 * we wait for all outstanding requests to complete.
1778 */
1779 synchronize_sched();
1780 raise_barrier(conf, 0);
1781 lower_barrier(conf);
1782 clear_bit(Unmerged, &rdev->flags);
1783 }
Andre Nollac5e7112009-08-03 10:59:47 +10001784 md_integrity_add_rdev(rdev, mddev);
Shaohua Li532a2a32012-10-11 13:30:52 +11001785 if (blk_queue_discard(bdev_get_queue(rdev->bdev)))
1786 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue);
1787
Linus Torvalds1da177e2005-04-16 15:20:36 -07001788 print_conf(conf);
Neil Brown199050e2008-06-28 08:31:33 +10001789 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001790}
1791
NeilBrownb8321b62011-12-23 10:17:51 +11001792static int raid10_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001793{
NeilBrowne879a872011-10-11 16:49:02 +11001794 struct r10conf *conf = mddev->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001795 int err = 0;
NeilBrownb8321b62011-12-23 10:17:51 +11001796 int number = rdev->raid_disk;
NeilBrownc8ab9032011-12-23 10:17:54 +11001797 struct md_rdev **rdevp;
Jonathan Brassowdc280d982012-07-31 10:03:52 +10001798 struct raid10_info *p = conf->mirrors + number;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001799
1800 print_conf(conf);
NeilBrownc8ab9032011-12-23 10:17:54 +11001801 if (rdev == p->rdev)
1802 rdevp = &p->rdev;
1803 else if (rdev == p->replacement)
1804 rdevp = &p->replacement;
1805 else
1806 return 0;
1807
1808 if (test_bit(In_sync, &rdev->flags) ||
1809 atomic_read(&rdev->nr_pending)) {
1810 err = -EBUSY;
1811 goto abort;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001812 }
NeilBrownc8ab9032011-12-23 10:17:54 +11001813 /* Only remove faulty devices if recovery
1814 * is not possible.
1815 */
1816 if (!test_bit(Faulty, &rdev->flags) &&
1817 mddev->recovery_disabled != p->recovery_disabled &&
NeilBrown4ca40c22011-12-23 10:17:55 +11001818 (!p->replacement || p->replacement == rdev) &&
NeilBrown63aced62012-05-22 13:55:33 +10001819 number < conf->geo.raid_disks &&
NeilBrownc8ab9032011-12-23 10:17:54 +11001820 enough(conf, -1)) {
1821 err = -EBUSY;
1822 goto abort;
1823 }
1824 *rdevp = NULL;
1825 synchronize_rcu();
1826 if (atomic_read(&rdev->nr_pending)) {
1827 /* lost the race, try later */
1828 err = -EBUSY;
1829 *rdevp = rdev;
1830 goto abort;
NeilBrown4ca40c22011-12-23 10:17:55 +11001831 } else if (p->replacement) {
1832 /* We must have just cleared 'rdev' */
1833 p->rdev = p->replacement;
1834 clear_bit(Replacement, &p->replacement->flags);
1835 smp_mb(); /* Make sure other CPUs may see both as identical
1836 * but will never see neither -- if they are careful.
1837 */
1838 p->replacement = NULL;
1839 clear_bit(WantReplacement, &rdev->flags);
1840 } else
1841 /* We might have just remove the Replacement as faulty
1842 * Clear the flag just in case
1843 */
1844 clear_bit(WantReplacement, &rdev->flags);
1845
NeilBrownc8ab9032011-12-23 10:17:54 +11001846 err = md_integrity_register(mddev);
1847
Linus Torvalds1da177e2005-04-16 15:20:36 -07001848abort:
1849
1850 print_conf(conf);
1851 return err;
1852}
1853
1854
NeilBrown6712ecf2007-09-27 12:47:43 +02001855static void end_sync_read(struct bio *bio, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001856{
NeilBrown9f2c9d12011-10-11 16:48:43 +11001857 struct r10bio *r10_bio = bio->bi_private;
NeilBrowne879a872011-10-11 16:49:02 +11001858 struct r10conf *conf = r10_bio->mddev->private;
Namhyung Kim778ca012011-07-18 17:38:47 +10001859 int d;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001860
NeilBrown3ea7daa2012-05-22 13:53:47 +10001861 if (bio == r10_bio->master_bio) {
1862 /* this is a reshape read */
1863 d = r10_bio->read_slot; /* really the read dev */
1864 } else
1865 d = find_bio_disk(conf, r10_bio, bio, NULL, NULL);
NeilBrown0eb3ff12006-01-06 00:20:29 -08001866
1867 if (test_bit(BIO_UPTODATE, &bio->bi_flags))
1868 set_bit(R10BIO_Uptodate, &r10_bio->state);
NeilBrowne684e412011-07-28 11:39:25 +10001869 else
1870 /* The write handler will notice the lack of
1871 * R10BIO_Uptodate and record any errors etc
1872 */
NeilBrown4dbcdc72006-01-06 00:20:52 -08001873 atomic_add(r10_bio->sectors,
1874 &conf->mirrors[d].rdev->corrected_errors);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001875
1876 /* for reconstruct, we always reschedule after a read.
1877 * for resync, only after all reads
1878 */
NeilBrown73d5c382009-02-25 13:18:47 +11001879 rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001880 if (test_bit(R10BIO_IsRecover, &r10_bio->state) ||
1881 atomic_dec_and_test(&r10_bio->remaining)) {
1882 /* we have read all the blocks,
1883 * do the comparison in process context in raid10d
1884 */
1885 reschedule_retry(r10_bio);
1886 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001887}
1888
NeilBrown9f2c9d12011-10-11 16:48:43 +11001889static void end_sync_request(struct r10bio *r10_bio)
NeilBrown5e570282011-07-28 11:39:25 +10001890{
NeilBrownfd01b882011-10-11 16:47:53 +11001891 struct mddev *mddev = r10_bio->mddev;
NeilBrown5e570282011-07-28 11:39:25 +10001892
1893 while (atomic_dec_and_test(&r10_bio->remaining)) {
1894 if (r10_bio->master_bio == NULL) {
1895 /* the primary of several recovery bios */
1896 sector_t s = r10_bio->sectors;
1897 if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
1898 test_bit(R10BIO_WriteError, &r10_bio->state))
1899 reschedule_retry(r10_bio);
1900 else
1901 put_buf(r10_bio);
1902 md_done_sync(mddev, s, 1);
1903 break;
1904 } else {
NeilBrown9f2c9d12011-10-11 16:48:43 +11001905 struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio;
NeilBrown5e570282011-07-28 11:39:25 +10001906 if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
1907 test_bit(R10BIO_WriteError, &r10_bio->state))
1908 reschedule_retry(r10_bio);
1909 else
1910 put_buf(r10_bio);
1911 r10_bio = r10_bio2;
1912 }
1913 }
1914}
1915
NeilBrown6712ecf2007-09-27 12:47:43 +02001916static void end_sync_write(struct bio *bio, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001917{
1918 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
NeilBrown9f2c9d12011-10-11 16:48:43 +11001919 struct r10bio *r10_bio = bio->bi_private;
NeilBrownfd01b882011-10-11 16:47:53 +11001920 struct mddev *mddev = r10_bio->mddev;
NeilBrowne879a872011-10-11 16:49:02 +11001921 struct r10conf *conf = mddev->private;
Namhyung Kim778ca012011-07-18 17:38:47 +10001922 int d;
NeilBrown749c55e2011-07-28 11:39:24 +10001923 sector_t first_bad;
1924 int bad_sectors;
1925 int slot;
NeilBrown9ad1aef2011-12-23 10:17:55 +11001926 int repl;
NeilBrown4ca40c22011-12-23 10:17:55 +11001927 struct md_rdev *rdev = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001928
NeilBrown9ad1aef2011-12-23 10:17:55 +11001929 d = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
1930 if (repl)
1931 rdev = conf->mirrors[d].replacement;
NeilBrown547414d2012-03-13 11:21:20 +11001932 else
NeilBrown9ad1aef2011-12-23 10:17:55 +11001933 rdev = conf->mirrors[d].rdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001934
NeilBrown1a0b7cd2011-07-28 11:39:25 +10001935 if (!uptodate) {
NeilBrown9ad1aef2011-12-23 10:17:55 +11001936 if (repl)
1937 md_error(mddev, rdev);
1938 else {
1939 set_bit(WriteErrorSeen, &rdev->flags);
NeilBrownb7044d42011-12-23 10:17:56 +11001940 if (!test_and_set_bit(WantReplacement, &rdev->flags))
1941 set_bit(MD_RECOVERY_NEEDED,
1942 &rdev->mddev->recovery);
NeilBrown9ad1aef2011-12-23 10:17:55 +11001943 set_bit(R10BIO_WriteError, &r10_bio->state);
1944 }
1945 } else if (is_badblock(rdev,
NeilBrown749c55e2011-07-28 11:39:24 +10001946 r10_bio->devs[slot].addr,
1947 r10_bio->sectors,
1948 &first_bad, &bad_sectors))
1949 set_bit(R10BIO_MadeGood, &r10_bio->state);
NeilBrowndfc70642008-05-23 13:04:39 -07001950
NeilBrown9ad1aef2011-12-23 10:17:55 +11001951 rdev_dec_pending(rdev, mddev);
NeilBrown5e570282011-07-28 11:39:25 +10001952
1953 end_sync_request(r10_bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001954}
1955
1956/*
1957 * Note: sync and recover and handled very differently for raid10
1958 * This code is for resync.
1959 * For resync, we read through virtual addresses and read all blocks.
1960 * If there is any error, we schedule a write. The lowest numbered
1961 * drive is authoritative.
1962 * However requests come for physical address, so we need to map.
1963 * For every physical address there are raid_disks/copies virtual addresses,
1964 * which is always are least one, but is not necessarly an integer.
1965 * This means that a physical address can span multiple chunks, so we may
1966 * have to submit multiple io requests for a single sync request.
1967 */
1968/*
1969 * We check if all blocks are in-sync and only write to blocks that
1970 * aren't in sync
1971 */
NeilBrown9f2c9d12011-10-11 16:48:43 +11001972static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001973{
NeilBrowne879a872011-10-11 16:49:02 +11001974 struct r10conf *conf = mddev->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001975 int i, first;
1976 struct bio *tbio, *fbio;
majianpengf4380a92012-04-12 16:04:47 +10001977 int vcnt;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001978
1979 atomic_set(&r10_bio->remaining, 1);
1980
1981 /* find the first device with a block */
1982 for (i=0; i<conf->copies; i++)
1983 if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags))
1984 break;
1985
1986 if (i == conf->copies)
1987 goto done;
1988
1989 first = i;
1990 fbio = r10_bio->devs[i].bio;
1991
majianpengf4380a92012-04-12 16:04:47 +10001992 vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001993 /* now find blocks with errors */
NeilBrown0eb3ff12006-01-06 00:20:29 -08001994 for (i=0 ; i < conf->copies ; i++) {
1995 int j, d;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001996
Linus Torvalds1da177e2005-04-16 15:20:36 -07001997 tbio = r10_bio->devs[i].bio;
NeilBrown0eb3ff12006-01-06 00:20:29 -08001998
1999 if (tbio->bi_end_io != end_sync_read)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002000 continue;
NeilBrown0eb3ff12006-01-06 00:20:29 -08002001 if (i == first)
2002 continue;
2003 if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags)) {
2004 /* We know that the bi_io_vec layout is the same for
2005 * both 'first' and 'i', so we just compare them.
2006 * All vec entries are PAGE_SIZE;
2007 */
2008 for (j = 0; j < vcnt; j++)
2009 if (memcmp(page_address(fbio->bi_io_vec[j].bv_page),
2010 page_address(tbio->bi_io_vec[j].bv_page),
NeilBrown5020ad72012-04-02 01:39:05 +10002011 fbio->bi_io_vec[j].bv_len))
NeilBrown0eb3ff12006-01-06 00:20:29 -08002012 break;
2013 if (j == vcnt)
2014 continue;
Jianpeng Ma7f7583d2012-10-11 14:17:59 +11002015 atomic64_add(r10_bio->sectors, &mddev->resync_mismatches);
NeilBrownf84ee362011-07-28 11:39:25 +10002016 if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery))
2017 /* Don't fix anything. */
2018 continue;
NeilBrown0eb3ff12006-01-06 00:20:29 -08002019 }
NeilBrownf84ee362011-07-28 11:39:25 +10002020 /* Ok, we need to write this bio, either to correct an
2021 * inconsistency or to correct an unreadable block.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002022 * First we need to fixup bv_offset, bv_len and
2023 * bi_vecs, as the read request might have corrupted these
2024 */
2025 tbio->bi_vcnt = vcnt;
2026 tbio->bi_size = r10_bio->sectors << 9;
2027 tbio->bi_idx = 0;
2028 tbio->bi_phys_segments = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002029 tbio->bi_flags &= ~(BIO_POOL_MASK - 1);
2030 tbio->bi_flags |= 1 << BIO_UPTODATE;
2031 tbio->bi_next = NULL;
2032 tbio->bi_rw = WRITE;
2033 tbio->bi_private = r10_bio;
2034 tbio->bi_sector = r10_bio->devs[i].addr;
2035
2036 for (j=0; j < vcnt ; j++) {
2037 tbio->bi_io_vec[j].bv_offset = 0;
2038 tbio->bi_io_vec[j].bv_len = PAGE_SIZE;
2039
2040 memcpy(page_address(tbio->bi_io_vec[j].bv_page),
2041 page_address(fbio->bi_io_vec[j].bv_page),
2042 PAGE_SIZE);
2043 }
2044 tbio->bi_end_io = end_sync_write;
2045
2046 d = r10_bio->devs[i].devnum;
2047 atomic_inc(&conf->mirrors[d].rdev->nr_pending);
2048 atomic_inc(&r10_bio->remaining);
2049 md_sync_acct(conf->mirrors[d].rdev->bdev, tbio->bi_size >> 9);
2050
2051 tbio->bi_sector += conf->mirrors[d].rdev->data_offset;
2052 tbio->bi_bdev = conf->mirrors[d].rdev->bdev;
2053 generic_make_request(tbio);
2054 }
2055
NeilBrown9ad1aef2011-12-23 10:17:55 +11002056 /* Now write out to any replacement devices
2057 * that are active
2058 */
2059 for (i = 0; i < conf->copies; i++) {
2060 int j, d;
NeilBrown9ad1aef2011-12-23 10:17:55 +11002061
2062 tbio = r10_bio->devs[i].repl_bio;
2063 if (!tbio || !tbio->bi_end_io)
2064 continue;
2065 if (r10_bio->devs[i].bio->bi_end_io != end_sync_write
2066 && r10_bio->devs[i].bio != fbio)
2067 for (j = 0; j < vcnt; j++)
2068 memcpy(page_address(tbio->bi_io_vec[j].bv_page),
2069 page_address(fbio->bi_io_vec[j].bv_page),
2070 PAGE_SIZE);
2071 d = r10_bio->devs[i].devnum;
2072 atomic_inc(&r10_bio->remaining);
2073 md_sync_acct(conf->mirrors[d].replacement->bdev,
2074 tbio->bi_size >> 9);
2075 generic_make_request(tbio);
2076 }
2077
Linus Torvalds1da177e2005-04-16 15:20:36 -07002078done:
2079 if (atomic_dec_and_test(&r10_bio->remaining)) {
2080 md_done_sync(mddev, r10_bio->sectors, 1);
2081 put_buf(r10_bio);
2082 }
2083}
2084
2085/*
2086 * Now for the recovery code.
2087 * Recovery happens across physical sectors.
2088 * We recover all non-is_sync drives by finding the virtual address of
2089 * each, and then choose a working drive that also has that virt address.
2090 * There is a separate r10_bio for each non-in_sync drive.
2091 * Only the first two slots are in use. The first for reading,
2092 * The second for writing.
2093 *
2094 */
NeilBrown9f2c9d12011-10-11 16:48:43 +11002095static void fix_recovery_read_error(struct r10bio *r10_bio)
NeilBrown5e570282011-07-28 11:39:25 +10002096{
2097 /* We got a read error during recovery.
2098 * We repeat the read in smaller page-sized sections.
2099 * If a read succeeds, write it to the new device or record
2100 * a bad block if we cannot.
2101 * If a read fails, record a bad block on both old and
2102 * new devices.
2103 */
NeilBrownfd01b882011-10-11 16:47:53 +11002104 struct mddev *mddev = r10_bio->mddev;
NeilBrowne879a872011-10-11 16:49:02 +11002105 struct r10conf *conf = mddev->private;
NeilBrown5e570282011-07-28 11:39:25 +10002106 struct bio *bio = r10_bio->devs[0].bio;
2107 sector_t sect = 0;
2108 int sectors = r10_bio->sectors;
2109 int idx = 0;
2110 int dr = r10_bio->devs[0].devnum;
2111 int dw = r10_bio->devs[1].devnum;
2112
2113 while (sectors) {
2114 int s = sectors;
NeilBrown3cb03002011-10-11 16:45:26 +11002115 struct md_rdev *rdev;
NeilBrown5e570282011-07-28 11:39:25 +10002116 sector_t addr;
2117 int ok;
2118
2119 if (s > (PAGE_SIZE>>9))
2120 s = PAGE_SIZE >> 9;
2121
2122 rdev = conf->mirrors[dr].rdev;
2123 addr = r10_bio->devs[0].addr + sect,
2124 ok = sync_page_io(rdev,
2125 addr,
2126 s << 9,
2127 bio->bi_io_vec[idx].bv_page,
2128 READ, false);
2129 if (ok) {
2130 rdev = conf->mirrors[dw].rdev;
2131 addr = r10_bio->devs[1].addr + sect;
2132 ok = sync_page_io(rdev,
2133 addr,
2134 s << 9,
2135 bio->bi_io_vec[idx].bv_page,
2136 WRITE, false);
NeilBrownb7044d42011-12-23 10:17:56 +11002137 if (!ok) {
NeilBrown5e570282011-07-28 11:39:25 +10002138 set_bit(WriteErrorSeen, &rdev->flags);
NeilBrownb7044d42011-12-23 10:17:56 +11002139 if (!test_and_set_bit(WantReplacement,
2140 &rdev->flags))
2141 set_bit(MD_RECOVERY_NEEDED,
2142 &rdev->mddev->recovery);
2143 }
NeilBrown5e570282011-07-28 11:39:25 +10002144 }
2145 if (!ok) {
2146 /* We don't worry if we cannot set a bad block -
2147 * it really is bad so there is no loss in not
2148 * recording it yet
2149 */
2150 rdev_set_badblocks(rdev, addr, s, 0);
2151
2152 if (rdev != conf->mirrors[dw].rdev) {
2153 /* need bad block on destination too */
NeilBrown3cb03002011-10-11 16:45:26 +11002154 struct md_rdev *rdev2 = conf->mirrors[dw].rdev;
NeilBrown5e570282011-07-28 11:39:25 +10002155 addr = r10_bio->devs[1].addr + sect;
2156 ok = rdev_set_badblocks(rdev2, addr, s, 0);
2157 if (!ok) {
2158 /* just abort the recovery */
2159 printk(KERN_NOTICE
2160 "md/raid10:%s: recovery aborted"
2161 " due to read error\n",
2162 mdname(mddev));
2163
2164 conf->mirrors[dw].recovery_disabled
2165 = mddev->recovery_disabled;
2166 set_bit(MD_RECOVERY_INTR,
2167 &mddev->recovery);
2168 break;
2169 }
2170 }
2171 }
2172
2173 sectors -= s;
2174 sect += s;
2175 idx++;
2176 }
2177}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002178
NeilBrown9f2c9d12011-10-11 16:48:43 +11002179static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002180{
NeilBrowne879a872011-10-11 16:49:02 +11002181 struct r10conf *conf = mddev->private;
Namhyung Kimc65060a2011-07-18 17:38:49 +10002182 int d;
NeilBrown24afd802011-12-23 10:17:55 +11002183 struct bio *wbio, *wbio2;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002184
NeilBrown5e570282011-07-28 11:39:25 +10002185 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) {
2186 fix_recovery_read_error(r10_bio);
2187 end_sync_request(r10_bio);
2188 return;
2189 }
2190
Namhyung Kimc65060a2011-07-18 17:38:49 +10002191 /*
2192 * share the pages with the first bio
Linus Torvalds1da177e2005-04-16 15:20:36 -07002193 * and submit the write request
2194 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002195 d = r10_bio->devs[1].devnum;
NeilBrown24afd802011-12-23 10:17:55 +11002196 wbio = r10_bio->devs[1].bio;
2197 wbio2 = r10_bio->devs[1].repl_bio;
2198 if (wbio->bi_end_io) {
2199 atomic_inc(&conf->mirrors[d].rdev->nr_pending);
2200 md_sync_acct(conf->mirrors[d].rdev->bdev, wbio->bi_size >> 9);
2201 generic_make_request(wbio);
2202 }
2203 if (wbio2 && wbio2->bi_end_io) {
2204 atomic_inc(&conf->mirrors[d].replacement->nr_pending);
2205 md_sync_acct(conf->mirrors[d].replacement->bdev,
2206 wbio2->bi_size >> 9);
2207 generic_make_request(wbio2);
2208 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002209}
2210
2211
2212/*
Robert Becker1e509152009-12-14 12:49:58 +11002213 * Used by fix_read_error() to decay the per rdev read_errors.
2214 * We halve the read error count for every hour that has elapsed
2215 * since the last recorded read error.
2216 *
2217 */
NeilBrownfd01b882011-10-11 16:47:53 +11002218static void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev)
Robert Becker1e509152009-12-14 12:49:58 +11002219{
2220 struct timespec cur_time_mon;
2221 unsigned long hours_since_last;
2222 unsigned int read_errors = atomic_read(&rdev->read_errors);
2223
2224 ktime_get_ts(&cur_time_mon);
2225
2226 if (rdev->last_read_error.tv_sec == 0 &&
2227 rdev->last_read_error.tv_nsec == 0) {
2228 /* first time we've seen a read error */
2229 rdev->last_read_error = cur_time_mon;
2230 return;
2231 }
2232
2233 hours_since_last = (cur_time_mon.tv_sec -
2234 rdev->last_read_error.tv_sec) / 3600;
2235
2236 rdev->last_read_error = cur_time_mon;
2237
2238 /*
2239 * if hours_since_last is > the number of bits in read_errors
2240 * just set read errors to 0. We do this to avoid
2241 * overflowing the shift of read_errors by hours_since_last.
2242 */
2243 if (hours_since_last >= 8 * sizeof(read_errors))
2244 atomic_set(&rdev->read_errors, 0);
2245 else
2246 atomic_set(&rdev->read_errors, read_errors >> hours_since_last);
2247}
2248
NeilBrown3cb03002011-10-11 16:45:26 +11002249static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector,
NeilBrown58c54fc2011-07-28 11:39:25 +10002250 int sectors, struct page *page, int rw)
2251{
2252 sector_t first_bad;
2253 int bad_sectors;
2254
2255 if (is_badblock(rdev, sector, sectors, &first_bad, &bad_sectors)
2256 && (rw == READ || test_bit(WriteErrorSeen, &rdev->flags)))
2257 return -1;
2258 if (sync_page_io(rdev, sector, sectors << 9, page, rw, false))
2259 /* success */
2260 return 1;
NeilBrownb7044d42011-12-23 10:17:56 +11002261 if (rw == WRITE) {
NeilBrown58c54fc2011-07-28 11:39:25 +10002262 set_bit(WriteErrorSeen, &rdev->flags);
NeilBrownb7044d42011-12-23 10:17:56 +11002263 if (!test_and_set_bit(WantReplacement, &rdev->flags))
2264 set_bit(MD_RECOVERY_NEEDED,
2265 &rdev->mddev->recovery);
2266 }
NeilBrown58c54fc2011-07-28 11:39:25 +10002267 /* need to record an error - either for the block or the device */
2268 if (!rdev_set_badblocks(rdev, sector, sectors, 0))
2269 md_error(rdev->mddev, rdev);
2270 return 0;
2271}
2272
Robert Becker1e509152009-12-14 12:49:58 +11002273/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002274 * This is a kernel thread which:
2275 *
2276 * 1. Retries failed read operations on working mirrors.
2277 * 2. Updates the raid superblock when problems encounter.
NeilBrown6814d532006-10-03 01:15:45 -07002278 * 3. Performs writes following reads for array synchronising.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002279 */
2280
NeilBrowne879a872011-10-11 16:49:02 +11002281static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio)
NeilBrown6814d532006-10-03 01:15:45 -07002282{
2283 int sect = 0; /* Offset from r10_bio->sector */
2284 int sectors = r10_bio->sectors;
NeilBrown3cb03002011-10-11 16:45:26 +11002285 struct md_rdev*rdev;
Robert Becker1e509152009-12-14 12:49:58 +11002286 int max_read_errors = atomic_read(&mddev->max_corr_read_errors);
Prasanna S. Panchamukhi0544a212010-06-24 13:31:03 +10002287 int d = r10_bio->devs[r10_bio->read_slot].devnum;
Robert Becker1e509152009-12-14 12:49:58 +11002288
NeilBrown7c4e06f2011-05-11 14:53:17 +10002289 /* still own a reference to this rdev, so it cannot
2290 * have been cleared recently.
2291 */
2292 rdev = conf->mirrors[d].rdev;
Robert Becker1e509152009-12-14 12:49:58 +11002293
NeilBrown7c4e06f2011-05-11 14:53:17 +10002294 if (test_bit(Faulty, &rdev->flags))
2295 /* drive has already been failed, just ignore any
2296 more fix_read_error() attempts */
2297 return;
2298
2299 check_decay_read_errors(mddev, rdev);
2300 atomic_inc(&rdev->read_errors);
2301 if (atomic_read(&rdev->read_errors) > max_read_errors) {
2302 char b[BDEVNAME_SIZE];
Robert Becker1e509152009-12-14 12:49:58 +11002303 bdevname(rdev->bdev, b);
2304
NeilBrown7c4e06f2011-05-11 14:53:17 +10002305 printk(KERN_NOTICE
2306 "md/raid10:%s: %s: Raid device exceeded "
2307 "read_error threshold [cur %d:max %d]\n",
2308 mdname(mddev), b,
2309 atomic_read(&rdev->read_errors), max_read_errors);
2310 printk(KERN_NOTICE
2311 "md/raid10:%s: %s: Failing raid device\n",
2312 mdname(mddev), b);
2313 md_error(mddev, conf->mirrors[d].rdev);
NeilBrownfae8cc5e2012-02-14 11:10:10 +11002314 r10_bio->devs[r10_bio->read_slot].bio = IO_BLOCKED;
NeilBrown7c4e06f2011-05-11 14:53:17 +10002315 return;
Robert Becker1e509152009-12-14 12:49:58 +11002316 }
Robert Becker1e509152009-12-14 12:49:58 +11002317
NeilBrown6814d532006-10-03 01:15:45 -07002318 while(sectors) {
2319 int s = sectors;
2320 int sl = r10_bio->read_slot;
2321 int success = 0;
2322 int start;
2323
2324 if (s > (PAGE_SIZE>>9))
2325 s = PAGE_SIZE >> 9;
2326
2327 rcu_read_lock();
2328 do {
NeilBrown8dbed5c2011-07-28 11:39:24 +10002329 sector_t first_bad;
2330 int bad_sectors;
2331
Prasanna S. Panchamukhi0544a212010-06-24 13:31:03 +10002332 d = r10_bio->devs[sl].devnum;
NeilBrown6814d532006-10-03 01:15:45 -07002333 rdev = rcu_dereference(conf->mirrors[d].rdev);
2334 if (rdev &&
NeilBrown050b6612012-03-19 12:46:39 +11002335 !test_bit(Unmerged, &rdev->flags) &&
NeilBrown8dbed5c2011-07-28 11:39:24 +10002336 test_bit(In_sync, &rdev->flags) &&
2337 is_badblock(rdev, r10_bio->devs[sl].addr + sect, s,
2338 &first_bad, &bad_sectors) == 0) {
NeilBrown6814d532006-10-03 01:15:45 -07002339 atomic_inc(&rdev->nr_pending);
2340 rcu_read_unlock();
NeilBrown2b193362010-10-27 15:16:40 +11002341 success = sync_page_io(rdev,
NeilBrown6814d532006-10-03 01:15:45 -07002342 r10_bio->devs[sl].addr +
Jonathan Brassowccebd4c2011-01-14 09:14:33 +11002343 sect,
NeilBrown6814d532006-10-03 01:15:45 -07002344 s<<9,
Jonathan Brassowccebd4c2011-01-14 09:14:33 +11002345 conf->tmppage, READ, false);
NeilBrown6814d532006-10-03 01:15:45 -07002346 rdev_dec_pending(rdev, mddev);
2347 rcu_read_lock();
2348 if (success)
2349 break;
2350 }
2351 sl++;
2352 if (sl == conf->copies)
2353 sl = 0;
2354 } while (!success && sl != r10_bio->read_slot);
2355 rcu_read_unlock();
2356
2357 if (!success) {
NeilBrown58c54fc2011-07-28 11:39:25 +10002358 /* Cannot read from anywhere, just mark the block
2359 * as bad on the first device to discourage future
2360 * reads.
2361 */
NeilBrown6814d532006-10-03 01:15:45 -07002362 int dn = r10_bio->devs[r10_bio->read_slot].devnum;
NeilBrown58c54fc2011-07-28 11:39:25 +10002363 rdev = conf->mirrors[dn].rdev;
2364
2365 if (!rdev_set_badblocks(
2366 rdev,
2367 r10_bio->devs[r10_bio->read_slot].addr
2368 + sect,
NeilBrownfae8cc5e2012-02-14 11:10:10 +11002369 s, 0)) {
NeilBrown58c54fc2011-07-28 11:39:25 +10002370 md_error(mddev, rdev);
NeilBrownfae8cc5e2012-02-14 11:10:10 +11002371 r10_bio->devs[r10_bio->read_slot].bio
2372 = IO_BLOCKED;
2373 }
NeilBrown6814d532006-10-03 01:15:45 -07002374 break;
2375 }
2376
2377 start = sl;
2378 /* write it back and re-read */
2379 rcu_read_lock();
2380 while (sl != r10_bio->read_slot) {
Robert Becker67b8dc42009-12-14 12:49:57 +11002381 char b[BDEVNAME_SIZE];
Prasanna S. Panchamukhi0544a212010-06-24 13:31:03 +10002382
NeilBrown6814d532006-10-03 01:15:45 -07002383 if (sl==0)
2384 sl = conf->copies;
2385 sl--;
2386 d = r10_bio->devs[sl].devnum;
2387 rdev = rcu_dereference(conf->mirrors[d].rdev);
NeilBrown1294b9c2011-07-28 11:39:23 +10002388 if (!rdev ||
NeilBrown050b6612012-03-19 12:46:39 +11002389 test_bit(Unmerged, &rdev->flags) ||
NeilBrown1294b9c2011-07-28 11:39:23 +10002390 !test_bit(In_sync, &rdev->flags))
2391 continue;
2392
2393 atomic_inc(&rdev->nr_pending);
2394 rcu_read_unlock();
NeilBrown58c54fc2011-07-28 11:39:25 +10002395 if (r10_sync_page_io(rdev,
2396 r10_bio->devs[sl].addr +
2397 sect,
NeilBrown055d3742012-07-03 15:55:33 +10002398 s, conf->tmppage, WRITE)
NeilBrown1294b9c2011-07-28 11:39:23 +10002399 == 0) {
2400 /* Well, this device is dead */
2401 printk(KERN_NOTICE
2402 "md/raid10:%s: read correction "
2403 "write failed"
2404 " (%d sectors at %llu on %s)\n",
2405 mdname(mddev), s,
2406 (unsigned long long)(
NeilBrownf8c9e742012-05-21 09:28:33 +10002407 sect +
2408 choose_data_offset(r10_bio,
2409 rdev)),
NeilBrown1294b9c2011-07-28 11:39:23 +10002410 bdevname(rdev->bdev, b));
2411 printk(KERN_NOTICE "md/raid10:%s: %s: failing "
2412 "drive\n",
2413 mdname(mddev),
2414 bdevname(rdev->bdev, b));
NeilBrown6814d532006-10-03 01:15:45 -07002415 }
NeilBrown1294b9c2011-07-28 11:39:23 +10002416 rdev_dec_pending(rdev, mddev);
2417 rcu_read_lock();
NeilBrown6814d532006-10-03 01:15:45 -07002418 }
2419 sl = start;
2420 while (sl != r10_bio->read_slot) {
NeilBrown1294b9c2011-07-28 11:39:23 +10002421 char b[BDEVNAME_SIZE];
Prasanna S. Panchamukhi0544a212010-06-24 13:31:03 +10002422
NeilBrown6814d532006-10-03 01:15:45 -07002423 if (sl==0)
2424 sl = conf->copies;
2425 sl--;
2426 d = r10_bio->devs[sl].devnum;
2427 rdev = rcu_dereference(conf->mirrors[d].rdev);
NeilBrown1294b9c2011-07-28 11:39:23 +10002428 if (!rdev ||
2429 !test_bit(In_sync, &rdev->flags))
2430 continue;
Robert Becker67b8dc42009-12-14 12:49:57 +11002431
NeilBrown1294b9c2011-07-28 11:39:23 +10002432 atomic_inc(&rdev->nr_pending);
2433 rcu_read_unlock();
NeilBrown58c54fc2011-07-28 11:39:25 +10002434 switch (r10_sync_page_io(rdev,
2435 r10_bio->devs[sl].addr +
2436 sect,
NeilBrown055d3742012-07-03 15:55:33 +10002437 s, conf->tmppage,
NeilBrown58c54fc2011-07-28 11:39:25 +10002438 READ)) {
2439 case 0:
NeilBrown1294b9c2011-07-28 11:39:23 +10002440 /* Well, this device is dead */
2441 printk(KERN_NOTICE
2442 "md/raid10:%s: unable to read back "
2443 "corrected sectors"
2444 " (%d sectors at %llu on %s)\n",
2445 mdname(mddev), s,
2446 (unsigned long long)(
NeilBrownf8c9e742012-05-21 09:28:33 +10002447 sect +
2448 choose_data_offset(r10_bio, rdev)),
NeilBrown1294b9c2011-07-28 11:39:23 +10002449 bdevname(rdev->bdev, b));
2450 printk(KERN_NOTICE "md/raid10:%s: %s: failing "
2451 "drive\n",
2452 mdname(mddev),
2453 bdevname(rdev->bdev, b));
NeilBrown58c54fc2011-07-28 11:39:25 +10002454 break;
2455 case 1:
NeilBrown1294b9c2011-07-28 11:39:23 +10002456 printk(KERN_INFO
2457 "md/raid10:%s: read error corrected"
2458 " (%d sectors at %llu on %s)\n",
2459 mdname(mddev), s,
2460 (unsigned long long)(
NeilBrownf8c9e742012-05-21 09:28:33 +10002461 sect +
2462 choose_data_offset(r10_bio, rdev)),
NeilBrown1294b9c2011-07-28 11:39:23 +10002463 bdevname(rdev->bdev, b));
2464 atomic_add(s, &rdev->corrected_errors);
NeilBrown6814d532006-10-03 01:15:45 -07002465 }
NeilBrown1294b9c2011-07-28 11:39:23 +10002466
2467 rdev_dec_pending(rdev, mddev);
2468 rcu_read_lock();
NeilBrown6814d532006-10-03 01:15:45 -07002469 }
2470 rcu_read_unlock();
2471
2472 sectors -= s;
2473 sect += s;
2474 }
2475}
2476
NeilBrownbd870a12011-07-28 11:39:24 +10002477static void bi_complete(struct bio *bio, int error)
2478{
2479 complete((struct completion *)bio->bi_private);
2480}
2481
2482static int submit_bio_wait(int rw, struct bio *bio)
2483{
2484 struct completion event;
2485 rw |= REQ_SYNC;
2486
2487 init_completion(&event);
2488 bio->bi_private = &event;
2489 bio->bi_end_io = bi_complete;
2490 submit_bio(rw, bio);
2491 wait_for_completion(&event);
2492
2493 return test_bit(BIO_UPTODATE, &bio->bi_flags);
2494}
2495
NeilBrown9f2c9d12011-10-11 16:48:43 +11002496static int narrow_write_error(struct r10bio *r10_bio, int i)
NeilBrownbd870a12011-07-28 11:39:24 +10002497{
2498 struct bio *bio = r10_bio->master_bio;
NeilBrownfd01b882011-10-11 16:47:53 +11002499 struct mddev *mddev = r10_bio->mddev;
NeilBrowne879a872011-10-11 16:49:02 +11002500 struct r10conf *conf = mddev->private;
NeilBrown3cb03002011-10-11 16:45:26 +11002501 struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev;
NeilBrownbd870a12011-07-28 11:39:24 +10002502 /* bio has the data to be written to slot 'i' where
2503 * we just recently had a write error.
2504 * We repeatedly clone the bio and trim down to one block,
2505 * then try the write. Where the write fails we record
2506 * a bad block.
2507 * It is conceivable that the bio doesn't exactly align with
2508 * blocks. We must handle this.
2509 *
2510 * We currently own a reference to the rdev.
2511 */
2512
2513 int block_sectors;
2514 sector_t sector;
2515 int sectors;
2516 int sect_to_write = r10_bio->sectors;
2517 int ok = 1;
2518
2519 if (rdev->badblocks.shift < 0)
2520 return 0;
2521
2522 block_sectors = 1 << rdev->badblocks.shift;
2523 sector = r10_bio->sector;
2524 sectors = ((r10_bio->sector + block_sectors)
2525 & ~(sector_t)(block_sectors - 1))
2526 - sector;
2527
2528 while (sect_to_write) {
2529 struct bio *wbio;
2530 if (sectors > sect_to_write)
2531 sectors = sect_to_write;
2532 /* Write at 'sector' for 'sectors' */
2533 wbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
2534 md_trim_bio(wbio, sector - bio->bi_sector, sectors);
2535 wbio->bi_sector = (r10_bio->devs[i].addr+
NeilBrownf8c9e742012-05-21 09:28:33 +10002536 choose_data_offset(r10_bio, rdev) +
NeilBrownbd870a12011-07-28 11:39:24 +10002537 (sector - r10_bio->sector));
2538 wbio->bi_bdev = rdev->bdev;
2539 if (submit_bio_wait(WRITE, wbio) == 0)
2540 /* Failure! */
2541 ok = rdev_set_badblocks(rdev, sector,
2542 sectors, 0)
2543 && ok;
2544
2545 bio_put(wbio);
2546 sect_to_write -= sectors;
2547 sector += sectors;
2548 sectors = block_sectors;
2549 }
2550 return ok;
2551}
2552
NeilBrown9f2c9d12011-10-11 16:48:43 +11002553static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio)
NeilBrown560f8e52011-07-28 11:39:23 +10002554{
2555 int slot = r10_bio->read_slot;
NeilBrown560f8e52011-07-28 11:39:23 +10002556 struct bio *bio;
NeilBrowne879a872011-10-11 16:49:02 +11002557 struct r10conf *conf = mddev->private;
NeilBrownabbf0982011-12-23 10:17:54 +11002558 struct md_rdev *rdev = r10_bio->devs[slot].rdev;
NeilBrown560f8e52011-07-28 11:39:23 +10002559 char b[BDEVNAME_SIZE];
2560 unsigned long do_sync;
NeilBrown856e08e2011-07-28 11:39:23 +10002561 int max_sectors;
NeilBrown560f8e52011-07-28 11:39:23 +10002562
2563 /* we got a read error. Maybe the drive is bad. Maybe just
2564 * the block and we can fix it.
2565 * We freeze all other IO, and try reading the block from
2566 * other devices. When we find one, we re-write
2567 * and check it that fixes the read error.
2568 * This is all done synchronously while the array is
2569 * frozen.
2570 */
NeilBrownfae8cc5e2012-02-14 11:10:10 +11002571 bio = r10_bio->devs[slot].bio;
2572 bdevname(bio->bi_bdev, b);
2573 bio_put(bio);
2574 r10_bio->devs[slot].bio = NULL;
2575
NeilBrown560f8e52011-07-28 11:39:23 +10002576 if (mddev->ro == 0) {
2577 freeze_array(conf);
2578 fix_read_error(conf, mddev, r10_bio);
2579 unfreeze_array(conf);
NeilBrownfae8cc5e2012-02-14 11:10:10 +11002580 } else
2581 r10_bio->devs[slot].bio = IO_BLOCKED;
2582
NeilBrownabbf0982011-12-23 10:17:54 +11002583 rdev_dec_pending(rdev, mddev);
NeilBrown560f8e52011-07-28 11:39:23 +10002584
NeilBrown7399c312011-07-28 11:39:23 +10002585read_more:
NeilBrown96c3fd12011-12-23 10:17:54 +11002586 rdev = read_balance(conf, r10_bio, &max_sectors);
2587 if (rdev == NULL) {
NeilBrown560f8e52011-07-28 11:39:23 +10002588 printk(KERN_ALERT "md/raid10:%s: %s: unrecoverable I/O"
2589 " read error for block %llu\n",
NeilBrown7399c312011-07-28 11:39:23 +10002590 mdname(mddev), b,
NeilBrown560f8e52011-07-28 11:39:23 +10002591 (unsigned long long)r10_bio->sector);
2592 raid_end_bio_io(r10_bio);
NeilBrown560f8e52011-07-28 11:39:23 +10002593 return;
2594 }
2595
2596 do_sync = (r10_bio->master_bio->bi_rw & REQ_SYNC);
NeilBrown560f8e52011-07-28 11:39:23 +10002597 slot = r10_bio->read_slot;
NeilBrown560f8e52011-07-28 11:39:23 +10002598 printk_ratelimited(
2599 KERN_ERR
NeilBrown055d3742012-07-03 15:55:33 +10002600 "md/raid10:%s: %s: redirecting "
NeilBrown560f8e52011-07-28 11:39:23 +10002601 "sector %llu to another mirror\n",
2602 mdname(mddev),
2603 bdevname(rdev->bdev, b),
2604 (unsigned long long)r10_bio->sector);
2605 bio = bio_clone_mddev(r10_bio->master_bio,
2606 GFP_NOIO, mddev);
NeilBrown7399c312011-07-28 11:39:23 +10002607 md_trim_bio(bio,
2608 r10_bio->sector - bio->bi_sector,
2609 max_sectors);
NeilBrown560f8e52011-07-28 11:39:23 +10002610 r10_bio->devs[slot].bio = bio;
NeilBrownabbf0982011-12-23 10:17:54 +11002611 r10_bio->devs[slot].rdev = rdev;
NeilBrown560f8e52011-07-28 11:39:23 +10002612 bio->bi_sector = r10_bio->devs[slot].addr
NeilBrownf8c9e742012-05-21 09:28:33 +10002613 + choose_data_offset(r10_bio, rdev);
NeilBrown560f8e52011-07-28 11:39:23 +10002614 bio->bi_bdev = rdev->bdev;
2615 bio->bi_rw = READ | do_sync;
2616 bio->bi_private = r10_bio;
2617 bio->bi_end_io = raid10_end_read_request;
NeilBrown7399c312011-07-28 11:39:23 +10002618 if (max_sectors < r10_bio->sectors) {
2619 /* Drat - have to split this up more */
2620 struct bio *mbio = r10_bio->master_bio;
2621 int sectors_handled =
2622 r10_bio->sector + max_sectors
2623 - mbio->bi_sector;
2624 r10_bio->sectors = max_sectors;
2625 spin_lock_irq(&conf->device_lock);
2626 if (mbio->bi_phys_segments == 0)
2627 mbio->bi_phys_segments = 2;
2628 else
2629 mbio->bi_phys_segments++;
2630 spin_unlock_irq(&conf->device_lock);
2631 generic_make_request(bio);
NeilBrown7399c312011-07-28 11:39:23 +10002632
2633 r10_bio = mempool_alloc(conf->r10bio_pool,
2634 GFP_NOIO);
2635 r10_bio->master_bio = mbio;
2636 r10_bio->sectors = (mbio->bi_size >> 9)
2637 - sectors_handled;
2638 r10_bio->state = 0;
2639 set_bit(R10BIO_ReadError,
2640 &r10_bio->state);
2641 r10_bio->mddev = mddev;
2642 r10_bio->sector = mbio->bi_sector
2643 + sectors_handled;
2644
2645 goto read_more;
2646 } else
2647 generic_make_request(bio);
NeilBrown560f8e52011-07-28 11:39:23 +10002648}
2649
NeilBrowne879a872011-10-11 16:49:02 +11002650static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio)
NeilBrown749c55e2011-07-28 11:39:24 +10002651{
2652 /* Some sort of write request has finished and it
2653 * succeeded in writing where we thought there was a
2654 * bad block. So forget the bad block.
NeilBrown1a0b7cd2011-07-28 11:39:25 +10002655 * Or possibly if failed and we need to record
2656 * a bad block.
NeilBrown749c55e2011-07-28 11:39:24 +10002657 */
2658 int m;
NeilBrown3cb03002011-10-11 16:45:26 +11002659 struct md_rdev *rdev;
NeilBrown749c55e2011-07-28 11:39:24 +10002660
2661 if (test_bit(R10BIO_IsSync, &r10_bio->state) ||
2662 test_bit(R10BIO_IsRecover, &r10_bio->state)) {
NeilBrown1a0b7cd2011-07-28 11:39:25 +10002663 for (m = 0; m < conf->copies; m++) {
2664 int dev = r10_bio->devs[m].devnum;
2665 rdev = conf->mirrors[dev].rdev;
2666 if (r10_bio->devs[m].bio == NULL)
2667 continue;
2668 if (test_bit(BIO_UPTODATE,
NeilBrown749c55e2011-07-28 11:39:24 +10002669 &r10_bio->devs[m].bio->bi_flags)) {
NeilBrown749c55e2011-07-28 11:39:24 +10002670 rdev_clear_badblocks(
2671 rdev,
2672 r10_bio->devs[m].addr,
NeilBrownc6563a82012-05-21 09:27:00 +10002673 r10_bio->sectors, 0);
NeilBrown1a0b7cd2011-07-28 11:39:25 +10002674 } else {
2675 if (!rdev_set_badblocks(
2676 rdev,
2677 r10_bio->devs[m].addr,
2678 r10_bio->sectors, 0))
2679 md_error(conf->mddev, rdev);
NeilBrown749c55e2011-07-28 11:39:24 +10002680 }
NeilBrown9ad1aef2011-12-23 10:17:55 +11002681 rdev = conf->mirrors[dev].replacement;
2682 if (r10_bio->devs[m].repl_bio == NULL)
2683 continue;
2684 if (test_bit(BIO_UPTODATE,
2685 &r10_bio->devs[m].repl_bio->bi_flags)) {
2686 rdev_clear_badblocks(
2687 rdev,
2688 r10_bio->devs[m].addr,
NeilBrownc6563a82012-05-21 09:27:00 +10002689 r10_bio->sectors, 0);
NeilBrown9ad1aef2011-12-23 10:17:55 +11002690 } else {
2691 if (!rdev_set_badblocks(
2692 rdev,
2693 r10_bio->devs[m].addr,
2694 r10_bio->sectors, 0))
2695 md_error(conf->mddev, rdev);
2696 }
NeilBrown1a0b7cd2011-07-28 11:39:25 +10002697 }
NeilBrown749c55e2011-07-28 11:39:24 +10002698 put_buf(r10_bio);
2699 } else {
NeilBrownbd870a12011-07-28 11:39:24 +10002700 for (m = 0; m < conf->copies; m++) {
2701 int dev = r10_bio->devs[m].devnum;
2702 struct bio *bio = r10_bio->devs[m].bio;
2703 rdev = conf->mirrors[dev].rdev;
2704 if (bio == IO_MADE_GOOD) {
NeilBrown749c55e2011-07-28 11:39:24 +10002705 rdev_clear_badblocks(
2706 rdev,
2707 r10_bio->devs[m].addr,
NeilBrownc6563a82012-05-21 09:27:00 +10002708 r10_bio->sectors, 0);
NeilBrown749c55e2011-07-28 11:39:24 +10002709 rdev_dec_pending(rdev, conf->mddev);
NeilBrownbd870a12011-07-28 11:39:24 +10002710 } else if (bio != NULL &&
2711 !test_bit(BIO_UPTODATE, &bio->bi_flags)) {
2712 if (!narrow_write_error(r10_bio, m)) {
2713 md_error(conf->mddev, rdev);
2714 set_bit(R10BIO_Degraded,
2715 &r10_bio->state);
2716 }
2717 rdev_dec_pending(rdev, conf->mddev);
NeilBrown749c55e2011-07-28 11:39:24 +10002718 }
NeilBrown475b0322011-12-23 10:17:55 +11002719 bio = r10_bio->devs[m].repl_bio;
2720 rdev = conf->mirrors[dev].replacement;
NeilBrown4ca40c22011-12-23 10:17:55 +11002721 if (rdev && bio == IO_MADE_GOOD) {
NeilBrown475b0322011-12-23 10:17:55 +11002722 rdev_clear_badblocks(
2723 rdev,
2724 r10_bio->devs[m].addr,
NeilBrownc6563a82012-05-21 09:27:00 +10002725 r10_bio->sectors, 0);
NeilBrown475b0322011-12-23 10:17:55 +11002726 rdev_dec_pending(rdev, conf->mddev);
2727 }
NeilBrownbd870a12011-07-28 11:39:24 +10002728 }
2729 if (test_bit(R10BIO_WriteError,
2730 &r10_bio->state))
2731 close_write(r10_bio);
NeilBrown749c55e2011-07-28 11:39:24 +10002732 raid_end_bio_io(r10_bio);
2733 }
2734}
2735
Shaohua Li4ed87312012-10-11 13:34:00 +11002736static void raid10d(struct md_thread *thread)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002737{
Shaohua Li4ed87312012-10-11 13:34:00 +11002738 struct mddev *mddev = thread->mddev;
NeilBrown9f2c9d12011-10-11 16:48:43 +11002739 struct r10bio *r10_bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002740 unsigned long flags;
NeilBrowne879a872011-10-11 16:49:02 +11002741 struct r10conf *conf = mddev->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002742 struct list_head *head = &conf->retry_list;
NeilBrowne1dfa0a2011-04-18 18:25:41 +10002743 struct blk_plug plug;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002744
2745 md_check_recovery(mddev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002746
NeilBrowne1dfa0a2011-04-18 18:25:41 +10002747 blk_start_plug(&plug);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002748 for (;;) {
NeilBrowna35e63e2008-03-04 14:29:29 -08002749
NeilBrown0021b7b2012-07-31 09:08:14 +02002750 flush_pending_writes(conf);
NeilBrowna35e63e2008-03-04 14:29:29 -08002751
Linus Torvalds1da177e2005-04-16 15:20:36 -07002752 spin_lock_irqsave(&conf->device_lock, flags);
NeilBrowna35e63e2008-03-04 14:29:29 -08002753 if (list_empty(head)) {
NeilBrown6cce3b232006-01-06 00:20:16 -08002754 spin_unlock_irqrestore(&conf->device_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002755 break;
NeilBrowna35e63e2008-03-04 14:29:29 -08002756 }
NeilBrown9f2c9d12011-10-11 16:48:43 +11002757 r10_bio = list_entry(head->prev, struct r10bio, retry_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002758 list_del(head->prev);
NeilBrown4443ae12006-01-06 00:20:28 -08002759 conf->nr_queued--;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002760 spin_unlock_irqrestore(&conf->device_lock, flags);
2761
2762 mddev = r10_bio->mddev;
NeilBrown070ec552009-06-16 16:54:21 +10002763 conf = mddev->private;
NeilBrownbd870a12011-07-28 11:39:24 +10002764 if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
2765 test_bit(R10BIO_WriteError, &r10_bio->state))
NeilBrown749c55e2011-07-28 11:39:24 +10002766 handle_write_completed(conf, r10_bio);
NeilBrown3ea7daa2012-05-22 13:53:47 +10002767 else if (test_bit(R10BIO_IsReshape, &r10_bio->state))
2768 reshape_request_write(mddev, r10_bio);
NeilBrown749c55e2011-07-28 11:39:24 +10002769 else if (test_bit(R10BIO_IsSync, &r10_bio->state))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002770 sync_request_write(mddev, r10_bio);
Jens Axboe7eaceac2011-03-10 08:52:07 +01002771 else if (test_bit(R10BIO_IsRecover, &r10_bio->state))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002772 recovery_request_write(mddev, r10_bio);
NeilBrown856e08e2011-07-28 11:39:23 +10002773 else if (test_bit(R10BIO_ReadError, &r10_bio->state))
NeilBrown560f8e52011-07-28 11:39:23 +10002774 handle_read_error(mddev, r10_bio);
NeilBrown856e08e2011-07-28 11:39:23 +10002775 else {
2776 /* just a partial read to be scheduled from a
2777 * separate context
2778 */
2779 int slot = r10_bio->read_slot;
2780 generic_make_request(r10_bio->devs[slot].bio);
2781 }
NeilBrown4443ae12006-01-06 00:20:28 -08002782
NeilBrown1d9d5242009-10-16 15:55:32 +11002783 cond_resched();
NeilBrownde393cd2011-07-28 11:31:48 +10002784 if (mddev->flags & ~(1<<MD_CHANGE_PENDING))
2785 md_check_recovery(mddev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002786 }
NeilBrowne1dfa0a2011-04-18 18:25:41 +10002787 blk_finish_plug(&plug);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002788}
2789
2790
NeilBrowne879a872011-10-11 16:49:02 +11002791static int init_resync(struct r10conf *conf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002792{
2793 int buffs;
NeilBrown69335ef2011-12-23 10:17:54 +11002794 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002795
2796 buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
Eric Sesterhennb6385482006-04-02 13:34:29 +02002797 BUG_ON(conf->r10buf_pool);
NeilBrown69335ef2011-12-23 10:17:54 +11002798 conf->have_replacement = 0;
NeilBrown5cf00fc2012-05-21 09:28:20 +10002799 for (i = 0; i < conf->geo.raid_disks; i++)
NeilBrown69335ef2011-12-23 10:17:54 +11002800 if (conf->mirrors[i].replacement)
2801 conf->have_replacement = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002802 conf->r10buf_pool = mempool_create(buffs, r10buf_pool_alloc, r10buf_pool_free, conf);
2803 if (!conf->r10buf_pool)
2804 return -ENOMEM;
2805 conf->next_resync = 0;
2806 return 0;
2807}
2808
2809/*
2810 * perform a "sync" on one "block"
2811 *
2812 * We need to make sure that no normal I/O request - particularly write
2813 * requests - conflict with active sync requests.
2814 *
2815 * This is achieved by tracking pending requests and a 'barrier' concept
2816 * that can be installed to exclude normal IO requests.
2817 *
2818 * Resync and recovery are handled very differently.
2819 * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery.
2820 *
2821 * For resync, we iterate over virtual addresses, read all copies,
2822 * and update if there are differences. If only one copy is live,
2823 * skip it.
2824 * For recovery, we iterate over physical addresses, read a good
2825 * value for each non-in_sync drive, and over-write.
2826 *
2827 * So, for recovery we may have several outstanding complex requests for a
2828 * given address, one for each out-of-sync device. We model this by allocating
2829 * a number of r10_bio structures, one for each out-of-sync device.
2830 * As we setup these structures, we collect all bio's together into a list
2831 * which we then process collectively to add pages, and then process again
2832 * to pass to generic_make_request.
2833 *
2834 * The r10_bio structures are linked using a borrowed master_bio pointer.
2835 * This link is counted in ->remaining. When the r10_bio that points to NULL
2836 * has its remaining count decremented to 0, the whole complex operation
2837 * is complete.
2838 *
2839 */
2840
NeilBrownfd01b882011-10-11 16:47:53 +11002841static sector_t sync_request(struct mddev *mddev, sector_t sector_nr,
NeilBrownab9d47e2011-05-11 14:54:41 +10002842 int *skipped, int go_faster)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002843{
NeilBrowne879a872011-10-11 16:49:02 +11002844 struct r10conf *conf = mddev->private;
NeilBrown9f2c9d12011-10-11 16:48:43 +11002845 struct r10bio *r10_bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002846 struct bio *biolist = NULL, *bio;
2847 sector_t max_sector, nr_sectors;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002848 int i;
NeilBrown6cce3b232006-01-06 00:20:16 -08002849 int max_sync;
NeilBrown57dab0b2010-10-19 10:03:39 +11002850 sector_t sync_blocks;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002851 sector_t sectors_skipped = 0;
2852 int chunks_skipped = 0;
NeilBrown5cf00fc2012-05-21 09:28:20 +10002853 sector_t chunk_mask = conf->geo.chunk_mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002854
2855 if (!conf->r10buf_pool)
2856 if (init_resync(conf))
NeilBrown57afd892005-06-21 17:17:13 -07002857 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002858
2859 skipped:
Andre Noll58c0fed2009-03-31 14:33:13 +11002860 max_sector = mddev->dev_sectors;
NeilBrown3ea7daa2012-05-22 13:53:47 +10002861 if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ||
2862 test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002863 max_sector = mddev->resync_max_sectors;
2864 if (sector_nr >= max_sector) {
NeilBrown6cce3b232006-01-06 00:20:16 -08002865 /* If we aborted, we need to abort the
2866 * sync on the 'current' bitmap chucks (there can
2867 * be several when recovering multiple devices).
2868 * as we may have started syncing it but not finished.
2869 * We can find the current address in
2870 * mddev->curr_resync, but for recovery,
2871 * we need to convert that to several
2872 * virtual addresses.
2873 */
NeilBrown3ea7daa2012-05-22 13:53:47 +10002874 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
2875 end_reshape(conf);
2876 return 0;
2877 }
2878
NeilBrown6cce3b232006-01-06 00:20:16 -08002879 if (mddev->curr_resync < max_sector) { /* aborted */
2880 if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
2881 bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
2882 &sync_blocks, 1);
NeilBrown5cf00fc2012-05-21 09:28:20 +10002883 else for (i = 0; i < conf->geo.raid_disks; i++) {
NeilBrown6cce3b232006-01-06 00:20:16 -08002884 sector_t sect =
2885 raid10_find_virt(conf, mddev->curr_resync, i);
2886 bitmap_end_sync(mddev->bitmap, sect,
2887 &sync_blocks, 1);
2888 }
NeilBrown9ad1aef2011-12-23 10:17:55 +11002889 } else {
2890 /* completed sync */
2891 if ((!mddev->bitmap || conf->fullsync)
2892 && conf->have_replacement
2893 && test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
2894 /* Completed a full sync so the replacements
2895 * are now fully recovered.
2896 */
NeilBrown5cf00fc2012-05-21 09:28:20 +10002897 for (i = 0; i < conf->geo.raid_disks; i++)
NeilBrown9ad1aef2011-12-23 10:17:55 +11002898 if (conf->mirrors[i].replacement)
2899 conf->mirrors[i].replacement
2900 ->recovery_offset
2901 = MaxSector;
2902 }
NeilBrown6cce3b232006-01-06 00:20:16 -08002903 conf->fullsync = 0;
NeilBrown9ad1aef2011-12-23 10:17:55 +11002904 }
NeilBrown6cce3b232006-01-06 00:20:16 -08002905 bitmap_close_sync(mddev->bitmap);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002906 close_sync(conf);
NeilBrown57afd892005-06-21 17:17:13 -07002907 *skipped = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002908 return sectors_skipped;
2909 }
NeilBrown3ea7daa2012-05-22 13:53:47 +10002910
2911 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
2912 return reshape_request(mddev, sector_nr, skipped);
2913
NeilBrown5cf00fc2012-05-21 09:28:20 +10002914 if (chunks_skipped >= conf->geo.raid_disks) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002915 /* if there has been nothing to do on any drive,
2916 * then there is nothing to do at all..
2917 */
NeilBrown57afd892005-06-21 17:17:13 -07002918 *skipped = 1;
2919 return (max_sector - sector_nr) + sectors_skipped;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002920 }
2921
NeilBrownc6207272008-02-06 01:39:52 -08002922 if (max_sector > mddev->resync_max)
2923 max_sector = mddev->resync_max; /* Don't do IO beyond here */
2924
Linus Torvalds1da177e2005-04-16 15:20:36 -07002925 /* make sure whole request will fit in a chunk - if chunks
2926 * are meaningful
2927 */
NeilBrown5cf00fc2012-05-21 09:28:20 +10002928 if (conf->geo.near_copies < conf->geo.raid_disks &&
2929 max_sector > (sector_nr | chunk_mask))
2930 max_sector = (sector_nr | chunk_mask) + 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002931 /*
2932 * If there is non-resync activity waiting for us then
2933 * put in a delay to throttle resync.
2934 */
NeilBrown0a27ec92006-01-06 00:20:13 -08002935 if (!go_faster && conf->nr_waiting)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002936 msleep_interruptible(1000);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002937
2938 /* Again, very different code for resync and recovery.
2939 * Both must result in an r10bio with a list of bios that
2940 * have bi_end_io, bi_sector, bi_bdev set,
2941 * and bi_private set to the r10bio.
2942 * For recovery, we may actually create several r10bios
2943 * with 2 bios in each, that correspond to the bios in the main one.
2944 * In this case, the subordinate r10bios link back through a
2945 * borrowed master_bio pointer, and the counter in the master
2946 * includes a ref from each subordinate.
2947 */
2948 /* First, we decide what to do and set ->bi_end_io
2949 * To end_sync_read if we want to read, and
2950 * end_sync_write if we will want to write.
2951 */
2952
NeilBrown6cce3b232006-01-06 00:20:16 -08002953 max_sync = RESYNC_PAGES << (PAGE_SHIFT-9);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002954 if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
2955 /* recovery... the complicated one */
NeilBrowne875ece2011-07-28 11:39:24 +10002956 int j;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002957 r10_bio = NULL;
2958
NeilBrown5cf00fc2012-05-21 09:28:20 +10002959 for (i = 0 ; i < conf->geo.raid_disks; i++) {
NeilBrownab9d47e2011-05-11 14:54:41 +10002960 int still_degraded;
NeilBrown9f2c9d12011-10-11 16:48:43 +11002961 struct r10bio *rb2;
NeilBrownab9d47e2011-05-11 14:54:41 +10002962 sector_t sect;
2963 int must_sync;
NeilBrowne875ece2011-07-28 11:39:24 +10002964 int any_working;
Jonathan Brassowdc280d982012-07-31 10:03:52 +10002965 struct raid10_info *mirror = &conf->mirrors[i];
NeilBrownab9d47e2011-05-11 14:54:41 +10002966
NeilBrown24afd802011-12-23 10:17:55 +11002967 if ((mirror->rdev == NULL ||
2968 test_bit(In_sync, &mirror->rdev->flags))
2969 &&
2970 (mirror->replacement == NULL ||
2971 test_bit(Faulty,
2972 &mirror->replacement->flags)))
NeilBrownab9d47e2011-05-11 14:54:41 +10002973 continue;
2974
2975 still_degraded = 0;
2976 /* want to reconstruct this device */
2977 rb2 = r10_bio;
2978 sect = raid10_find_virt(conf, sector_nr, i);
NeilBrownfc448a12012-07-03 10:37:30 +10002979 if (sect >= mddev->resync_max_sectors) {
2980 /* last stripe is not complete - don't
2981 * try to recover this sector.
2982 */
2983 continue;
2984 }
NeilBrown24afd802011-12-23 10:17:55 +11002985 /* Unless we are doing a full sync, or a replacement
2986 * we only need to recover the block if it is set in
2987 * the bitmap
NeilBrownab9d47e2011-05-11 14:54:41 +10002988 */
2989 must_sync = bitmap_start_sync(mddev->bitmap, sect,
2990 &sync_blocks, 1);
2991 if (sync_blocks < max_sync)
2992 max_sync = sync_blocks;
2993 if (!must_sync &&
NeilBrown24afd802011-12-23 10:17:55 +11002994 mirror->replacement == NULL &&
NeilBrownab9d47e2011-05-11 14:54:41 +10002995 !conf->fullsync) {
2996 /* yep, skip the sync_blocks here, but don't assume
2997 * that there will never be anything to do here
NeilBrown6cce3b232006-01-06 00:20:16 -08002998 */
NeilBrownab9d47e2011-05-11 14:54:41 +10002999 chunks_skipped = -1;
3000 continue;
3001 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003002
NeilBrownab9d47e2011-05-11 14:54:41 +10003003 r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
3004 raise_barrier(conf, rb2 != NULL);
3005 atomic_set(&r10_bio->remaining, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003006
NeilBrownab9d47e2011-05-11 14:54:41 +10003007 r10_bio->master_bio = (struct bio*)rb2;
3008 if (rb2)
3009 atomic_inc(&rb2->remaining);
3010 r10_bio->mddev = mddev;
3011 set_bit(R10BIO_IsRecover, &r10_bio->state);
3012 r10_bio->sector = sect;
NeilBrown6cce3b232006-01-06 00:20:16 -08003013
NeilBrownab9d47e2011-05-11 14:54:41 +10003014 raid10_find_phys(conf, r10_bio);
NeilBrown18055562009-05-07 12:48:10 +10003015
NeilBrownab9d47e2011-05-11 14:54:41 +10003016 /* Need to check if the array will still be
3017 * degraded
3018 */
NeilBrown5cf00fc2012-05-21 09:28:20 +10003019 for (j = 0; j < conf->geo.raid_disks; j++)
NeilBrownab9d47e2011-05-11 14:54:41 +10003020 if (conf->mirrors[j].rdev == NULL ||
3021 test_bit(Faulty, &conf->mirrors[j].rdev->flags)) {
3022 still_degraded = 1;
NeilBrown87fc7672005-09-09 16:24:04 -07003023 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003024 }
NeilBrownab9d47e2011-05-11 14:54:41 +10003025
3026 must_sync = bitmap_start_sync(mddev->bitmap, sect,
3027 &sync_blocks, still_degraded);
3028
NeilBrowne875ece2011-07-28 11:39:24 +10003029 any_working = 0;
NeilBrownab9d47e2011-05-11 14:54:41 +10003030 for (j=0; j<conf->copies;j++) {
NeilBrowne875ece2011-07-28 11:39:24 +10003031 int k;
NeilBrownab9d47e2011-05-11 14:54:41 +10003032 int d = r10_bio->devs[j].devnum;
NeilBrown5e570282011-07-28 11:39:25 +10003033 sector_t from_addr, to_addr;
NeilBrown3cb03002011-10-11 16:45:26 +11003034 struct md_rdev *rdev;
NeilBrown40c356c2011-07-28 11:39:24 +10003035 sector_t sector, first_bad;
3036 int bad_sectors;
NeilBrownab9d47e2011-05-11 14:54:41 +10003037 if (!conf->mirrors[d].rdev ||
3038 !test_bit(In_sync, &conf->mirrors[d].rdev->flags))
3039 continue;
3040 /* This is where we read from */
NeilBrowne875ece2011-07-28 11:39:24 +10003041 any_working = 1;
NeilBrown40c356c2011-07-28 11:39:24 +10003042 rdev = conf->mirrors[d].rdev;
3043 sector = r10_bio->devs[j].addr;
3044
3045 if (is_badblock(rdev, sector, max_sync,
3046 &first_bad, &bad_sectors)) {
3047 if (first_bad > sector)
3048 max_sync = first_bad - sector;
3049 else {
3050 bad_sectors -= (sector
3051 - first_bad);
3052 if (max_sync > bad_sectors)
3053 max_sync = bad_sectors;
3054 continue;
3055 }
3056 }
NeilBrownab9d47e2011-05-11 14:54:41 +10003057 bio = r10_bio->devs[0].bio;
3058 bio->bi_next = biolist;
3059 biolist = bio;
3060 bio->bi_private = r10_bio;
3061 bio->bi_end_io = end_sync_read;
3062 bio->bi_rw = READ;
NeilBrown5e570282011-07-28 11:39:25 +10003063 from_addr = r10_bio->devs[j].addr;
NeilBrown24afd802011-12-23 10:17:55 +11003064 bio->bi_sector = from_addr + rdev->data_offset;
3065 bio->bi_bdev = rdev->bdev;
3066 atomic_inc(&rdev->nr_pending);
3067 /* and we write to 'i' (if not in_sync) */
NeilBrownab9d47e2011-05-11 14:54:41 +10003068
3069 for (k=0; k<conf->copies; k++)
3070 if (r10_bio->devs[k].devnum == i)
3071 break;
3072 BUG_ON(k == conf->copies);
NeilBrown5e570282011-07-28 11:39:25 +10003073 to_addr = r10_bio->devs[k].addr;
NeilBrownab9d47e2011-05-11 14:54:41 +10003074 r10_bio->devs[0].devnum = d;
NeilBrown5e570282011-07-28 11:39:25 +10003075 r10_bio->devs[0].addr = from_addr;
NeilBrownab9d47e2011-05-11 14:54:41 +10003076 r10_bio->devs[1].devnum = i;
NeilBrown5e570282011-07-28 11:39:25 +10003077 r10_bio->devs[1].addr = to_addr;
NeilBrownab9d47e2011-05-11 14:54:41 +10003078
NeilBrown24afd802011-12-23 10:17:55 +11003079 rdev = mirror->rdev;
3080 if (!test_bit(In_sync, &rdev->flags)) {
3081 bio = r10_bio->devs[1].bio;
3082 bio->bi_next = biolist;
3083 biolist = bio;
3084 bio->bi_private = r10_bio;
3085 bio->bi_end_io = end_sync_write;
3086 bio->bi_rw = WRITE;
3087 bio->bi_sector = to_addr
3088 + rdev->data_offset;
3089 bio->bi_bdev = rdev->bdev;
3090 atomic_inc(&r10_bio->remaining);
3091 } else
3092 r10_bio->devs[1].bio->bi_end_io = NULL;
3093
3094 /* and maybe write to replacement */
3095 bio = r10_bio->devs[1].repl_bio;
3096 if (bio)
3097 bio->bi_end_io = NULL;
3098 rdev = mirror->replacement;
3099 /* Note: if rdev != NULL, then bio
3100 * cannot be NULL as r10buf_pool_alloc will
3101 * have allocated it.
3102 * So the second test here is pointless.
3103 * But it keeps semantic-checkers happy, and
3104 * this comment keeps human reviewers
3105 * happy.
3106 */
3107 if (rdev == NULL || bio == NULL ||
3108 test_bit(Faulty, &rdev->flags))
3109 break;
3110 bio->bi_next = biolist;
3111 biolist = bio;
3112 bio->bi_private = r10_bio;
3113 bio->bi_end_io = end_sync_write;
3114 bio->bi_rw = WRITE;
3115 bio->bi_sector = to_addr + rdev->data_offset;
3116 bio->bi_bdev = rdev->bdev;
3117 atomic_inc(&r10_bio->remaining);
NeilBrownab9d47e2011-05-11 14:54:41 +10003118 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003119 }
NeilBrownab9d47e2011-05-11 14:54:41 +10003120 if (j == conf->copies) {
NeilBrowne875ece2011-07-28 11:39:24 +10003121 /* Cannot recover, so abort the recovery or
3122 * record a bad block */
NeilBrownab9d47e2011-05-11 14:54:41 +10003123 put_buf(r10_bio);
3124 if (rb2)
3125 atomic_dec(&rb2->remaining);
3126 r10_bio = rb2;
NeilBrowne875ece2011-07-28 11:39:24 +10003127 if (any_working) {
3128 /* problem is that there are bad blocks
3129 * on other device(s)
3130 */
3131 int k;
3132 for (k = 0; k < conf->copies; k++)
3133 if (r10_bio->devs[k].devnum == i)
3134 break;
NeilBrown24afd802011-12-23 10:17:55 +11003135 if (!test_bit(In_sync,
3136 &mirror->rdev->flags)
3137 && !rdev_set_badblocks(
3138 mirror->rdev,
3139 r10_bio->devs[k].addr,
3140 max_sync, 0))
3141 any_working = 0;
3142 if (mirror->replacement &&
3143 !rdev_set_badblocks(
3144 mirror->replacement,
NeilBrowne875ece2011-07-28 11:39:24 +10003145 r10_bio->devs[k].addr,
3146 max_sync, 0))
3147 any_working = 0;
3148 }
3149 if (!any_working) {
3150 if (!test_and_set_bit(MD_RECOVERY_INTR,
3151 &mddev->recovery))
3152 printk(KERN_INFO "md/raid10:%s: insufficient "
3153 "working devices for recovery.\n",
3154 mdname(mddev));
NeilBrown24afd802011-12-23 10:17:55 +11003155 mirror->recovery_disabled
NeilBrowne875ece2011-07-28 11:39:24 +10003156 = mddev->recovery_disabled;
3157 }
NeilBrownab9d47e2011-05-11 14:54:41 +10003158 break;
3159 }
3160 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003161 if (biolist == NULL) {
3162 while (r10_bio) {
NeilBrown9f2c9d12011-10-11 16:48:43 +11003163 struct r10bio *rb2 = r10_bio;
3164 r10_bio = (struct r10bio*) rb2->master_bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003165 rb2->master_bio = NULL;
3166 put_buf(rb2);
3167 }
3168 goto giveup;
3169 }
3170 } else {
3171 /* resync. Schedule a read for every block at this virt offset */
3172 int count = 0;
NeilBrown6cce3b232006-01-06 00:20:16 -08003173
NeilBrown78200d42009-02-25 13:18:47 +11003174 bitmap_cond_end_sync(mddev->bitmap, sector_nr);
3175
NeilBrown6cce3b232006-01-06 00:20:16 -08003176 if (!bitmap_start_sync(mddev->bitmap, sector_nr,
3177 &sync_blocks, mddev->degraded) &&
NeilBrownab9d47e2011-05-11 14:54:41 +10003178 !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED,
3179 &mddev->recovery)) {
NeilBrown6cce3b232006-01-06 00:20:16 -08003180 /* We can skip this block */
3181 *skipped = 1;
3182 return sync_blocks + sectors_skipped;
3183 }
3184 if (sync_blocks < max_sync)
3185 max_sync = sync_blocks;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003186 r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
3187
Linus Torvalds1da177e2005-04-16 15:20:36 -07003188 r10_bio->mddev = mddev;
3189 atomic_set(&r10_bio->remaining, 0);
NeilBrown6cce3b232006-01-06 00:20:16 -08003190 raise_barrier(conf, 0);
3191 conf->next_resync = sector_nr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003192
3193 r10_bio->master_bio = NULL;
3194 r10_bio->sector = sector_nr;
3195 set_bit(R10BIO_IsSync, &r10_bio->state);
3196 raid10_find_phys(conf, r10_bio);
NeilBrown5cf00fc2012-05-21 09:28:20 +10003197 r10_bio->sectors = (sector_nr | chunk_mask) - sector_nr + 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003198
NeilBrown5cf00fc2012-05-21 09:28:20 +10003199 for (i = 0; i < conf->copies; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003200 int d = r10_bio->devs[i].devnum;
NeilBrown40c356c2011-07-28 11:39:24 +10003201 sector_t first_bad, sector;
3202 int bad_sectors;
3203
NeilBrown9ad1aef2011-12-23 10:17:55 +11003204 if (r10_bio->devs[i].repl_bio)
3205 r10_bio->devs[i].repl_bio->bi_end_io = NULL;
3206
Linus Torvalds1da177e2005-04-16 15:20:36 -07003207 bio = r10_bio->devs[i].bio;
3208 bio->bi_end_io = NULL;
NeilBrownaf03b8e2007-06-16 10:16:06 -07003209 clear_bit(BIO_UPTODATE, &bio->bi_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003210 if (conf->mirrors[d].rdev == NULL ||
NeilBrownb2d444d2005-11-08 21:39:31 -08003211 test_bit(Faulty, &conf->mirrors[d].rdev->flags))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003212 continue;
NeilBrown40c356c2011-07-28 11:39:24 +10003213 sector = r10_bio->devs[i].addr;
3214 if (is_badblock(conf->mirrors[d].rdev,
3215 sector, max_sync,
3216 &first_bad, &bad_sectors)) {
3217 if (first_bad > sector)
3218 max_sync = first_bad - sector;
3219 else {
3220 bad_sectors -= (sector - first_bad);
3221 if (max_sync > bad_sectors)
Dan Carpenter91502f02012-10-11 14:20:58 +11003222 max_sync = bad_sectors;
NeilBrown40c356c2011-07-28 11:39:24 +10003223 continue;
3224 }
3225 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003226 atomic_inc(&conf->mirrors[d].rdev->nr_pending);
3227 atomic_inc(&r10_bio->remaining);
3228 bio->bi_next = biolist;
3229 biolist = bio;
3230 bio->bi_private = r10_bio;
3231 bio->bi_end_io = end_sync_read;
NeilBrown802ba062006-12-13 00:34:13 -08003232 bio->bi_rw = READ;
NeilBrown40c356c2011-07-28 11:39:24 +10003233 bio->bi_sector = sector +
Linus Torvalds1da177e2005-04-16 15:20:36 -07003234 conf->mirrors[d].rdev->data_offset;
3235 bio->bi_bdev = conf->mirrors[d].rdev->bdev;
3236 count++;
NeilBrown9ad1aef2011-12-23 10:17:55 +11003237
3238 if (conf->mirrors[d].replacement == NULL ||
3239 test_bit(Faulty,
3240 &conf->mirrors[d].replacement->flags))
3241 continue;
3242
3243 /* Need to set up for writing to the replacement */
3244 bio = r10_bio->devs[i].repl_bio;
3245 clear_bit(BIO_UPTODATE, &bio->bi_flags);
3246
3247 sector = r10_bio->devs[i].addr;
3248 atomic_inc(&conf->mirrors[d].rdev->nr_pending);
3249 bio->bi_next = biolist;
3250 biolist = bio;
3251 bio->bi_private = r10_bio;
3252 bio->bi_end_io = end_sync_write;
3253 bio->bi_rw = WRITE;
3254 bio->bi_sector = sector +
3255 conf->mirrors[d].replacement->data_offset;
3256 bio->bi_bdev = conf->mirrors[d].replacement->bdev;
3257 count++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003258 }
3259
3260 if (count < 2) {
3261 for (i=0; i<conf->copies; i++) {
3262 int d = r10_bio->devs[i].devnum;
3263 if (r10_bio->devs[i].bio->bi_end_io)
NeilBrownab9d47e2011-05-11 14:54:41 +10003264 rdev_dec_pending(conf->mirrors[d].rdev,
3265 mddev);
NeilBrown9ad1aef2011-12-23 10:17:55 +11003266 if (r10_bio->devs[i].repl_bio &&
3267 r10_bio->devs[i].repl_bio->bi_end_io)
3268 rdev_dec_pending(
3269 conf->mirrors[d].replacement,
3270 mddev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003271 }
3272 put_buf(r10_bio);
3273 biolist = NULL;
3274 goto giveup;
3275 }
3276 }
3277
3278 for (bio = biolist; bio ; bio=bio->bi_next) {
3279
3280 bio->bi_flags &= ~(BIO_POOL_MASK - 1);
3281 if (bio->bi_end_io)
3282 bio->bi_flags |= 1 << BIO_UPTODATE;
3283 bio->bi_vcnt = 0;
3284 bio->bi_idx = 0;
3285 bio->bi_phys_segments = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003286 bio->bi_size = 0;
3287 }
3288
3289 nr_sectors = 0;
NeilBrown6cce3b232006-01-06 00:20:16 -08003290 if (sector_nr + max_sync < max_sector)
3291 max_sector = sector_nr + max_sync;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003292 do {
3293 struct page *page;
3294 int len = PAGE_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003295 if (sector_nr + (len>>9) > max_sector)
3296 len = (max_sector - sector_nr) << 9;
3297 if (len == 0)
3298 break;
3299 for (bio= biolist ; bio ; bio=bio->bi_next) {
NeilBrownab9d47e2011-05-11 14:54:41 +10003300 struct bio *bio2;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003301 page = bio->bi_io_vec[bio->bi_vcnt].bv_page;
NeilBrownab9d47e2011-05-11 14:54:41 +10003302 if (bio_add_page(bio, page, len, 0))
3303 continue;
3304
3305 /* stop here */
3306 bio->bi_io_vec[bio->bi_vcnt].bv_page = page;
3307 for (bio2 = biolist;
3308 bio2 && bio2 != bio;
3309 bio2 = bio2->bi_next) {
3310 /* remove last page from this bio */
3311 bio2->bi_vcnt--;
3312 bio2->bi_size -= len;
3313 bio2->bi_flags &= ~(1<< BIO_SEG_VALID);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003314 }
NeilBrownab9d47e2011-05-11 14:54:41 +10003315 goto bio_full;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003316 }
3317 nr_sectors += len>>9;
3318 sector_nr += len>>9;
3319 } while (biolist->bi_vcnt < RESYNC_PAGES);
3320 bio_full:
3321 r10_bio->sectors = nr_sectors;
3322
3323 while (biolist) {
3324 bio = biolist;
3325 biolist = biolist->bi_next;
3326
3327 bio->bi_next = NULL;
3328 r10_bio = bio->bi_private;
3329 r10_bio->sectors = nr_sectors;
3330
3331 if (bio->bi_end_io == end_sync_read) {
3332 md_sync_acct(bio->bi_bdev, nr_sectors);
3333 generic_make_request(bio);
3334 }
3335 }
3336
NeilBrown57afd892005-06-21 17:17:13 -07003337 if (sectors_skipped)
3338 /* pretend they weren't skipped, it makes
3339 * no important difference in this case
3340 */
3341 md_done_sync(mddev, sectors_skipped, 1);
3342
Linus Torvalds1da177e2005-04-16 15:20:36 -07003343 return sectors_skipped + nr_sectors;
3344 giveup:
3345 /* There is nowhere to write, so all non-sync
NeilBrowne875ece2011-07-28 11:39:24 +10003346 * drives must be failed or in resync, all drives
3347 * have a bad block, so try the next chunk...
Linus Torvalds1da177e2005-04-16 15:20:36 -07003348 */
NeilBrown09b40682009-02-25 13:18:47 +11003349 if (sector_nr + max_sync < max_sector)
3350 max_sector = sector_nr + max_sync;
3351
3352 sectors_skipped += (max_sector - sector_nr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003353 chunks_skipped ++;
3354 sector_nr = max_sector;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003355 goto skipped;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003356}
3357
Dan Williams80c3a6c2009-03-17 18:10:40 -07003358static sector_t
NeilBrownfd01b882011-10-11 16:47:53 +11003359raid10_size(struct mddev *mddev, sector_t sectors, int raid_disks)
Dan Williams80c3a6c2009-03-17 18:10:40 -07003360{
3361 sector_t size;
NeilBrowne879a872011-10-11 16:49:02 +11003362 struct r10conf *conf = mddev->private;
Dan Williams80c3a6c2009-03-17 18:10:40 -07003363
3364 if (!raid_disks)
NeilBrown3ea7daa2012-05-22 13:53:47 +10003365 raid_disks = min(conf->geo.raid_disks,
3366 conf->prev.raid_disks);
Dan Williams80c3a6c2009-03-17 18:10:40 -07003367 if (!sectors)
Trela, Maciejdab8b292010-03-08 16:02:45 +11003368 sectors = conf->dev_sectors;
Dan Williams80c3a6c2009-03-17 18:10:40 -07003369
NeilBrown5cf00fc2012-05-21 09:28:20 +10003370 size = sectors >> conf->geo.chunk_shift;
3371 sector_div(size, conf->geo.far_copies);
Dan Williams80c3a6c2009-03-17 18:10:40 -07003372 size = size * raid_disks;
NeilBrown5cf00fc2012-05-21 09:28:20 +10003373 sector_div(size, conf->geo.near_copies);
Dan Williams80c3a6c2009-03-17 18:10:40 -07003374
NeilBrown5cf00fc2012-05-21 09:28:20 +10003375 return size << conf->geo.chunk_shift;
Dan Williams80c3a6c2009-03-17 18:10:40 -07003376}
3377
NeilBrown6508fdb2012-05-17 10:08:45 +10003378static void calc_sectors(struct r10conf *conf, sector_t size)
3379{
3380 /* Calculate the number of sectors-per-device that will
3381 * actually be used, and set conf->dev_sectors and
3382 * conf->stride
3383 */
3384
NeilBrown5cf00fc2012-05-21 09:28:20 +10003385 size = size >> conf->geo.chunk_shift;
3386 sector_div(size, conf->geo.far_copies);
3387 size = size * conf->geo.raid_disks;
3388 sector_div(size, conf->geo.near_copies);
NeilBrown6508fdb2012-05-17 10:08:45 +10003389 /* 'size' is now the number of chunks in the array */
3390 /* calculate "used chunks per device" */
3391 size = size * conf->copies;
3392
3393 /* We need to round up when dividing by raid_disks to
3394 * get the stride size.
3395 */
NeilBrown5cf00fc2012-05-21 09:28:20 +10003396 size = DIV_ROUND_UP_SECTOR_T(size, conf->geo.raid_disks);
NeilBrown6508fdb2012-05-17 10:08:45 +10003397
NeilBrown5cf00fc2012-05-21 09:28:20 +10003398 conf->dev_sectors = size << conf->geo.chunk_shift;
NeilBrown6508fdb2012-05-17 10:08:45 +10003399
NeilBrown5cf00fc2012-05-21 09:28:20 +10003400 if (conf->geo.far_offset)
3401 conf->geo.stride = 1 << conf->geo.chunk_shift;
NeilBrown6508fdb2012-05-17 10:08:45 +10003402 else {
NeilBrown5cf00fc2012-05-21 09:28:20 +10003403 sector_div(size, conf->geo.far_copies);
3404 conf->geo.stride = size << conf->geo.chunk_shift;
NeilBrown6508fdb2012-05-17 10:08:45 +10003405 }
3406}
Trela, Maciejdab8b292010-03-08 16:02:45 +11003407
NeilBrowndeb200d2012-05-21 09:28:33 +10003408enum geo_type {geo_new, geo_old, geo_start};
3409static int setup_geo(struct geom *geo, struct mddev *mddev, enum geo_type new)
3410{
3411 int nc, fc, fo;
3412 int layout, chunk, disks;
3413 switch (new) {
3414 case geo_old:
3415 layout = mddev->layout;
3416 chunk = mddev->chunk_sectors;
3417 disks = mddev->raid_disks - mddev->delta_disks;
3418 break;
3419 case geo_new:
3420 layout = mddev->new_layout;
3421 chunk = mddev->new_chunk_sectors;
3422 disks = mddev->raid_disks;
3423 break;
3424 default: /* avoid 'may be unused' warnings */
3425 case geo_start: /* new when starting reshape - raid_disks not
3426 * updated yet. */
3427 layout = mddev->new_layout;
3428 chunk = mddev->new_chunk_sectors;
3429 disks = mddev->raid_disks + mddev->delta_disks;
3430 break;
3431 }
3432 if (layout >> 17)
3433 return -1;
3434 if (chunk < (PAGE_SIZE >> 9) ||
3435 !is_power_of_2(chunk))
3436 return -2;
3437 nc = layout & 255;
3438 fc = (layout >> 8) & 255;
3439 fo = layout & (1<<16);
3440 geo->raid_disks = disks;
3441 geo->near_copies = nc;
3442 geo->far_copies = fc;
3443 geo->far_offset = fo;
3444 geo->chunk_mask = chunk - 1;
3445 geo->chunk_shift = ffz(~chunk);
3446 return nc*fc;
3447}
3448
NeilBrowne879a872011-10-11 16:49:02 +11003449static struct r10conf *setup_conf(struct mddev *mddev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003450{
NeilBrowne879a872011-10-11 16:49:02 +11003451 struct r10conf *conf = NULL;
Trela, Maciejdab8b292010-03-08 16:02:45 +11003452 int err = -EINVAL;
NeilBrowndeb200d2012-05-21 09:28:33 +10003453 struct geom geo;
3454 int copies;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003455
NeilBrowndeb200d2012-05-21 09:28:33 +10003456 copies = setup_geo(&geo, mddev, geo_new);
3457
3458 if (copies == -2) {
NeilBrown128595e2010-05-03 14:47:14 +10003459 printk(KERN_ERR "md/raid10:%s: chunk size must be "
3460 "at least PAGE_SIZE(%ld) and be a power of 2.\n",
3461 mdname(mddev), PAGE_SIZE);
Trela, Maciejdab8b292010-03-08 16:02:45 +11003462 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003463 }
NeilBrown2604b702006-01-06 00:20:36 -08003464
NeilBrowndeb200d2012-05-21 09:28:33 +10003465 if (copies < 2 || copies > mddev->raid_disks) {
NeilBrown128595e2010-05-03 14:47:14 +10003466 printk(KERN_ERR "md/raid10:%s: unsupported raid10 layout: 0x%8x\n",
Maciej Trelaf73ea872010-06-16 11:46:29 +01003467 mdname(mddev), mddev->new_layout);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003468 goto out;
3469 }
Trela, Maciejdab8b292010-03-08 16:02:45 +11003470
3471 err = -ENOMEM;
NeilBrowne879a872011-10-11 16:49:02 +11003472 conf = kzalloc(sizeof(struct r10conf), GFP_KERNEL);
Trela, Maciejdab8b292010-03-08 16:02:45 +11003473 if (!conf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003474 goto out;
Trela, Maciejdab8b292010-03-08 16:02:45 +11003475
NeilBrown3ea7daa2012-05-22 13:53:47 +10003476 /* FIXME calc properly */
Jonathan Brassowdc280d982012-07-31 10:03:52 +10003477 conf->mirrors = kzalloc(sizeof(struct raid10_info)*(mddev->raid_disks +
NeilBrown3ea7daa2012-05-22 13:53:47 +10003478 max(0,mddev->delta_disks)),
Trela, Maciejdab8b292010-03-08 16:02:45 +11003479 GFP_KERNEL);
3480 if (!conf->mirrors)
3481 goto out;
NeilBrown4443ae12006-01-06 00:20:28 -08003482
3483 conf->tmppage = alloc_page(GFP_KERNEL);
3484 if (!conf->tmppage)
Trela, Maciejdab8b292010-03-08 16:02:45 +11003485 goto out;
3486
NeilBrowndeb200d2012-05-21 09:28:33 +10003487 conf->geo = geo;
3488 conf->copies = copies;
Trela, Maciejdab8b292010-03-08 16:02:45 +11003489 conf->r10bio_pool = mempool_create(NR_RAID10_BIOS, r10bio_pool_alloc,
3490 r10bio_pool_free, conf);
3491 if (!conf->r10bio_pool)
3492 goto out;
3493
NeilBrown6508fdb2012-05-17 10:08:45 +10003494 calc_sectors(conf, mddev->dev_sectors);
NeilBrown3ea7daa2012-05-22 13:53:47 +10003495 if (mddev->reshape_position == MaxSector) {
3496 conf->prev = conf->geo;
3497 conf->reshape_progress = MaxSector;
3498 } else {
3499 if (setup_geo(&conf->prev, mddev, geo_old) != conf->copies) {
3500 err = -EINVAL;
3501 goto out;
3502 }
3503 conf->reshape_progress = mddev->reshape_position;
3504 if (conf->prev.far_offset)
3505 conf->prev.stride = 1 << conf->prev.chunk_shift;
3506 else
3507 /* far_copies must be 1 */
3508 conf->prev.stride = conf->dev_sectors;
3509 }
Neil Browne7e72bf2008-05-14 16:05:54 -07003510 spin_lock_init(&conf->device_lock);
Trela, Maciejdab8b292010-03-08 16:02:45 +11003511 INIT_LIST_HEAD(&conf->retry_list);
3512
3513 spin_lock_init(&conf->resync_lock);
3514 init_waitqueue_head(&conf->wait_barrier);
3515
NeilBrown02326052012-07-03 15:56:52 +10003516 conf->thread = md_register_thread(raid10d, mddev, "raid10");
Trela, Maciejdab8b292010-03-08 16:02:45 +11003517 if (!conf->thread)
3518 goto out;
3519
Trela, Maciejdab8b292010-03-08 16:02:45 +11003520 conf->mddev = mddev;
3521 return conf;
3522
3523 out:
NeilBrown3ea7daa2012-05-22 13:53:47 +10003524 if (err == -ENOMEM)
3525 printk(KERN_ERR "md/raid10:%s: couldn't allocate memory.\n",
3526 mdname(mddev));
Trela, Maciejdab8b292010-03-08 16:02:45 +11003527 if (conf) {
3528 if (conf->r10bio_pool)
3529 mempool_destroy(conf->r10bio_pool);
3530 kfree(conf->mirrors);
3531 safe_put_page(conf->tmppage);
3532 kfree(conf);
3533 }
3534 return ERR_PTR(err);
3535}
3536
NeilBrownfd01b882011-10-11 16:47:53 +11003537static int run(struct mddev *mddev)
Trela, Maciejdab8b292010-03-08 16:02:45 +11003538{
NeilBrowne879a872011-10-11 16:49:02 +11003539 struct r10conf *conf;
Trela, Maciejdab8b292010-03-08 16:02:45 +11003540 int i, disk_idx, chunk_size;
Jonathan Brassowdc280d982012-07-31 10:03:52 +10003541 struct raid10_info *disk;
NeilBrown3cb03002011-10-11 16:45:26 +11003542 struct md_rdev *rdev;
Trela, Maciejdab8b292010-03-08 16:02:45 +11003543 sector_t size;
NeilBrown3ea7daa2012-05-22 13:53:47 +10003544 sector_t min_offset_diff = 0;
3545 int first = 1;
Shaohua Li532a2a32012-10-11 13:30:52 +11003546 bool discard_supported = false;
Trela, Maciejdab8b292010-03-08 16:02:45 +11003547
3548 if (mddev->private == NULL) {
3549 conf = setup_conf(mddev);
3550 if (IS_ERR(conf))
3551 return PTR_ERR(conf);
3552 mddev->private = conf;
3553 }
3554 conf = mddev->private;
3555 if (!conf)
3556 goto out;
3557
Trela, Maciejdab8b292010-03-08 16:02:45 +11003558 mddev->thread = conf->thread;
3559 conf->thread = NULL;
3560
Martin K. Petersen8f6c2e42009-07-01 11:13:45 +10003561 chunk_size = mddev->chunk_sectors << 9;
Jonathan Brassowcc4d1ef2012-07-31 10:03:53 +10003562 if (mddev->queue) {
Shaohua Li532a2a32012-10-11 13:30:52 +11003563 blk_queue_max_discard_sectors(mddev->queue,
3564 mddev->chunk_sectors);
Jonathan Brassowcc4d1ef2012-07-31 10:03:53 +10003565 blk_queue_io_min(mddev->queue, chunk_size);
3566 if (conf->geo.raid_disks % conf->geo.near_copies)
3567 blk_queue_io_opt(mddev->queue, chunk_size * conf->geo.raid_disks);
3568 else
3569 blk_queue_io_opt(mddev->queue, chunk_size *
3570 (conf->geo.raid_disks / conf->geo.near_copies));
3571 }
Martin K. Petersen8f6c2e42009-07-01 11:13:45 +10003572
NeilBrowndafb20f2012-03-19 12:46:39 +11003573 rdev_for_each(rdev, mddev) {
NeilBrown3ea7daa2012-05-22 13:53:47 +10003574 long long diff;
NeilBrownaba336b2012-05-31 15:39:11 +10003575 struct request_queue *q;
NeilBrown34b343c2011-07-28 11:31:47 +10003576
Linus Torvalds1da177e2005-04-16 15:20:36 -07003577 disk_idx = rdev->raid_disk;
NeilBrownf8c9e742012-05-21 09:28:33 +10003578 if (disk_idx < 0)
3579 continue;
3580 if (disk_idx >= conf->geo.raid_disks &&
3581 disk_idx >= conf->prev.raid_disks)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003582 continue;
3583 disk = conf->mirrors + disk_idx;
3584
NeilBrown56a25592011-12-23 10:17:55 +11003585 if (test_bit(Replacement, &rdev->flags)) {
3586 if (disk->replacement)
3587 goto out_free_conf;
3588 disk->replacement = rdev;
3589 } else {
3590 if (disk->rdev)
3591 goto out_free_conf;
3592 disk->rdev = rdev;
3593 }
NeilBrownaba336b2012-05-31 15:39:11 +10003594 q = bdev_get_queue(rdev->bdev);
3595 if (q->merge_bvec_fn)
3596 mddev->merge_check_needed = 1;
NeilBrown3ea7daa2012-05-22 13:53:47 +10003597 diff = (rdev->new_data_offset - rdev->data_offset);
3598 if (!mddev->reshape_backwards)
3599 diff = -diff;
3600 if (diff < 0)
3601 diff = 0;
3602 if (first || diff < min_offset_diff)
3603 min_offset_diff = diff;
NeilBrown56a25592011-12-23 10:17:55 +11003604
Jonathan Brassowcc4d1ef2012-07-31 10:03:53 +10003605 if (mddev->gendisk)
3606 disk_stack_limits(mddev->gendisk, rdev->bdev,
3607 rdev->data_offset << 9);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003608
3609 disk->head_position = 0;
Shaohua Li532a2a32012-10-11 13:30:52 +11003610
3611 if (blk_queue_discard(bdev_get_queue(rdev->bdev)))
3612 discard_supported = true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003613 }
NeilBrown3ea7daa2012-05-22 13:53:47 +10003614
Shaohua Li532a2a32012-10-11 13:30:52 +11003615 if (discard_supported)
3616 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue);
3617 else
3618 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, mddev->queue);
3619
NeilBrown6d508242005-09-09 16:24:03 -07003620 /* need to check that every block has at least one working mirror */
NeilBrown700c7212011-07-27 11:00:36 +10003621 if (!enough(conf, -1)) {
NeilBrown128595e2010-05-03 14:47:14 +10003622 printk(KERN_ERR "md/raid10:%s: not enough operational mirrors.\n",
NeilBrown6d508242005-09-09 16:24:03 -07003623 mdname(mddev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003624 goto out_free_conf;
3625 }
3626
NeilBrown3ea7daa2012-05-22 13:53:47 +10003627 if (conf->reshape_progress != MaxSector) {
3628 /* must ensure that shape change is supported */
3629 if (conf->geo.far_copies != 1 &&
3630 conf->geo.far_offset == 0)
3631 goto out_free_conf;
3632 if (conf->prev.far_copies != 1 &&
3633 conf->geo.far_offset == 0)
3634 goto out_free_conf;
3635 }
3636
Linus Torvalds1da177e2005-04-16 15:20:36 -07003637 mddev->degraded = 0;
NeilBrownf8c9e742012-05-21 09:28:33 +10003638 for (i = 0;
3639 i < conf->geo.raid_disks
3640 || i < conf->prev.raid_disks;
3641 i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003642
3643 disk = conf->mirrors + i;
3644
NeilBrown56a25592011-12-23 10:17:55 +11003645 if (!disk->rdev && disk->replacement) {
3646 /* The replacement is all we have - use it */
3647 disk->rdev = disk->replacement;
3648 disk->replacement = NULL;
3649 clear_bit(Replacement, &disk->rdev->flags);
3650 }
3651
NeilBrown5fd6c1d2006-06-26 00:27:40 -07003652 if (!disk->rdev ||
NeilBrown2e333e82006-10-21 10:24:07 -07003653 !test_bit(In_sync, &disk->rdev->flags)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003654 disk->head_position = 0;
3655 mddev->degraded++;
Neil Brown8c2e8702008-06-28 08:30:52 +10003656 if (disk->rdev)
3657 conf->fullsync = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003658 }
NeilBrownd890fa22011-10-26 11:54:39 +11003659 disk->recovery_disabled = mddev->recovery_disabled - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003660 }
3661
Andre Noll8c6ac862009-06-18 08:48:06 +10003662 if (mddev->recovery_cp != MaxSector)
NeilBrown128595e2010-05-03 14:47:14 +10003663 printk(KERN_NOTICE "md/raid10:%s: not clean"
Andre Noll8c6ac862009-06-18 08:48:06 +10003664 " -- starting background reconstruction\n",
3665 mdname(mddev));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003666 printk(KERN_INFO
NeilBrown128595e2010-05-03 14:47:14 +10003667 "md/raid10:%s: active with %d out of %d devices\n",
NeilBrown5cf00fc2012-05-21 09:28:20 +10003668 mdname(mddev), conf->geo.raid_disks - mddev->degraded,
3669 conf->geo.raid_disks);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003670 /*
3671 * Ok, everything is just fine now
3672 */
Trela, Maciejdab8b292010-03-08 16:02:45 +11003673 mddev->dev_sectors = conf->dev_sectors;
3674 size = raid10_size(mddev, 0, 0);
3675 md_set_array_sectors(mddev, size);
3676 mddev->resync_max_sectors = size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003677
Jonathan Brassowcc4d1ef2012-07-31 10:03:53 +10003678 if (mddev->queue) {
NeilBrown5cf00fc2012-05-21 09:28:20 +10003679 int stripe = conf->geo.raid_disks *
Andre Noll9d8f0362009-06-18 08:45:01 +10003680 ((mddev->chunk_sectors << 9) / PAGE_SIZE);
Jonathan Brassowcc4d1ef2012-07-31 10:03:53 +10003681 mddev->queue->backing_dev_info.congested_fn = raid10_congested;
3682 mddev->queue->backing_dev_info.congested_data = mddev;
3683
3684 /* Calculate max read-ahead size.
3685 * We need to readahead at least twice a whole stripe....
3686 * maybe...
3687 */
NeilBrown5cf00fc2012-05-21 09:28:20 +10003688 stripe /= conf->geo.near_copies;
NeilBrown3ea7daa2012-05-22 13:53:47 +10003689 if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
3690 mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
Jonathan Brassowcc4d1ef2012-07-31 10:03:53 +10003691 blk_queue_merge_bvec(mddev->queue, raid10_mergeable_bvec);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003692 }
3693
Martin K. Petersena91a2782011-03-17 11:11:05 +01003694
3695 if (md_integrity_register(mddev))
3696 goto out_free_conf;
3697
NeilBrown3ea7daa2012-05-22 13:53:47 +10003698 if (conf->reshape_progress != MaxSector) {
3699 unsigned long before_length, after_length;
3700
3701 before_length = ((1 << conf->prev.chunk_shift) *
3702 conf->prev.far_copies);
3703 after_length = ((1 << conf->geo.chunk_shift) *
3704 conf->geo.far_copies);
3705
3706 if (max(before_length, after_length) > min_offset_diff) {
3707 /* This cannot work */
3708 printk("md/raid10: offset difference not enough to continue reshape\n");
3709 goto out_free_conf;
3710 }
3711 conf->offset_diff = min_offset_diff;
3712
3713 conf->reshape_safe = conf->reshape_progress;
3714 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
3715 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
3716 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
3717 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
3718 mddev->sync_thread = md_register_thread(md_do_sync, mddev,
3719 "reshape");
3720 }
3721
Linus Torvalds1da177e2005-04-16 15:20:36 -07003722 return 0;
3723
3724out_free_conf:
NeilBrown01f96c02011-09-21 15:30:20 +10003725 md_unregister_thread(&mddev->thread);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003726 if (conf->r10bio_pool)
3727 mempool_destroy(conf->r10bio_pool);
NeilBrown1345b1d2006-01-06 00:20:40 -08003728 safe_put_page(conf->tmppage);
Jesper Juhl990a8ba2005-06-21 17:17:30 -07003729 kfree(conf->mirrors);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003730 kfree(conf);
3731 mddev->private = NULL;
3732out:
3733 return -EIO;
3734}
3735
NeilBrownfd01b882011-10-11 16:47:53 +11003736static int stop(struct mddev *mddev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003737{
NeilBrowne879a872011-10-11 16:49:02 +11003738 struct r10conf *conf = mddev->private;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003739
NeilBrown409c57f2009-03-31 14:39:39 +11003740 raise_barrier(conf, 0);
3741 lower_barrier(conf);
3742
NeilBrown01f96c02011-09-21 15:30:20 +10003743 md_unregister_thread(&mddev->thread);
Jonathan Brassowcc4d1ef2012-07-31 10:03:53 +10003744 if (mddev->queue)
3745 /* the unplug fn references 'conf'*/
3746 blk_sync_queue(mddev->queue);
3747
Linus Torvalds1da177e2005-04-16 15:20:36 -07003748 if (conf->r10bio_pool)
3749 mempool_destroy(conf->r10bio_pool);
Jesper Juhl990a8ba2005-06-21 17:17:30 -07003750 kfree(conf->mirrors);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003751 kfree(conf);
3752 mddev->private = NULL;
3753 return 0;
3754}
3755
NeilBrownfd01b882011-10-11 16:47:53 +11003756static void raid10_quiesce(struct mddev *mddev, int state)
NeilBrown6cce3b232006-01-06 00:20:16 -08003757{
NeilBrowne879a872011-10-11 16:49:02 +11003758 struct r10conf *conf = mddev->private;
NeilBrown6cce3b232006-01-06 00:20:16 -08003759
3760 switch(state) {
3761 case 1:
3762 raise_barrier(conf, 0);
3763 break;
3764 case 0:
3765 lower_barrier(conf);
3766 break;
3767 }
NeilBrown6cce3b232006-01-06 00:20:16 -08003768}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003769
NeilBrown006a09a2012-03-19 12:46:40 +11003770static int raid10_resize(struct mddev *mddev, sector_t sectors)
3771{
3772 /* Resize of 'far' arrays is not supported.
3773 * For 'near' and 'offset' arrays we can set the
3774 * number of sectors used to be an appropriate multiple
3775 * of the chunk size.
3776 * For 'offset', this is far_copies*chunksize.
3777 * For 'near' the multiplier is the LCM of
3778 * near_copies and raid_disks.
3779 * So if far_copies > 1 && !far_offset, fail.
3780 * Else find LCM(raid_disks, near_copy)*far_copies and
3781 * multiply by chunk_size. Then round to this number.
3782 * This is mostly done by raid10_size()
3783 */
3784 struct r10conf *conf = mddev->private;
3785 sector_t oldsize, size;
3786
NeilBrownf8c9e742012-05-21 09:28:33 +10003787 if (mddev->reshape_position != MaxSector)
3788 return -EBUSY;
3789
NeilBrown5cf00fc2012-05-21 09:28:20 +10003790 if (conf->geo.far_copies > 1 && !conf->geo.far_offset)
NeilBrown006a09a2012-03-19 12:46:40 +11003791 return -EINVAL;
3792
3793 oldsize = raid10_size(mddev, 0, 0);
3794 size = raid10_size(mddev, sectors, 0);
NeilBrowna4a61252012-05-22 13:55:27 +10003795 if (mddev->external_size &&
3796 mddev->array_sectors > size)
NeilBrown006a09a2012-03-19 12:46:40 +11003797 return -EINVAL;
NeilBrowna4a61252012-05-22 13:55:27 +10003798 if (mddev->bitmap) {
3799 int ret = bitmap_resize(mddev->bitmap, size, 0, 0);
3800 if (ret)
3801 return ret;
3802 }
3803 md_set_array_sectors(mddev, size);
NeilBrown006a09a2012-03-19 12:46:40 +11003804 set_capacity(mddev->gendisk, mddev->array_sectors);
3805 revalidate_disk(mddev->gendisk);
3806 if (sectors > mddev->dev_sectors &&
3807 mddev->recovery_cp > oldsize) {
3808 mddev->recovery_cp = oldsize;
3809 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
3810 }
NeilBrown6508fdb2012-05-17 10:08:45 +10003811 calc_sectors(conf, sectors);
3812 mddev->dev_sectors = conf->dev_sectors;
NeilBrown006a09a2012-03-19 12:46:40 +11003813 mddev->resync_max_sectors = size;
3814 return 0;
3815}
3816
NeilBrownfd01b882011-10-11 16:47:53 +11003817static void *raid10_takeover_raid0(struct mddev *mddev)
Trela, Maciejdab8b292010-03-08 16:02:45 +11003818{
NeilBrown3cb03002011-10-11 16:45:26 +11003819 struct md_rdev *rdev;
NeilBrowne879a872011-10-11 16:49:02 +11003820 struct r10conf *conf;
Trela, Maciejdab8b292010-03-08 16:02:45 +11003821
3822 if (mddev->degraded > 0) {
NeilBrown128595e2010-05-03 14:47:14 +10003823 printk(KERN_ERR "md/raid10:%s: Error: degraded raid0!\n",
3824 mdname(mddev));
Trela, Maciejdab8b292010-03-08 16:02:45 +11003825 return ERR_PTR(-EINVAL);
3826 }
3827
Trela, Maciejdab8b292010-03-08 16:02:45 +11003828 /* Set new parameters */
3829 mddev->new_level = 10;
3830 /* new layout: far_copies = 1, near_copies = 2 */
3831 mddev->new_layout = (1<<8) + 2;
3832 mddev->new_chunk_sectors = mddev->chunk_sectors;
3833 mddev->delta_disks = mddev->raid_disks;
Trela, Maciejdab8b292010-03-08 16:02:45 +11003834 mddev->raid_disks *= 2;
3835 /* make sure it will be not marked as dirty */
3836 mddev->recovery_cp = MaxSector;
3837
3838 conf = setup_conf(mddev);
Krzysztof Wojcik02214dc2011-02-04 14:18:26 +01003839 if (!IS_ERR(conf)) {
NeilBrowndafb20f2012-03-19 12:46:39 +11003840 rdev_for_each(rdev, mddev)
NeilBrowne93f68a2010-06-15 09:36:03 +01003841 if (rdev->raid_disk >= 0)
3842 rdev->new_raid_disk = rdev->raid_disk * 2;
Krzysztof Wojcik02214dc2011-02-04 14:18:26 +01003843 conf->barrier = 1;
3844 }
3845
Trela, Maciejdab8b292010-03-08 16:02:45 +11003846 return conf;
3847}
3848
NeilBrownfd01b882011-10-11 16:47:53 +11003849static void *raid10_takeover(struct mddev *mddev)
Trela, Maciejdab8b292010-03-08 16:02:45 +11003850{
NeilBrowne373ab12011-10-11 16:48:59 +11003851 struct r0conf *raid0_conf;
Trela, Maciejdab8b292010-03-08 16:02:45 +11003852
3853 /* raid10 can take over:
3854 * raid0 - providing it has only two drives
3855 */
3856 if (mddev->level == 0) {
3857 /* for raid0 takeover only one zone is supported */
NeilBrowne373ab12011-10-11 16:48:59 +11003858 raid0_conf = mddev->private;
3859 if (raid0_conf->nr_strip_zones > 1) {
NeilBrown128595e2010-05-03 14:47:14 +10003860 printk(KERN_ERR "md/raid10:%s: cannot takeover raid 0"
3861 " with more than one zone.\n",
3862 mdname(mddev));
Trela, Maciejdab8b292010-03-08 16:02:45 +11003863 return ERR_PTR(-EINVAL);
3864 }
3865 return raid10_takeover_raid0(mddev);
3866 }
3867 return ERR_PTR(-EINVAL);
3868}
3869
NeilBrown3ea7daa2012-05-22 13:53:47 +10003870static int raid10_check_reshape(struct mddev *mddev)
3871{
3872 /* Called when there is a request to change
3873 * - layout (to ->new_layout)
3874 * - chunk size (to ->new_chunk_sectors)
3875 * - raid_disks (by delta_disks)
3876 * or when trying to restart a reshape that was ongoing.
3877 *
3878 * We need to validate the request and possibly allocate
3879 * space if that might be an issue later.
3880 *
3881 * Currently we reject any reshape of a 'far' mode array,
3882 * allow chunk size to change if new is generally acceptable,
3883 * allow raid_disks to increase, and allow
3884 * a switch between 'near' mode and 'offset' mode.
3885 */
3886 struct r10conf *conf = mddev->private;
3887 struct geom geo;
3888
3889 if (conf->geo.far_copies != 1 && !conf->geo.far_offset)
3890 return -EINVAL;
3891
3892 if (setup_geo(&geo, mddev, geo_start) != conf->copies)
3893 /* mustn't change number of copies */
3894 return -EINVAL;
3895 if (geo.far_copies > 1 && !geo.far_offset)
3896 /* Cannot switch to 'far' mode */
3897 return -EINVAL;
3898
3899 if (mddev->array_sectors & geo.chunk_mask)
3900 /* not factor of array size */
3901 return -EINVAL;
3902
NeilBrown3ea7daa2012-05-22 13:53:47 +10003903 if (!enough(conf, -1))
3904 return -EINVAL;
3905
3906 kfree(conf->mirrors_new);
3907 conf->mirrors_new = NULL;
3908 if (mddev->delta_disks > 0) {
3909 /* allocate new 'mirrors' list */
3910 conf->mirrors_new = kzalloc(
Jonathan Brassowdc280d982012-07-31 10:03:52 +10003911 sizeof(struct raid10_info)
NeilBrown3ea7daa2012-05-22 13:53:47 +10003912 *(mddev->raid_disks +
3913 mddev->delta_disks),
3914 GFP_KERNEL);
3915 if (!conf->mirrors_new)
3916 return -ENOMEM;
3917 }
3918 return 0;
3919}
3920
3921/*
3922 * Need to check if array has failed when deciding whether to:
3923 * - start an array
3924 * - remove non-faulty devices
3925 * - add a spare
3926 * - allow a reshape
3927 * This determination is simple when no reshape is happening.
3928 * However if there is a reshape, we need to carefully check
3929 * both the before and after sections.
3930 * This is because some failed devices may only affect one
3931 * of the two sections, and some non-in_sync devices may
3932 * be insync in the section most affected by failed devices.
3933 */
3934static int calc_degraded(struct r10conf *conf)
3935{
3936 int degraded, degraded2;
3937 int i;
3938
3939 rcu_read_lock();
3940 degraded = 0;
3941 /* 'prev' section first */
3942 for (i = 0; i < conf->prev.raid_disks; i++) {
3943 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
3944 if (!rdev || test_bit(Faulty, &rdev->flags))
3945 degraded++;
3946 else if (!test_bit(In_sync, &rdev->flags))
3947 /* When we can reduce the number of devices in
3948 * an array, this might not contribute to
3949 * 'degraded'. It does now.
3950 */
3951 degraded++;
3952 }
3953 rcu_read_unlock();
3954 if (conf->geo.raid_disks == conf->prev.raid_disks)
3955 return degraded;
3956 rcu_read_lock();
3957 degraded2 = 0;
3958 for (i = 0; i < conf->geo.raid_disks; i++) {
3959 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
3960 if (!rdev || test_bit(Faulty, &rdev->flags))
3961 degraded2++;
3962 else if (!test_bit(In_sync, &rdev->flags)) {
3963 /* If reshape is increasing the number of devices,
3964 * this section has already been recovered, so
3965 * it doesn't contribute to degraded.
3966 * else it does.
3967 */
3968 if (conf->geo.raid_disks <= conf->prev.raid_disks)
3969 degraded2++;
3970 }
3971 }
3972 rcu_read_unlock();
3973 if (degraded2 > degraded)
3974 return degraded2;
3975 return degraded;
3976}
3977
3978static int raid10_start_reshape(struct mddev *mddev)
3979{
3980 /* A 'reshape' has been requested. This commits
3981 * the various 'new' fields and sets MD_RECOVER_RESHAPE
3982 * This also checks if there are enough spares and adds them
3983 * to the array.
3984 * We currently require enough spares to make the final
3985 * array non-degraded. We also require that the difference
3986 * between old and new data_offset - on each device - is
3987 * enough that we never risk over-writing.
3988 */
3989
3990 unsigned long before_length, after_length;
3991 sector_t min_offset_diff = 0;
3992 int first = 1;
3993 struct geom new;
3994 struct r10conf *conf = mddev->private;
3995 struct md_rdev *rdev;
3996 int spares = 0;
NeilBrownbb63a702012-05-22 13:55:28 +10003997 int ret;
NeilBrown3ea7daa2012-05-22 13:53:47 +10003998
3999 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
4000 return -EBUSY;
4001
4002 if (setup_geo(&new, mddev, geo_start) != conf->copies)
4003 return -EINVAL;
4004
4005 before_length = ((1 << conf->prev.chunk_shift) *
4006 conf->prev.far_copies);
4007 after_length = ((1 << conf->geo.chunk_shift) *
4008 conf->geo.far_copies);
4009
4010 rdev_for_each(rdev, mddev) {
4011 if (!test_bit(In_sync, &rdev->flags)
4012 && !test_bit(Faulty, &rdev->flags))
4013 spares++;
4014 if (rdev->raid_disk >= 0) {
4015 long long diff = (rdev->new_data_offset
4016 - rdev->data_offset);
4017 if (!mddev->reshape_backwards)
4018 diff = -diff;
4019 if (diff < 0)
4020 diff = 0;
4021 if (first || diff < min_offset_diff)
4022 min_offset_diff = diff;
4023 }
4024 }
4025
4026 if (max(before_length, after_length) > min_offset_diff)
4027 return -EINVAL;
4028
4029 if (spares < mddev->delta_disks)
4030 return -EINVAL;
4031
4032 conf->offset_diff = min_offset_diff;
4033 spin_lock_irq(&conf->device_lock);
4034 if (conf->mirrors_new) {
4035 memcpy(conf->mirrors_new, conf->mirrors,
Jonathan Brassowdc280d982012-07-31 10:03:52 +10004036 sizeof(struct raid10_info)*conf->prev.raid_disks);
NeilBrown3ea7daa2012-05-22 13:53:47 +10004037 smp_mb();
4038 kfree(conf->mirrors_old); /* FIXME and elsewhere */
4039 conf->mirrors_old = conf->mirrors;
4040 conf->mirrors = conf->mirrors_new;
4041 conf->mirrors_new = NULL;
4042 }
4043 setup_geo(&conf->geo, mddev, geo_start);
4044 smp_mb();
4045 if (mddev->reshape_backwards) {
4046 sector_t size = raid10_size(mddev, 0, 0);
4047 if (size < mddev->array_sectors) {
4048 spin_unlock_irq(&conf->device_lock);
4049 printk(KERN_ERR "md/raid10:%s: array size must be reduce before number of disks\n",
4050 mdname(mddev));
4051 return -EINVAL;
4052 }
4053 mddev->resync_max_sectors = size;
4054 conf->reshape_progress = size;
4055 } else
4056 conf->reshape_progress = 0;
4057 spin_unlock_irq(&conf->device_lock);
4058
NeilBrownbb63a702012-05-22 13:55:28 +10004059 if (mddev->delta_disks && mddev->bitmap) {
4060 ret = bitmap_resize(mddev->bitmap,
4061 raid10_size(mddev, 0,
4062 conf->geo.raid_disks),
4063 0, 0);
4064 if (ret)
4065 goto abort;
4066 }
NeilBrown3ea7daa2012-05-22 13:53:47 +10004067 if (mddev->delta_disks > 0) {
4068 rdev_for_each(rdev, mddev)
4069 if (rdev->raid_disk < 0 &&
4070 !test_bit(Faulty, &rdev->flags)) {
4071 if (raid10_add_disk(mddev, rdev) == 0) {
4072 if (rdev->raid_disk >=
4073 conf->prev.raid_disks)
4074 set_bit(In_sync, &rdev->flags);
4075 else
4076 rdev->recovery_offset = 0;
4077
4078 if (sysfs_link_rdev(mddev, rdev))
4079 /* Failure here is OK */;
4080 }
4081 } else if (rdev->raid_disk >= conf->prev.raid_disks
4082 && !test_bit(Faulty, &rdev->flags)) {
4083 /* This is a spare that was manually added */
4084 set_bit(In_sync, &rdev->flags);
4085 }
4086 }
4087 /* When a reshape changes the number of devices,
4088 * ->degraded is measured against the larger of the
4089 * pre and post numbers.
4090 */
4091 spin_lock_irq(&conf->device_lock);
4092 mddev->degraded = calc_degraded(conf);
4093 spin_unlock_irq(&conf->device_lock);
4094 mddev->raid_disks = conf->geo.raid_disks;
4095 mddev->reshape_position = conf->reshape_progress;
4096 set_bit(MD_CHANGE_DEVS, &mddev->flags);
4097
4098 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
4099 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
4100 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
4101 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
4102
4103 mddev->sync_thread = md_register_thread(md_do_sync, mddev,
4104 "reshape");
4105 if (!mddev->sync_thread) {
NeilBrownbb63a702012-05-22 13:55:28 +10004106 ret = -EAGAIN;
4107 goto abort;
NeilBrown3ea7daa2012-05-22 13:53:47 +10004108 }
4109 conf->reshape_checkpoint = jiffies;
4110 md_wakeup_thread(mddev->sync_thread);
4111 md_new_event(mddev);
4112 return 0;
NeilBrownbb63a702012-05-22 13:55:28 +10004113
4114abort:
4115 mddev->recovery = 0;
4116 spin_lock_irq(&conf->device_lock);
4117 conf->geo = conf->prev;
4118 mddev->raid_disks = conf->geo.raid_disks;
4119 rdev_for_each(rdev, mddev)
4120 rdev->new_data_offset = rdev->data_offset;
4121 smp_wmb();
4122 conf->reshape_progress = MaxSector;
4123 mddev->reshape_position = MaxSector;
4124 spin_unlock_irq(&conf->device_lock);
4125 return ret;
NeilBrown3ea7daa2012-05-22 13:53:47 +10004126}
4127
4128/* Calculate the last device-address that could contain
4129 * any block from the chunk that includes the array-address 's'
4130 * and report the next address.
4131 * i.e. the address returned will be chunk-aligned and after
4132 * any data that is in the chunk containing 's'.
4133 */
4134static sector_t last_dev_address(sector_t s, struct geom *geo)
4135{
4136 s = (s | geo->chunk_mask) + 1;
4137 s >>= geo->chunk_shift;
4138 s *= geo->near_copies;
4139 s = DIV_ROUND_UP_SECTOR_T(s, geo->raid_disks);
4140 s *= geo->far_copies;
4141 s <<= geo->chunk_shift;
4142 return s;
4143}
4144
4145/* Calculate the first device-address that could contain
4146 * any block from the chunk that includes the array-address 's'.
4147 * This too will be the start of a chunk
4148 */
4149static sector_t first_dev_address(sector_t s, struct geom *geo)
4150{
4151 s >>= geo->chunk_shift;
4152 s *= geo->near_copies;
4153 sector_div(s, geo->raid_disks);
4154 s *= geo->far_copies;
4155 s <<= geo->chunk_shift;
4156 return s;
4157}
4158
4159static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
4160 int *skipped)
4161{
4162 /* We simply copy at most one chunk (smallest of old and new)
4163 * at a time, possibly less if that exceeds RESYNC_PAGES,
4164 * or we hit a bad block or something.
4165 * This might mean we pause for normal IO in the middle of
4166 * a chunk, but that is not a problem was mddev->reshape_position
4167 * can record any location.
4168 *
4169 * If we will want to write to a location that isn't
4170 * yet recorded as 'safe' (i.e. in metadata on disk) then
4171 * we need to flush all reshape requests and update the metadata.
4172 *
4173 * When reshaping forwards (e.g. to more devices), we interpret
4174 * 'safe' as the earliest block which might not have been copied
4175 * down yet. We divide this by previous stripe size and multiply
4176 * by previous stripe length to get lowest device offset that we
4177 * cannot write to yet.
4178 * We interpret 'sector_nr' as an address that we want to write to.
4179 * From this we use last_device_address() to find where we might
4180 * write to, and first_device_address on the 'safe' position.
4181 * If this 'next' write position is after the 'safe' position,
4182 * we must update the metadata to increase the 'safe' position.
4183 *
4184 * When reshaping backwards, we round in the opposite direction
4185 * and perform the reverse test: next write position must not be
4186 * less than current safe position.
4187 *
4188 * In all this the minimum difference in data offsets
4189 * (conf->offset_diff - always positive) allows a bit of slack,
4190 * so next can be after 'safe', but not by more than offset_disk
4191 *
4192 * We need to prepare all the bios here before we start any IO
4193 * to ensure the size we choose is acceptable to all devices.
4194 * The means one for each copy for write-out and an extra one for
4195 * read-in.
4196 * We store the read-in bio in ->master_bio and the others in
4197 * ->devs[x].bio and ->devs[x].repl_bio.
4198 */
4199 struct r10conf *conf = mddev->private;
4200 struct r10bio *r10_bio;
4201 sector_t next, safe, last;
4202 int max_sectors;
4203 int nr_sectors;
4204 int s;
4205 struct md_rdev *rdev;
4206 int need_flush = 0;
4207 struct bio *blist;
4208 struct bio *bio, *read_bio;
4209 int sectors_done = 0;
4210
4211 if (sector_nr == 0) {
4212 /* If restarting in the middle, skip the initial sectors */
4213 if (mddev->reshape_backwards &&
4214 conf->reshape_progress < raid10_size(mddev, 0, 0)) {
4215 sector_nr = (raid10_size(mddev, 0, 0)
4216 - conf->reshape_progress);
4217 } else if (!mddev->reshape_backwards &&
4218 conf->reshape_progress > 0)
4219 sector_nr = conf->reshape_progress;
4220 if (sector_nr) {
4221 mddev->curr_resync_completed = sector_nr;
4222 sysfs_notify(&mddev->kobj, NULL, "sync_completed");
4223 *skipped = 1;
4224 return sector_nr;
4225 }
4226 }
4227
4228 /* We don't use sector_nr to track where we are up to
4229 * as that doesn't work well for ->reshape_backwards.
4230 * So just use ->reshape_progress.
4231 */
4232 if (mddev->reshape_backwards) {
4233 /* 'next' is the earliest device address that we might
4234 * write to for this chunk in the new layout
4235 */
4236 next = first_dev_address(conf->reshape_progress - 1,
4237 &conf->geo);
4238
4239 /* 'safe' is the last device address that we might read from
4240 * in the old layout after a restart
4241 */
4242 safe = last_dev_address(conf->reshape_safe - 1,
4243 &conf->prev);
4244
4245 if (next + conf->offset_diff < safe)
4246 need_flush = 1;
4247
4248 last = conf->reshape_progress - 1;
4249 sector_nr = last & ~(sector_t)(conf->geo.chunk_mask
4250 & conf->prev.chunk_mask);
4251 if (sector_nr + RESYNC_BLOCK_SIZE/512 < last)
4252 sector_nr = last + 1 - RESYNC_BLOCK_SIZE/512;
4253 } else {
4254 /* 'next' is after the last device address that we
4255 * might write to for this chunk in the new layout
4256 */
4257 next = last_dev_address(conf->reshape_progress, &conf->geo);
4258
4259 /* 'safe' is the earliest device address that we might
4260 * read from in the old layout after a restart
4261 */
4262 safe = first_dev_address(conf->reshape_safe, &conf->prev);
4263
4264 /* Need to update metadata if 'next' might be beyond 'safe'
4265 * as that would possibly corrupt data
4266 */
4267 if (next > safe + conf->offset_diff)
4268 need_flush = 1;
4269
4270 sector_nr = conf->reshape_progress;
4271 last = sector_nr | (conf->geo.chunk_mask
4272 & conf->prev.chunk_mask);
4273
4274 if (sector_nr + RESYNC_BLOCK_SIZE/512 <= last)
4275 last = sector_nr + RESYNC_BLOCK_SIZE/512 - 1;
4276 }
4277
4278 if (need_flush ||
4279 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
4280 /* Need to update reshape_position in metadata */
4281 wait_barrier(conf);
4282 mddev->reshape_position = conf->reshape_progress;
4283 if (mddev->reshape_backwards)
4284 mddev->curr_resync_completed = raid10_size(mddev, 0, 0)
4285 - conf->reshape_progress;
4286 else
4287 mddev->curr_resync_completed = conf->reshape_progress;
4288 conf->reshape_checkpoint = jiffies;
4289 set_bit(MD_CHANGE_DEVS, &mddev->flags);
4290 md_wakeup_thread(mddev->thread);
4291 wait_event(mddev->sb_wait, mddev->flags == 0 ||
4292 kthread_should_stop());
4293 conf->reshape_safe = mddev->reshape_position;
4294 allow_barrier(conf);
4295 }
4296
4297read_more:
4298 /* Now schedule reads for blocks from sector_nr to last */
4299 r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
4300 raise_barrier(conf, sectors_done != 0);
4301 atomic_set(&r10_bio->remaining, 0);
4302 r10_bio->mddev = mddev;
4303 r10_bio->sector = sector_nr;
4304 set_bit(R10BIO_IsReshape, &r10_bio->state);
4305 r10_bio->sectors = last - sector_nr + 1;
4306 rdev = read_balance(conf, r10_bio, &max_sectors);
4307 BUG_ON(!test_bit(R10BIO_Previous, &r10_bio->state));
4308
4309 if (!rdev) {
4310 /* Cannot read from here, so need to record bad blocks
4311 * on all the target devices.
4312 */
4313 // FIXME
4314 set_bit(MD_RECOVERY_INTR, &mddev->recovery);
4315 return sectors_done;
4316 }
4317
4318 read_bio = bio_alloc_mddev(GFP_KERNEL, RESYNC_PAGES, mddev);
4319
4320 read_bio->bi_bdev = rdev->bdev;
4321 read_bio->bi_sector = (r10_bio->devs[r10_bio->read_slot].addr
4322 + rdev->data_offset);
4323 read_bio->bi_private = r10_bio;
4324 read_bio->bi_end_io = end_sync_read;
4325 read_bio->bi_rw = READ;
4326 read_bio->bi_flags &= ~(BIO_POOL_MASK - 1);
4327 read_bio->bi_flags |= 1 << BIO_UPTODATE;
4328 read_bio->bi_vcnt = 0;
4329 read_bio->bi_idx = 0;
4330 read_bio->bi_size = 0;
4331 r10_bio->master_bio = read_bio;
4332 r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum;
4333
4334 /* Now find the locations in the new layout */
4335 __raid10_find_phys(&conf->geo, r10_bio);
4336
4337 blist = read_bio;
4338 read_bio->bi_next = NULL;
4339
4340 for (s = 0; s < conf->copies*2; s++) {
4341 struct bio *b;
4342 int d = r10_bio->devs[s/2].devnum;
4343 struct md_rdev *rdev2;
4344 if (s&1) {
4345 rdev2 = conf->mirrors[d].replacement;
4346 b = r10_bio->devs[s/2].repl_bio;
4347 } else {
4348 rdev2 = conf->mirrors[d].rdev;
4349 b = r10_bio->devs[s/2].bio;
4350 }
4351 if (!rdev2 || test_bit(Faulty, &rdev2->flags))
4352 continue;
4353 b->bi_bdev = rdev2->bdev;
4354 b->bi_sector = r10_bio->devs[s/2].addr + rdev2->new_data_offset;
4355 b->bi_private = r10_bio;
4356 b->bi_end_io = end_reshape_write;
4357 b->bi_rw = WRITE;
4358 b->bi_flags &= ~(BIO_POOL_MASK - 1);
4359 b->bi_flags |= 1 << BIO_UPTODATE;
4360 b->bi_next = blist;
4361 b->bi_vcnt = 0;
4362 b->bi_idx = 0;
4363 b->bi_size = 0;
4364 blist = b;
4365 }
4366
4367 /* Now add as many pages as possible to all of these bios. */
4368
4369 nr_sectors = 0;
4370 for (s = 0 ; s < max_sectors; s += PAGE_SIZE >> 9) {
4371 struct page *page = r10_bio->devs[0].bio->bi_io_vec[s/(PAGE_SIZE>>9)].bv_page;
4372 int len = (max_sectors - s) << 9;
4373 if (len > PAGE_SIZE)
4374 len = PAGE_SIZE;
4375 for (bio = blist; bio ; bio = bio->bi_next) {
4376 struct bio *bio2;
4377 if (bio_add_page(bio, page, len, 0))
4378 continue;
4379
4380 /* Didn't fit, must stop */
4381 for (bio2 = blist;
4382 bio2 && bio2 != bio;
4383 bio2 = bio2->bi_next) {
4384 /* Remove last page from this bio */
4385 bio2->bi_vcnt--;
4386 bio2->bi_size -= len;
4387 bio2->bi_flags &= ~(1<<BIO_SEG_VALID);
4388 }
4389 goto bio_full;
4390 }
4391 sector_nr += len >> 9;
4392 nr_sectors += len >> 9;
4393 }
4394bio_full:
4395 r10_bio->sectors = nr_sectors;
4396
4397 /* Now submit the read */
4398 md_sync_acct(read_bio->bi_bdev, r10_bio->sectors);
4399 atomic_inc(&r10_bio->remaining);
4400 read_bio->bi_next = NULL;
4401 generic_make_request(read_bio);
4402 sector_nr += nr_sectors;
4403 sectors_done += nr_sectors;
4404 if (sector_nr <= last)
4405 goto read_more;
4406
4407 /* Now that we have done the whole section we can
4408 * update reshape_progress
4409 */
4410 if (mddev->reshape_backwards)
4411 conf->reshape_progress -= sectors_done;
4412 else
4413 conf->reshape_progress += sectors_done;
4414
4415 return sectors_done;
4416}
4417
4418static void end_reshape_request(struct r10bio *r10_bio);
4419static int handle_reshape_read_error(struct mddev *mddev,
4420 struct r10bio *r10_bio);
4421static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio)
4422{
4423 /* Reshape read completed. Hopefully we have a block
4424 * to write out.
4425 * If we got a read error then we do sync 1-page reads from
4426 * elsewhere until we find the data - or give up.
4427 */
4428 struct r10conf *conf = mddev->private;
4429 int s;
4430
4431 if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
4432 if (handle_reshape_read_error(mddev, r10_bio) < 0) {
4433 /* Reshape has been aborted */
4434 md_done_sync(mddev, r10_bio->sectors, 0);
4435 return;
4436 }
4437
4438 /* We definitely have the data in the pages, schedule the
4439 * writes.
4440 */
4441 atomic_set(&r10_bio->remaining, 1);
4442 for (s = 0; s < conf->copies*2; s++) {
4443 struct bio *b;
4444 int d = r10_bio->devs[s/2].devnum;
4445 struct md_rdev *rdev;
4446 if (s&1) {
4447 rdev = conf->mirrors[d].replacement;
4448 b = r10_bio->devs[s/2].repl_bio;
4449 } else {
4450 rdev = conf->mirrors[d].rdev;
4451 b = r10_bio->devs[s/2].bio;
4452 }
4453 if (!rdev || test_bit(Faulty, &rdev->flags))
4454 continue;
4455 atomic_inc(&rdev->nr_pending);
4456 md_sync_acct(b->bi_bdev, r10_bio->sectors);
4457 atomic_inc(&r10_bio->remaining);
4458 b->bi_next = NULL;
4459 generic_make_request(b);
4460 }
4461 end_reshape_request(r10_bio);
4462}
4463
4464static void end_reshape(struct r10conf *conf)
4465{
4466 if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery))
4467 return;
4468
4469 spin_lock_irq(&conf->device_lock);
4470 conf->prev = conf->geo;
4471 md_finish_reshape(conf->mddev);
4472 smp_wmb();
4473 conf->reshape_progress = MaxSector;
4474 spin_unlock_irq(&conf->device_lock);
4475
4476 /* read-ahead size must cover two whole stripes, which is
4477 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
4478 */
4479 if (conf->mddev->queue) {
4480 int stripe = conf->geo.raid_disks *
4481 ((conf->mddev->chunk_sectors << 9) / PAGE_SIZE);
4482 stripe /= conf->geo.near_copies;
4483 if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
4484 conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
4485 }
4486 conf->fullsync = 0;
4487}
4488
4489
4490static int handle_reshape_read_error(struct mddev *mddev,
4491 struct r10bio *r10_bio)
4492{
4493 /* Use sync reads to get the blocks from somewhere else */
4494 int sectors = r10_bio->sectors;
NeilBrown3ea7daa2012-05-22 13:53:47 +10004495 struct r10conf *conf = mddev->private;
NeilBrowne0ee7782012-08-18 09:51:42 +10004496 struct {
4497 struct r10bio r10_bio;
4498 struct r10dev devs[conf->copies];
4499 } on_stack;
4500 struct r10bio *r10b = &on_stack.r10_bio;
NeilBrown3ea7daa2012-05-22 13:53:47 +10004501 int slot = 0;
4502 int idx = 0;
4503 struct bio_vec *bvec = r10_bio->master_bio->bi_io_vec;
4504
NeilBrowne0ee7782012-08-18 09:51:42 +10004505 r10b->sector = r10_bio->sector;
4506 __raid10_find_phys(&conf->prev, r10b);
NeilBrown3ea7daa2012-05-22 13:53:47 +10004507
4508 while (sectors) {
4509 int s = sectors;
4510 int success = 0;
4511 int first_slot = slot;
4512
4513 if (s > (PAGE_SIZE >> 9))
4514 s = PAGE_SIZE >> 9;
4515
4516 while (!success) {
NeilBrowne0ee7782012-08-18 09:51:42 +10004517 int d = r10b->devs[slot].devnum;
NeilBrown3ea7daa2012-05-22 13:53:47 +10004518 struct md_rdev *rdev = conf->mirrors[d].rdev;
4519 sector_t addr;
4520 if (rdev == NULL ||
4521 test_bit(Faulty, &rdev->flags) ||
4522 !test_bit(In_sync, &rdev->flags))
4523 goto failed;
4524
NeilBrowne0ee7782012-08-18 09:51:42 +10004525 addr = r10b->devs[slot].addr + idx * PAGE_SIZE;
NeilBrown3ea7daa2012-05-22 13:53:47 +10004526 success = sync_page_io(rdev,
4527 addr,
4528 s << 9,
4529 bvec[idx].bv_page,
4530 READ, false);
4531 if (success)
4532 break;
4533 failed:
4534 slot++;
4535 if (slot >= conf->copies)
4536 slot = 0;
4537 if (slot == first_slot)
4538 break;
4539 }
4540 if (!success) {
4541 /* couldn't read this block, must give up */
4542 set_bit(MD_RECOVERY_INTR,
4543 &mddev->recovery);
4544 return -EIO;
4545 }
4546 sectors -= s;
4547 idx++;
4548 }
4549 return 0;
4550}
4551
4552static void end_reshape_write(struct bio *bio, int error)
4553{
4554 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
4555 struct r10bio *r10_bio = bio->bi_private;
4556 struct mddev *mddev = r10_bio->mddev;
4557 struct r10conf *conf = mddev->private;
4558 int d;
4559 int slot;
4560 int repl;
4561 struct md_rdev *rdev = NULL;
4562
4563 d = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
4564 if (repl)
4565 rdev = conf->mirrors[d].replacement;
4566 if (!rdev) {
4567 smp_mb();
4568 rdev = conf->mirrors[d].rdev;
4569 }
4570
4571 if (!uptodate) {
4572 /* FIXME should record badblock */
4573 md_error(mddev, rdev);
4574 }
4575
4576 rdev_dec_pending(rdev, mddev);
4577 end_reshape_request(r10_bio);
4578}
4579
4580static void end_reshape_request(struct r10bio *r10_bio)
4581{
4582 if (!atomic_dec_and_test(&r10_bio->remaining))
4583 return;
4584 md_done_sync(r10_bio->mddev, r10_bio->sectors, 1);
4585 bio_put(r10_bio->master_bio);
4586 put_buf(r10_bio);
4587}
4588
4589static void raid10_finish_reshape(struct mddev *mddev)
4590{
4591 struct r10conf *conf = mddev->private;
4592
4593 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
4594 return;
4595
4596 if (mddev->delta_disks > 0) {
4597 sector_t size = raid10_size(mddev, 0, 0);
4598 md_set_array_sectors(mddev, size);
4599 if (mddev->recovery_cp > mddev->resync_max_sectors) {
4600 mddev->recovery_cp = mddev->resync_max_sectors;
4601 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
4602 }
4603 mddev->resync_max_sectors = size;
4604 set_capacity(mddev->gendisk, mddev->array_sectors);
4605 revalidate_disk(mddev->gendisk);
NeilBrown63aced62012-05-22 13:55:33 +10004606 } else {
4607 int d;
4608 for (d = conf->geo.raid_disks ;
4609 d < conf->geo.raid_disks - mddev->delta_disks;
4610 d++) {
4611 struct md_rdev *rdev = conf->mirrors[d].rdev;
4612 if (rdev)
4613 clear_bit(In_sync, &rdev->flags);
4614 rdev = conf->mirrors[d].replacement;
4615 if (rdev)
4616 clear_bit(In_sync, &rdev->flags);
4617 }
NeilBrown3ea7daa2012-05-22 13:53:47 +10004618 }
4619 mddev->layout = mddev->new_layout;
4620 mddev->chunk_sectors = 1 << conf->geo.chunk_shift;
4621 mddev->reshape_position = MaxSector;
4622 mddev->delta_disks = 0;
4623 mddev->reshape_backwards = 0;
4624}
4625
NeilBrown84fc4b52011-10-11 16:49:58 +11004626static struct md_personality raid10_personality =
Linus Torvalds1da177e2005-04-16 15:20:36 -07004627{
4628 .name = "raid10",
NeilBrown2604b702006-01-06 00:20:36 -08004629 .level = 10,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004630 .owner = THIS_MODULE,
4631 .make_request = make_request,
4632 .run = run,
4633 .stop = stop,
4634 .status = status,
4635 .error_handler = error,
4636 .hot_add_disk = raid10_add_disk,
4637 .hot_remove_disk= raid10_remove_disk,
4638 .spare_active = raid10_spare_active,
4639 .sync_request = sync_request,
NeilBrown6cce3b232006-01-06 00:20:16 -08004640 .quiesce = raid10_quiesce,
Dan Williams80c3a6c2009-03-17 18:10:40 -07004641 .size = raid10_size,
NeilBrown006a09a2012-03-19 12:46:40 +11004642 .resize = raid10_resize,
Trela, Maciejdab8b292010-03-08 16:02:45 +11004643 .takeover = raid10_takeover,
NeilBrown3ea7daa2012-05-22 13:53:47 +10004644 .check_reshape = raid10_check_reshape,
4645 .start_reshape = raid10_start_reshape,
4646 .finish_reshape = raid10_finish_reshape,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004647};
4648
4649static int __init raid_init(void)
4650{
NeilBrown2604b702006-01-06 00:20:36 -08004651 return register_md_personality(&raid10_personality);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004652}
4653
4654static void raid_exit(void)
4655{
NeilBrown2604b702006-01-06 00:20:36 -08004656 unregister_md_personality(&raid10_personality);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004657}
4658
4659module_init(raid_init);
4660module_exit(raid_exit);
4661MODULE_LICENSE("GPL");
NeilBrown0efb9e62009-12-14 12:49:58 +11004662MODULE_DESCRIPTION("RAID10 (striped mirror) personality for MD");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004663MODULE_ALIAS("md-personality-9"); /* RAID10 */
NeilBrownd9d166c2006-01-06 00:20:51 -08004664MODULE_ALIAS("md-raid10");
NeilBrown2604b702006-01-06 00:20:36 -08004665MODULE_ALIAS("md-level-10");
NeilBrown34db0cd2011-10-11 16:50:01 +11004666
4667module_param(max_queued_requests, int, S_IRUGO|S_IWUSR);