blob: 63c86c48014b14b9fb245fb0084beae4aa6df9fc [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Nathan Scott7b718762005-11-02 14:58:39 +11002 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
3 * All Rights Reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
Nathan Scott7b718762005-11-02 14:58:39 +11005 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License as
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 * published by the Free Software Foundation.
8 *
Nathan Scott7b718762005-11-02 14:58:39 +11009 * This program is distributed in the hope that it would be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 *
Nathan Scott7b718762005-11-02 14:58:39 +110014 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Linus Torvalds1da177e2005-04-16 15:20:36 -070017 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include "xfs.h"
Nathan Scotta844f452005-11-02 14:38:42 +110019#include "xfs_fs.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include "xfs_types.h"
Nathan Scotta844f452005-11-02 14:38:42 +110021#include "xfs_bit.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070022#include "xfs_log.h"
23#include "xfs_trans.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include "xfs_sb.h"
David Chinnerda353b02007-08-28 14:00:13 +100025#include "xfs_ag.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070026#include "xfs_mount.h"
Nathan Scotta844f452005-11-02 14:38:42 +110027#include "xfs_buf_item.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070028#include "xfs_trans_priv.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include "xfs_error.h"
Christoph Hellwig0b1b2132009-12-14 23:14:59 +000030#include "xfs_trace.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070031
32
33kmem_zone_t *xfs_buf_item_zone;
34
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +100035static inline struct xfs_buf_log_item *BUF_ITEM(struct xfs_log_item *lip)
36{
37 return container_of(lip, struct xfs_buf_log_item, bli_item);
38}
39
Dave Chinnerc90821a2010-12-03 17:00:52 +110040STATIC void xfs_buf_do_callbacks(struct xfs_buf *bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -070041
42/*
43 * This returns the number of log iovecs needed to log the
44 * given buf log item.
45 *
46 * It calculates this as 1 iovec for the buf log format structure
47 * and 1 for each stretch of non-contiguous chunks to be logged.
48 * Contiguous chunks are logged in a single iovec.
49 *
50 * If the XFS_BLI_STALE flag has been set, then log nothing.
51 */
Christoph Hellwigba0f32d2005-06-21 15:36:52 +100052STATIC uint
Dave Chinner372cc85e2012-06-22 18:50:12 +100053xfs_buf_item_size_segment(
54 struct xfs_buf_log_item *bip,
55 struct xfs_buf_log_format *blfp)
Linus Torvalds1da177e2005-04-16 15:20:36 -070056{
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +100057 struct xfs_buf *bp = bip->bli_buf;
58 uint nvecs;
59 int next_bit;
60 int last_bit;
Linus Torvalds1da177e2005-04-16 15:20:36 -070061
Dave Chinner372cc85e2012-06-22 18:50:12 +100062 last_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size, 0);
63 if (last_bit == -1)
64 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070065
Dave Chinner372cc85e2012-06-22 18:50:12 +100066 /*
67 * initial count for a dirty buffer is 2 vectors - the format structure
68 * and the first dirty region.
69 */
70 nvecs = 2;
71
Linus Torvalds1da177e2005-04-16 15:20:36 -070072 while (last_bit != -1) {
73 /*
74 * This takes the bit number to start looking from and
75 * returns the next set bit from there. It returns -1
76 * if there are no more bits set or the start bit is
77 * beyond the end of the bitmap.
78 */
Dave Chinner372cc85e2012-06-22 18:50:12 +100079 next_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size,
80 last_bit + 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -070081 /*
82 * If we run out of bits, leave the loop,
83 * else if we find a new set of bits bump the number of vecs,
84 * else keep scanning the current set of bits.
85 */
86 if (next_bit == -1) {
Dave Chinner372cc85e2012-06-22 18:50:12 +100087 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -070088 } else if (next_bit != last_bit + 1) {
89 last_bit = next_bit;
90 nvecs++;
Dave Chinnerc1155412010-05-07 11:05:19 +100091 } else if (xfs_buf_offset(bp, next_bit * XFS_BLF_CHUNK) !=
92 (xfs_buf_offset(bp, last_bit * XFS_BLF_CHUNK) +
93 XFS_BLF_CHUNK)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -070094 last_bit = next_bit;
95 nvecs++;
96 } else {
97 last_bit++;
98 }
99 }
100
Dave Chinner372cc85e2012-06-22 18:50:12 +1000101 return nvecs;
102}
103
104/*
105 * This returns the number of log iovecs needed to log the given buf log item.
106 *
107 * It calculates this as 1 iovec for the buf log format structure and 1 for each
108 * stretch of non-contiguous chunks to be logged. Contiguous chunks are logged
109 * in a single iovec.
110 *
111 * Discontiguous buffers need a format structure per region that that is being
112 * logged. This makes the changes in the buffer appear to log recovery as though
113 * they came from separate buffers, just like would occur if multiple buffers
114 * were used instead of a single discontiguous buffer. This enables
115 * discontiguous buffers to be in-memory constructs, completely transparent to
116 * what ends up on disk.
117 *
118 * If the XFS_BLI_STALE flag has been set, then log nothing but the buf log
119 * format structures.
120 */
121STATIC uint
122xfs_buf_item_size(
123 struct xfs_log_item *lip)
124{
125 struct xfs_buf_log_item *bip = BUF_ITEM(lip);
126 uint nvecs;
127 int i;
128
129 ASSERT(atomic_read(&bip->bli_refcount) > 0);
130 if (bip->bli_flags & XFS_BLI_STALE) {
131 /*
132 * The buffer is stale, so all we need to log
133 * is the buf log format structure with the
134 * cancel flag in it.
135 */
136 trace_xfs_buf_item_size_stale(bip);
Mark Tinguelyb9438172012-12-04 17:18:03 -0600137 ASSERT(bip->__bli_format.blf_flags & XFS_BLF_CANCEL);
Dave Chinner372cc85e2012-06-22 18:50:12 +1000138 return bip->bli_format_count;
139 }
140
141 ASSERT(bip->bli_flags & XFS_BLI_LOGGED);
142
143 /*
144 * the vector count is based on the number of buffer vectors we have
145 * dirty bits in. This will only be greater than one when we have a
146 * compound buffer with more than one segment dirty. Hence for compound
147 * buffers we need to track which segment the dirty bits correspond to,
148 * and when we move from one segment to the next increment the vector
149 * count for the extra buf log format structure that will need to be
150 * written.
151 */
152 nvecs = 0;
153 for (i = 0; i < bip->bli_format_count; i++) {
154 nvecs += xfs_buf_item_size_segment(bip, &bip->bli_formats[i]);
155 }
156
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000157 trace_xfs_buf_item_size(bip);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158 return nvecs;
159}
160
Dave Chinner372cc85e2012-06-22 18:50:12 +1000161static struct xfs_log_iovec *
162xfs_buf_item_format_segment(
163 struct xfs_buf_log_item *bip,
164 struct xfs_log_iovec *vecp,
165 uint offset,
166 struct xfs_buf_log_format *blfp)
167{
168 struct xfs_buf *bp = bip->bli_buf;
169 uint base_size;
170 uint nvecs;
171 int first_bit;
172 int last_bit;
173 int next_bit;
174 uint nbits;
175 uint buffer_offset;
176
177 /* copy the flags across from the base format item */
Mark Tinguelyb9438172012-12-04 17:18:03 -0600178 blfp->blf_flags = bip->__bli_format.blf_flags;
Dave Chinner372cc85e2012-06-22 18:50:12 +1000179
180 /*
181 * Base size is the actual size of the ondisk structure - it reflects
182 * the actual size of the dirty bitmap rather than the size of the in
183 * memory structure.
184 */
185 base_size = offsetof(struct xfs_buf_log_format, blf_data_map) +
186 (blfp->blf_map_size * sizeof(blfp->blf_data_map[0]));
Mark Tinguely820a5542012-12-04 17:18:04 -0600187
188 nvecs = 0;
189 first_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size, 0);
190 if (!(bip->bli_flags & XFS_BLI_STALE) && first_bit == -1) {
191 /*
192 * If the map is not be dirty in the transaction, mark
193 * the size as zero and do not advance the vector pointer.
194 */
195 goto out;
196 }
197
Dave Chinner372cc85e2012-06-22 18:50:12 +1000198 vecp->i_addr = blfp;
199 vecp->i_len = base_size;
200 vecp->i_type = XLOG_REG_TYPE_BFORMAT;
201 vecp++;
202 nvecs = 1;
203
204 if (bip->bli_flags & XFS_BLI_STALE) {
205 /*
206 * The buffer is stale, so all we need to log
207 * is the buf log format structure with the
208 * cancel flag in it.
209 */
210 trace_xfs_buf_item_format_stale(bip);
211 ASSERT(blfp->blf_flags & XFS_BLF_CANCEL);
Mark Tinguely820a5542012-12-04 17:18:04 -0600212 goto out;
Dave Chinner372cc85e2012-06-22 18:50:12 +1000213 }
214
215 /*
216 * Fill in an iovec for each set of contiguous chunks.
217 */
Mark Tinguely820a5542012-12-04 17:18:04 -0600218
Dave Chinner372cc85e2012-06-22 18:50:12 +1000219 last_bit = first_bit;
220 nbits = 1;
221 for (;;) {
222 /*
223 * This takes the bit number to start looking from and
224 * returns the next set bit from there. It returns -1
225 * if there are no more bits set or the start bit is
226 * beyond the end of the bitmap.
227 */
228 next_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size,
229 (uint)last_bit + 1);
230 /*
231 * If we run out of bits fill in the last iovec and get
232 * out of the loop.
233 * Else if we start a new set of bits then fill in the
234 * iovec for the series we were looking at and start
235 * counting the bits in the new one.
236 * Else we're still in the same set of bits so just
237 * keep counting and scanning.
238 */
239 if (next_bit == -1) {
240 buffer_offset = offset + first_bit * XFS_BLF_CHUNK;
241 vecp->i_addr = xfs_buf_offset(bp, buffer_offset);
242 vecp->i_len = nbits * XFS_BLF_CHUNK;
243 vecp->i_type = XLOG_REG_TYPE_BCHUNK;
244 nvecs++;
245 break;
246 } else if (next_bit != last_bit + 1) {
247 buffer_offset = offset + first_bit * XFS_BLF_CHUNK;
248 vecp->i_addr = xfs_buf_offset(bp, buffer_offset);
249 vecp->i_len = nbits * XFS_BLF_CHUNK;
250 vecp->i_type = XLOG_REG_TYPE_BCHUNK;
251 nvecs++;
252 vecp++;
253 first_bit = next_bit;
254 last_bit = next_bit;
255 nbits = 1;
256 } else if (xfs_buf_offset(bp, offset +
257 (next_bit << XFS_BLF_SHIFT)) !=
258 (xfs_buf_offset(bp, offset +
259 (last_bit << XFS_BLF_SHIFT)) +
260 XFS_BLF_CHUNK)) {
261 buffer_offset = offset + first_bit * XFS_BLF_CHUNK;
262 vecp->i_addr = xfs_buf_offset(bp, buffer_offset);
263 vecp->i_len = nbits * XFS_BLF_CHUNK;
264 vecp->i_type = XLOG_REG_TYPE_BCHUNK;
265/*
266 * You would think we need to bump the nvecs here too, but we do not
267 * this number is used by recovery, and it gets confused by the boundary
268 * split here
269 * nvecs++;
270 */
271 vecp++;
272 first_bit = next_bit;
273 last_bit = next_bit;
274 nbits = 1;
275 } else {
276 last_bit++;
277 nbits++;
278 }
279 }
Mark Tinguely820a5542012-12-04 17:18:04 -0600280out:
281 blfp->blf_size = nvecs;
Dave Chinner372cc85e2012-06-22 18:50:12 +1000282 return vecp;
283}
284
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285/*
286 * This is called to fill in the vector of log iovecs for the
287 * given log buf item. It fills the first entry with a buf log
288 * format structure, and the rest point to contiguous chunks
289 * within the buffer.
290 */
Christoph Hellwigba0f32d2005-06-21 15:36:52 +1000291STATIC void
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292xfs_buf_item_format(
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000293 struct xfs_log_item *lip,
294 struct xfs_log_iovec *vecp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295{
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000296 struct xfs_buf_log_item *bip = BUF_ITEM(lip);
Dave Chinner372cc85e2012-06-22 18:50:12 +1000297 struct xfs_buf *bp = bip->bli_buf;
298 uint offset = 0;
299 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300
301 ASSERT(atomic_read(&bip->bli_refcount) > 0);
302 ASSERT((bip->bli_flags & XFS_BLI_LOGGED) ||
303 (bip->bli_flags & XFS_BLI_STALE));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304
305 /*
Dave Chinnerccf7c232010-05-20 23:19:42 +1000306 * If it is an inode buffer, transfer the in-memory state to the
307 * format flags and clear the in-memory state. We do not transfer
308 * this state if the inode buffer allocation has not yet been committed
309 * to the log as setting the XFS_BLI_INODE_BUF flag will prevent
310 * correct replay of the inode allocation.
311 */
312 if (bip->bli_flags & XFS_BLI_INODE_BUF) {
313 if (!((bip->bli_flags & XFS_BLI_INODE_ALLOC_BUF) &&
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000314 xfs_log_item_in_current_chkpt(lip)))
Mark Tinguelyb9438172012-12-04 17:18:03 -0600315 bip->__bli_format.blf_flags |= XFS_BLF_INODE_BUF;
Dave Chinnerccf7c232010-05-20 23:19:42 +1000316 bip->bli_flags &= ~XFS_BLI_INODE_BUF;
317 }
318
Dave Chinner372cc85e2012-06-22 18:50:12 +1000319 for (i = 0; i < bip->bli_format_count; i++) {
320 vecp = xfs_buf_item_format_segment(bip, vecp, offset,
321 &bip->bli_formats[i]);
322 offset += bp->b_maps[i].bm_len;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323 }
324
325 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326 * Check to make sure everything is consistent.
327 */
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000328 trace_xfs_buf_item_format(bip);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700329}
330
331/*
Dave Chinner64fc35d2010-05-07 11:04:34 +1000332 * This is called to pin the buffer associated with the buf log item in memory
Christoph Hellwig4d16e922010-06-23 18:11:15 +1000333 * so it cannot be written out.
Dave Chinner64fc35d2010-05-07 11:04:34 +1000334 *
335 * We also always take a reference to the buffer log item here so that the bli
336 * is held while the item is pinned in memory. This means that we can
337 * unconditionally drop the reference count a transaction holds when the
338 * transaction is completed.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339 */
Christoph Hellwigba0f32d2005-06-21 15:36:52 +1000340STATIC void
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341xfs_buf_item_pin(
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000342 struct xfs_log_item *lip)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343{
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000344 struct xfs_buf_log_item *bip = BUF_ITEM(lip);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346 ASSERT(atomic_read(&bip->bli_refcount) > 0);
347 ASSERT((bip->bli_flags & XFS_BLI_LOGGED) ||
348 (bip->bli_flags & XFS_BLI_STALE));
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000349
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000350 trace_xfs_buf_item_pin(bip);
Christoph Hellwig4d16e922010-06-23 18:11:15 +1000351
352 atomic_inc(&bip->bli_refcount);
353 atomic_inc(&bip->bli_buf->b_pin_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700354}
355
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356/*
357 * This is called to unpin the buffer associated with the buf log
358 * item which was previously pinned with a call to xfs_buf_item_pin().
Linus Torvalds1da177e2005-04-16 15:20:36 -0700359 *
360 * Also drop the reference to the buf item for the current transaction.
361 * If the XFS_BLI_STALE flag is set and we are the last reference,
362 * then free up the buf log item and unlock the buffer.
Christoph Hellwig9412e312010-06-23 18:11:15 +1000363 *
364 * If the remove flag is set we are called from uncommit in the
365 * forced-shutdown path. If that is true and the reference count on
366 * the log item is going to drop to zero we need to free the item's
367 * descriptor in the transaction.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368 */
Christoph Hellwigba0f32d2005-06-21 15:36:52 +1000369STATIC void
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370xfs_buf_item_unpin(
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000371 struct xfs_log_item *lip,
Christoph Hellwig9412e312010-06-23 18:11:15 +1000372 int remove)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373{
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000374 struct xfs_buf_log_item *bip = BUF_ITEM(lip);
Christoph Hellwig9412e312010-06-23 18:11:15 +1000375 xfs_buf_t *bp = bip->bli_buf;
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000376 struct xfs_ail *ailp = lip->li_ailp;
Dave Chinner8e123852010-03-08 11:26:03 +1100377 int stale = bip->bli_flags & XFS_BLI_STALE;
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000378 int freed;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379
Christoph Hellwigadadbee2011-07-13 13:43:49 +0200380 ASSERT(bp->b_fspriv == bip);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381 ASSERT(atomic_read(&bip->bli_refcount) > 0);
Christoph Hellwig9412e312010-06-23 18:11:15 +1000382
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000383 trace_xfs_buf_item_unpin(bip);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384
385 freed = atomic_dec_and_test(&bip->bli_refcount);
Christoph Hellwig4d16e922010-06-23 18:11:15 +1000386
387 if (atomic_dec_and_test(&bp->b_pin_count))
388 wake_up_all(&bp->b_waiters);
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000389
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390 if (freed && stale) {
391 ASSERT(bip->bli_flags & XFS_BLI_STALE);
Christoph Hellwig0c842ad2011-07-08 14:36:19 +0200392 ASSERT(xfs_buf_islocked(bp));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393 ASSERT(XFS_BUF_ISSTALE(bp));
Mark Tinguelyb9438172012-12-04 17:18:03 -0600394 ASSERT(bip->__bli_format.blf_flags & XFS_BLF_CANCEL);
Christoph Hellwig9412e312010-06-23 18:11:15 +1000395
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000396 trace_xfs_buf_item_unpin_stale(bip);
397
Christoph Hellwig9412e312010-06-23 18:11:15 +1000398 if (remove) {
399 /*
Dave Chinnere34a3142011-01-27 12:13:35 +1100400 * If we are in a transaction context, we have to
401 * remove the log item from the transaction as we are
402 * about to release our reference to the buffer. If we
403 * don't, the unlock that occurs later in
404 * xfs_trans_uncommit() will try to reference the
Christoph Hellwig9412e312010-06-23 18:11:15 +1000405 * buffer which we no longer have a hold on.
406 */
Dave Chinnere34a3142011-01-27 12:13:35 +1100407 if (lip->li_desc)
408 xfs_trans_del_item(lip);
Christoph Hellwig9412e312010-06-23 18:11:15 +1000409
410 /*
411 * Since the transaction no longer refers to the buffer,
412 * the buffer should no longer refer to the transaction.
413 */
Christoph Hellwigbf9d9012011-07-13 13:43:49 +0200414 bp->b_transp = NULL;
Christoph Hellwig9412e312010-06-23 18:11:15 +1000415 }
416
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417 /*
418 * If we get called here because of an IO error, we may
David Chinner783a2f62008-10-30 17:39:58 +1100419 * or may not have the item on the AIL. xfs_trans_ail_delete()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420 * will take care of that situation.
David Chinner783a2f62008-10-30 17:39:58 +1100421 * xfs_trans_ail_delete() drops the AIL lock.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422 */
423 if (bip->bli_flags & XFS_BLI_STALE_INODE) {
Dave Chinnerc90821a2010-12-03 17:00:52 +1100424 xfs_buf_do_callbacks(bp);
Christoph Hellwigadadbee2011-07-13 13:43:49 +0200425 bp->b_fspriv = NULL;
Christoph Hellwigcb669ca2011-07-13 13:43:49 +0200426 bp->b_iodone = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427 } else {
David Chinner783a2f62008-10-30 17:39:58 +1100428 spin_lock(&ailp->xa_lock);
Dave Chinner04913fd2012-04-23 15:58:41 +1000429 xfs_trans_ail_delete(ailp, lip, SHUTDOWN_LOG_IO_ERROR);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430 xfs_buf_item_relse(bp);
Christoph Hellwigadadbee2011-07-13 13:43:49 +0200431 ASSERT(bp->b_fspriv == NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432 }
433 xfs_buf_relse(bp);
Christoph Hellwig960c60a2012-04-23 15:58:38 +1000434 } else if (freed && remove) {
Dave Chinner137fff02012-11-02 14:23:12 +1100435 /*
436 * There are currently two references to the buffer - the active
437 * LRU reference and the buf log item. What we are about to do
438 * here - simulate a failed IO completion - requires 3
439 * references.
440 *
441 * The LRU reference is removed by the xfs_buf_stale() call. The
442 * buf item reference is removed by the xfs_buf_iodone()
443 * callback that is run by xfs_buf_do_callbacks() during ioend
444 * processing (via the bp->b_iodone callback), and then finally
445 * the ioend processing will drop the IO reference if the buffer
446 * is marked XBF_ASYNC.
447 *
448 * Hence we need to take an additional reference here so that IO
449 * completion processing doesn't free the buffer prematurely.
450 */
Christoph Hellwig960c60a2012-04-23 15:58:38 +1000451 xfs_buf_lock(bp);
Dave Chinner137fff02012-11-02 14:23:12 +1100452 xfs_buf_hold(bp);
453 bp->b_flags |= XBF_ASYNC;
Christoph Hellwig960c60a2012-04-23 15:58:38 +1000454 xfs_buf_ioerror(bp, EIO);
455 XFS_BUF_UNDONE(bp);
456 xfs_buf_stale(bp);
457 xfs_buf_ioend(bp, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700458 }
459}
460
Christoph Hellwigba0f32d2005-06-21 15:36:52 +1000461STATIC uint
Christoph Hellwig43ff2122012-04-23 15:58:39 +1000462xfs_buf_item_push(
463 struct xfs_log_item *lip,
464 struct list_head *buffer_list)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465{
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000466 struct xfs_buf_log_item *bip = BUF_ITEM(lip);
467 struct xfs_buf *bp = bip->bli_buf;
Christoph Hellwig43ff2122012-04-23 15:58:39 +1000468 uint rval = XFS_ITEM_SUCCESS;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469
Chandra Seetharaman811e64c2011-07-22 23:40:27 +0000470 if (xfs_buf_ispinned(bp))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471 return XFS_ITEM_PINNED;
Christoph Hellwig0c842ad2011-07-08 14:36:19 +0200472 if (!xfs_buf_trylock(bp))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473 return XFS_ITEM_LOCKED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474
Linus Torvalds1da177e2005-04-16 15:20:36 -0700475 ASSERT(!(bip->bli_flags & XFS_BLI_STALE));
Christoph Hellwig43ff2122012-04-23 15:58:39 +1000476
477 trace_xfs_buf_item_push(bip);
478
479 if (!xfs_buf_delwri_queue(bp, buffer_list))
480 rval = XFS_ITEM_FLUSHING;
481 xfs_buf_unlock(bp);
482 return rval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483}
484
485/*
Dave Chinner64fc35d2010-05-07 11:04:34 +1000486 * Release the buffer associated with the buf log item. If there is no dirty
487 * logged data associated with the buffer recorded in the buf log item, then
488 * free the buf log item and remove the reference to it in the buffer.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700489 *
Dave Chinner64fc35d2010-05-07 11:04:34 +1000490 * This call ignores the recursion count. It is only called when the buffer
491 * should REALLY be unlocked, regardless of the recursion count.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492 *
Dave Chinner64fc35d2010-05-07 11:04:34 +1000493 * We unconditionally drop the transaction's reference to the log item. If the
494 * item was logged, then another reference was taken when it was pinned, so we
495 * can safely drop the transaction reference now. This also allows us to avoid
496 * potential races with the unpin code freeing the bli by not referencing the
497 * bli after we've dropped the reference count.
498 *
499 * If the XFS_BLI_HOLD flag is set in the buf log item, then free the log item
500 * if necessary but do not unlock the buffer. This is for support of
501 * xfs_trans_bhold(). Make sure the XFS_BLI_HOLD field is cleared if we don't
502 * free the item.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503 */
Christoph Hellwigba0f32d2005-06-21 15:36:52 +1000504STATIC void
Linus Torvalds1da177e2005-04-16 15:20:36 -0700505xfs_buf_item_unlock(
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000506 struct xfs_log_item *lip)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700507{
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000508 struct xfs_buf_log_item *bip = BUF_ITEM(lip);
509 struct xfs_buf *bp = bip->bli_buf;
Mark Tinguelyc883d0c2012-12-04 17:18:05 -0600510 int aborted, clean, i;
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000511 uint hold;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700512
Dave Chinner64fc35d2010-05-07 11:04:34 +1000513 /* Clear the buffer's association with this transaction. */
Christoph Hellwigbf9d9012011-07-13 13:43:49 +0200514 bp->b_transp = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700515
516 /*
Dave Chinner64fc35d2010-05-07 11:04:34 +1000517 * If this is a transaction abort, don't return early. Instead, allow
518 * the brelse to happen. Normally it would be done for stale
519 * (cancelled) buffers at unpin time, but we'll never go through the
520 * pin/unpin cycle if we abort inside commit.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521 */
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000522 aborted = (lip->li_flags & XFS_LI_ABORTED) != 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523
524 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525 * Before possibly freeing the buf item, determine if we should
526 * release the buffer at the end of this routine.
527 */
528 hold = bip->bli_flags & XFS_BLI_HOLD;
Dave Chinner64fc35d2010-05-07 11:04:34 +1000529
530 /* Clear the per transaction state. */
531 bip->bli_flags &= ~(XFS_BLI_LOGGED | XFS_BLI_HOLD);
532
533 /*
534 * If the buf item is marked stale, then don't do anything. We'll
535 * unlock the buffer and free the buf item when the buffer is unpinned
536 * for the last time.
537 */
538 if (bip->bli_flags & XFS_BLI_STALE) {
539 trace_xfs_buf_item_unlock_stale(bip);
Mark Tinguelyb9438172012-12-04 17:18:03 -0600540 ASSERT(bip->__bli_format.blf_flags & XFS_BLF_CANCEL);
Dave Chinner64fc35d2010-05-07 11:04:34 +1000541 if (!aborted) {
542 atomic_dec(&bip->bli_refcount);
543 return;
544 }
545 }
546
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000547 trace_xfs_buf_item_unlock(bip);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700548
549 /*
Dave Chinner64fc35d2010-05-07 11:04:34 +1000550 * If the buf item isn't tracking any data, free it, otherwise drop the
551 * reference we hold to it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552 */
Mark Tinguelyc883d0c2012-12-04 17:18:05 -0600553 clean = 1;
554 for (i = 0; i < bip->bli_format_count; i++) {
555 if (!xfs_bitmap_empty(bip->bli_formats[i].blf_data_map,
556 bip->bli_formats[i].blf_map_size)) {
557 clean = 0;
558 break;
559 }
560 }
561 if (clean)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700562 xfs_buf_item_relse(bp);
Dave Chinner64fc35d2010-05-07 11:04:34 +1000563 else
564 atomic_dec(&bip->bli_refcount);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565
Dave Chinner64fc35d2010-05-07 11:04:34 +1000566 if (!hold)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700567 xfs_buf_relse(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568}
569
570/*
571 * This is called to find out where the oldest active copy of the
572 * buf log item in the on disk log resides now that the last log
573 * write of it completed at the given lsn.
574 * We always re-log all the dirty data in a buffer, so usually the
575 * latest copy in the on disk log is the only one that matters. For
576 * those cases we simply return the given lsn.
577 *
578 * The one exception to this is for buffers full of newly allocated
579 * inodes. These buffers are only relogged with the XFS_BLI_INODE_BUF
580 * flag set, indicating that only the di_next_unlinked fields from the
581 * inodes in the buffers will be replayed during recovery. If the
582 * original newly allocated inode images have not yet been flushed
583 * when the buffer is so relogged, then we need to make sure that we
584 * keep the old images in the 'active' portion of the log. We do this
585 * by returning the original lsn of that transaction here rather than
586 * the current one.
587 */
Christoph Hellwigba0f32d2005-06-21 15:36:52 +1000588STATIC xfs_lsn_t
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589xfs_buf_item_committed(
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000590 struct xfs_log_item *lip,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700591 xfs_lsn_t lsn)
592{
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000593 struct xfs_buf_log_item *bip = BUF_ITEM(lip);
594
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000595 trace_xfs_buf_item_committed(bip);
596
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000597 if ((bip->bli_flags & XFS_BLI_INODE_ALLOC_BUF) && lip->li_lsn != 0)
598 return lip->li_lsn;
599 return lsn;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600}
601
Christoph Hellwigba0f32d2005-06-21 15:36:52 +1000602STATIC void
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000603xfs_buf_item_committing(
604 struct xfs_log_item *lip,
605 xfs_lsn_t commit_lsn)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606{
607}
608
609/*
610 * This is the ops vector shared by all buf log items.
611 */
Christoph Hellwig272e42b2011-10-28 09:54:24 +0000612static const struct xfs_item_ops xfs_buf_item_ops = {
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000613 .iop_size = xfs_buf_item_size,
614 .iop_format = xfs_buf_item_format,
615 .iop_pin = xfs_buf_item_pin,
616 .iop_unpin = xfs_buf_item_unpin,
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000617 .iop_unlock = xfs_buf_item_unlock,
618 .iop_committed = xfs_buf_item_committed,
619 .iop_push = xfs_buf_item_push,
Christoph Hellwig7bfa31d2010-06-23 18:11:15 +1000620 .iop_committing = xfs_buf_item_committing
Linus Torvalds1da177e2005-04-16 15:20:36 -0700621};
622
Dave Chinner372cc85e2012-06-22 18:50:12 +1000623STATIC int
624xfs_buf_item_get_format(
625 struct xfs_buf_log_item *bip,
626 int count)
627{
628 ASSERT(bip->bli_formats == NULL);
629 bip->bli_format_count = count;
630
631 if (count == 1) {
Mark Tinguelyb9438172012-12-04 17:18:03 -0600632 bip->bli_formats = &bip->__bli_format;
Dave Chinner372cc85e2012-06-22 18:50:12 +1000633 return 0;
634 }
635
636 bip->bli_formats = kmem_zalloc(count * sizeof(struct xfs_buf_log_format),
637 KM_SLEEP);
638 if (!bip->bli_formats)
639 return ENOMEM;
640 return 0;
641}
642
643STATIC void
644xfs_buf_item_free_format(
645 struct xfs_buf_log_item *bip)
646{
Mark Tinguelyb9438172012-12-04 17:18:03 -0600647 if (bip->bli_formats != &bip->__bli_format) {
Dave Chinner372cc85e2012-06-22 18:50:12 +1000648 kmem_free(bip->bli_formats);
649 bip->bli_formats = NULL;
650 }
651}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700652
653/*
654 * Allocate a new buf log item to go with the given buffer.
655 * Set the buffer's b_fsprivate field to point to the new
656 * buf log item. If there are other item's attached to the
657 * buffer (see xfs_buf_attach_iodone() below), then put the
658 * buf log item at the front.
659 */
660void
661xfs_buf_item_init(
662 xfs_buf_t *bp,
663 xfs_mount_t *mp)
664{
Christoph Hellwigadadbee2011-07-13 13:43:49 +0200665 xfs_log_item_t *lip = bp->b_fspriv;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666 xfs_buf_log_item_t *bip;
667 int chunks;
668 int map_size;
Dave Chinner372cc85e2012-06-22 18:50:12 +1000669 int error;
670 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671
672 /*
673 * Check to see if there is already a buf log item for
674 * this buffer. If there is, it is guaranteed to be
675 * the first. If we do already have one, there is
676 * nothing to do here so return.
677 */
Dave Chinnerebad8612010-09-22 10:47:20 +1000678 ASSERT(bp->b_target->bt_mount == mp);
Christoph Hellwigadadbee2011-07-13 13:43:49 +0200679 if (lip != NULL && lip->li_type == XFS_LI_BUF)
680 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700681
Dave Chinner372cc85e2012-06-22 18:50:12 +1000682 bip = kmem_zone_zalloc(xfs_buf_item_zone, KM_SLEEP);
Dave Chinner43f5efc2010-03-23 10:10:00 +1100683 xfs_log_item_init(mp, &bip->bli_item, XFS_LI_BUF, &xfs_buf_item_ops);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700684 bip->bli_buf = bp;
Lachlan McIlroye1f5dbd2008-09-17 16:52:13 +1000685 xfs_buf_hold(bp);
Dave Chinner372cc85e2012-06-22 18:50:12 +1000686
687 /*
688 * chunks is the number of XFS_BLF_CHUNK size pieces the buffer
689 * can be divided into. Make sure not to truncate any pieces.
690 * map_size is the size of the bitmap needed to describe the
691 * chunks of the buffer.
692 *
693 * Discontiguous buffer support follows the layout of the underlying
694 * buffer. This makes the implementation as simple as possible.
695 */
696 error = xfs_buf_item_get_format(bip, bp->b_map_count);
697 ASSERT(error == 0);
698
699 for (i = 0; i < bip->bli_format_count; i++) {
700 chunks = DIV_ROUND_UP(BBTOB(bp->b_maps[i].bm_len),
701 XFS_BLF_CHUNK);
702 map_size = DIV_ROUND_UP(chunks, NBWORD);
703
704 bip->bli_formats[i].blf_type = XFS_LI_BUF;
705 bip->bli_formats[i].blf_blkno = bp->b_maps[i].bm_bn;
706 bip->bli_formats[i].blf_len = bp->b_maps[i].bm_len;
707 bip->bli_formats[i].blf_map_size = map_size;
708 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709
710#ifdef XFS_TRANS_DEBUG
711 /*
712 * Allocate the arrays for tracking what needs to be logged
713 * and what our callers request to be logged. bli_orig
714 * holds a copy of the original, clean buffer for comparison
715 * against, and bli_logged keeps a 1 bit flag per byte in
716 * the buffer to indicate which bytes the callers have asked
717 * to have logged.
718 */
Dave Chinneraa0e8832012-04-23 15:58:52 +1000719 bip->bli_orig = kmem_alloc(BBTOB(bp->b_length), KM_SLEEP);
720 memcpy(bip->bli_orig, bp->b_addr, BBTOB(bp->b_length));
721 bip->bli_logged = kmem_zalloc(BBTOB(bp->b_length) / NBBY, KM_SLEEP);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700722#endif
723
724 /*
725 * Put the buf item into the list of items attached to the
726 * buffer at the front.
727 */
Christoph Hellwigadadbee2011-07-13 13:43:49 +0200728 if (bp->b_fspriv)
729 bip->bli_item.li_bio_list = bp->b_fspriv;
730 bp->b_fspriv = bip;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731}
732
733
734/*
735 * Mark bytes first through last inclusive as dirty in the buf
736 * item's bitmap.
737 */
738void
Dave Chinner372cc85e2012-06-22 18:50:12 +1000739xfs_buf_item_log_segment(
740 struct xfs_buf_log_item *bip,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741 uint first,
Dave Chinner372cc85e2012-06-22 18:50:12 +1000742 uint last,
743 uint *map)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700744{
745 uint first_bit;
746 uint last_bit;
747 uint bits_to_set;
748 uint bits_set;
749 uint word_num;
750 uint *wordp;
751 uint bit;
752 uint end_bit;
753 uint mask;
754
755 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756 * Convert byte offsets to bit numbers.
757 */
Dave Chinnerc1155412010-05-07 11:05:19 +1000758 first_bit = first >> XFS_BLF_SHIFT;
759 last_bit = last >> XFS_BLF_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700760
761 /*
762 * Calculate the total number of bits to be set.
763 */
764 bits_to_set = last_bit - first_bit + 1;
765
766 /*
767 * Get a pointer to the first word in the bitmap
768 * to set a bit in.
769 */
770 word_num = first_bit >> BIT_TO_WORD_SHIFT;
Dave Chinner372cc85e2012-06-22 18:50:12 +1000771 wordp = &map[word_num];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772
773 /*
774 * Calculate the starting bit in the first word.
775 */
776 bit = first_bit & (uint)(NBWORD - 1);
777
778 /*
779 * First set any bits in the first word of our range.
780 * If it starts at bit 0 of the word, it will be
781 * set below rather than here. That is what the variable
782 * bit tells us. The variable bits_set tracks the number
783 * of bits that have been set so far. End_bit is the number
784 * of the last bit to be set in this word plus one.
785 */
786 if (bit) {
787 end_bit = MIN(bit + bits_to_set, (uint)NBWORD);
788 mask = ((1 << (end_bit - bit)) - 1) << bit;
789 *wordp |= mask;
790 wordp++;
791 bits_set = end_bit - bit;
792 } else {
793 bits_set = 0;
794 }
795
796 /*
797 * Now set bits a whole word at a time that are between
798 * first_bit and last_bit.
799 */
800 while ((bits_to_set - bits_set) >= NBWORD) {
801 *wordp |= 0xffffffff;
802 bits_set += NBWORD;
803 wordp++;
804 }
805
806 /*
807 * Finally, set any bits left to be set in one last partial word.
808 */
809 end_bit = bits_to_set - bits_set;
810 if (end_bit) {
811 mask = (1 << end_bit) - 1;
812 *wordp |= mask;
813 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700814}
815
Dave Chinner372cc85e2012-06-22 18:50:12 +1000816/*
817 * Mark bytes first through last inclusive as dirty in the buf
818 * item's bitmap.
819 */
820void
821xfs_buf_item_log(
822 xfs_buf_log_item_t *bip,
823 uint first,
824 uint last)
825{
826 int i;
827 uint start;
828 uint end;
829 struct xfs_buf *bp = bip->bli_buf;
830
831 /*
832 * Mark the item as having some dirty data for
833 * quick reference in xfs_buf_item_dirty.
834 */
835 bip->bli_flags |= XFS_BLI_DIRTY;
836
837 /*
838 * walk each buffer segment and mark them dirty appropriately.
839 */
840 start = 0;
841 for (i = 0; i < bip->bli_format_count; i++) {
842 if (start > last)
843 break;
844 end = start + BBTOB(bp->b_maps[i].bm_len);
845 if (first > end) {
846 start += BBTOB(bp->b_maps[i].bm_len);
847 continue;
848 }
849 if (first < start)
850 first = start;
851 if (end > last)
852 end = last;
853
854 xfs_buf_item_log_segment(bip, first, end,
855 &bip->bli_formats[i].blf_data_map[0]);
856
857 start += bp->b_maps[i].bm_len;
858 }
859}
860
Linus Torvalds1da177e2005-04-16 15:20:36 -0700861
862/*
863 * Return 1 if the buffer has some data that has been logged (at any
864 * point, not just the current transaction) and 0 if not.
865 */
866uint
867xfs_buf_item_dirty(
868 xfs_buf_log_item_t *bip)
869{
870 return (bip->bli_flags & XFS_BLI_DIRTY);
871}
872
Lachlan McIlroye1f5dbd2008-09-17 16:52:13 +1000873STATIC void
874xfs_buf_item_free(
875 xfs_buf_log_item_t *bip)
876{
877#ifdef XFS_TRANS_DEBUG
878 kmem_free(bip->bli_orig);
879 kmem_free(bip->bli_logged);
880#endif /* XFS_TRANS_DEBUG */
881
Dave Chinner372cc85e2012-06-22 18:50:12 +1000882 xfs_buf_item_free_format(bip);
Lachlan McIlroye1f5dbd2008-09-17 16:52:13 +1000883 kmem_zone_free(xfs_buf_item_zone, bip);
884}
885
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886/*
887 * This is called when the buf log item is no longer needed. It should
888 * free the buf log item associated with the given buffer and clear
889 * the buffer's pointer to the buf log item. If there are no more
890 * items in the list, clear the b_iodone field of the buffer (see
891 * xfs_buf_attach_iodone() below).
892 */
893void
894xfs_buf_item_relse(
895 xfs_buf_t *bp)
896{
897 xfs_buf_log_item_t *bip;
898
Christoph Hellwig0b1b2132009-12-14 23:14:59 +0000899 trace_xfs_buf_item_relse(bp, _RET_IP_);
900
Christoph Hellwigadadbee2011-07-13 13:43:49 +0200901 bip = bp->b_fspriv;
902 bp->b_fspriv = bip->bli_item.li_bio_list;
Christoph Hellwigcb669ca2011-07-13 13:43:49 +0200903 if (bp->b_fspriv == NULL)
904 bp->b_iodone = NULL;
Christoph Hellwigadadbee2011-07-13 13:43:49 +0200905
Lachlan McIlroye1f5dbd2008-09-17 16:52:13 +1000906 xfs_buf_rele(bp);
907 xfs_buf_item_free(bip);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908}
909
910
911/*
912 * Add the given log item with its callback to the list of callbacks
913 * to be called when the buffer's I/O completes. If it is not set
914 * already, set the buffer's b_iodone() routine to be
915 * xfs_buf_iodone_callbacks() and link the log item into the list of
916 * items rooted at b_fsprivate. Items are always added as the second
917 * entry in the list if there is a first, because the buf item code
918 * assumes that the buf log item is first.
919 */
920void
921xfs_buf_attach_iodone(
922 xfs_buf_t *bp,
923 void (*cb)(xfs_buf_t *, xfs_log_item_t *),
924 xfs_log_item_t *lip)
925{
926 xfs_log_item_t *head_lip;
927
Christoph Hellwig0c842ad2011-07-08 14:36:19 +0200928 ASSERT(xfs_buf_islocked(bp));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929
930 lip->li_cb = cb;
Christoph Hellwigadadbee2011-07-13 13:43:49 +0200931 head_lip = bp->b_fspriv;
932 if (head_lip) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700933 lip->li_bio_list = head_lip->li_bio_list;
934 head_lip->li_bio_list = lip;
935 } else {
Christoph Hellwigadadbee2011-07-13 13:43:49 +0200936 bp->b_fspriv = lip;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700937 }
938
Christoph Hellwigcb669ca2011-07-13 13:43:49 +0200939 ASSERT(bp->b_iodone == NULL ||
940 bp->b_iodone == xfs_buf_iodone_callbacks);
941 bp->b_iodone = xfs_buf_iodone_callbacks;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700942}
943
Dave Chinnerc90821a2010-12-03 17:00:52 +1100944/*
945 * We can have many callbacks on a buffer. Running the callbacks individually
946 * can cause a lot of contention on the AIL lock, so we allow for a single
947 * callback to be able to scan the remaining lip->li_bio_list for other items
948 * of the same type and callback to be processed in the first call.
949 *
950 * As a result, the loop walking the callback list below will also modify the
951 * list. it removes the first item from the list and then runs the callback.
952 * The loop then restarts from the new head of the list. This allows the
953 * callback to scan and modify the list attached to the buffer and we don't
954 * have to care about maintaining a next item pointer.
955 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700956STATIC void
957xfs_buf_do_callbacks(
Dave Chinnerc90821a2010-12-03 17:00:52 +1100958 struct xfs_buf *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959{
Dave Chinnerc90821a2010-12-03 17:00:52 +1100960 struct xfs_log_item *lip;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961
Christoph Hellwigadadbee2011-07-13 13:43:49 +0200962 while ((lip = bp->b_fspriv) != NULL) {
963 bp->b_fspriv = lip->li_bio_list;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700964 ASSERT(lip->li_cb != NULL);
965 /*
966 * Clear the next pointer so we don't have any
967 * confusion if the item is added to another buf.
968 * Don't touch the log item after calling its
969 * callback, because it could have freed itself.
970 */
971 lip->li_bio_list = NULL;
972 lip->li_cb(bp, lip);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700973 }
974}
975
976/*
977 * This is the iodone() function for buffers which have had callbacks
978 * attached to them by xfs_buf_attach_iodone(). It should remove each
979 * log item from the buffer's list and call the callback of each in turn.
980 * When done, the buffer's fsprivate field is set to NULL and the buffer
981 * is unlocked with a call to iodone().
982 */
983void
984xfs_buf_iodone_callbacks(
Christoph Hellwigbfc60172011-01-07 13:02:23 +0000985 struct xfs_buf *bp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700986{
Christoph Hellwigbfc60172011-01-07 13:02:23 +0000987 struct xfs_log_item *lip = bp->b_fspriv;
988 struct xfs_mount *mp = lip->li_mountp;
989 static ulong lasttime;
990 static xfs_buftarg_t *lasttarg;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700991
Chandra Seetharaman5a52c2a582011-07-22 23:39:51 +0000992 if (likely(!xfs_buf_geterror(bp)))
Christoph Hellwigbfc60172011-01-07 13:02:23 +0000993 goto do_callbacks;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700994
Christoph Hellwigbfc60172011-01-07 13:02:23 +0000995 /*
996 * If we've already decided to shutdown the filesystem because of
997 * I/O errors, there's no point in giving this a retry.
998 */
999 if (XFS_FORCED_SHUTDOWN(mp)) {
Christoph Hellwigc867cb62011-10-10 16:52:46 +00001000 xfs_buf_stale(bp);
Christoph Hellwigc867cb62011-10-10 16:52:46 +00001001 XFS_BUF_DONE(bp);
Christoph Hellwigbfc60172011-01-07 13:02:23 +00001002 trace_xfs_buf_item_iodone(bp, _RET_IP_);
1003 goto do_callbacks;
1004 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001005
Chandra Seetharaman49074c02011-07-22 23:40:40 +00001006 if (bp->b_target != lasttarg ||
Christoph Hellwigbfc60172011-01-07 13:02:23 +00001007 time_after(jiffies, (lasttime + 5*HZ))) {
1008 lasttime = jiffies;
Christoph Hellwigb38505b2011-10-10 16:52:50 +00001009 xfs_buf_ioerror_alert(bp, __func__);
Christoph Hellwigbfc60172011-01-07 13:02:23 +00001010 }
Chandra Seetharaman49074c02011-07-22 23:40:40 +00001011 lasttarg = bp->b_target;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001012
Christoph Hellwigbfc60172011-01-07 13:02:23 +00001013 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03001014 * If the write was asynchronous then no one will be looking for the
Christoph Hellwigbfc60172011-01-07 13:02:23 +00001015 * error. Clear the error state and write the buffer out again.
1016 *
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001017 * XXX: This helps against transient write errors, but we need to find
1018 * a way to shut the filesystem down if the writes keep failing.
1019 *
1020 * In practice we'll shut the filesystem down soon as non-transient
1021 * erorrs tend to affect the whole device and a failing log write
1022 * will make us give up. But we really ought to do better here.
Christoph Hellwigbfc60172011-01-07 13:02:23 +00001023 */
1024 if (XFS_BUF_ISASYNC(bp)) {
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001025 ASSERT(bp->b_iodone != NULL);
1026
1027 trace_xfs_buf_item_iodone_async(bp, _RET_IP_);
1028
Chandra Seetharaman5a52c2a582011-07-22 23:39:51 +00001029 xfs_buf_ioerror(bp, 0); /* errno of 0 unsets the flag */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001030
Christoph Hellwigbfc60172011-01-07 13:02:23 +00001031 if (!XFS_BUF_ISSTALE(bp)) {
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001032 bp->b_flags |= XBF_WRITE | XBF_ASYNC | XBF_DONE;
Christoph Hellwiga2dcf5d2012-07-13 02:24:10 -04001033 xfs_buf_iorequest(bp);
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001034 } else {
1035 xfs_buf_relse(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001036 }
Christoph Hellwig43ff2122012-04-23 15:58:39 +10001037
Linus Torvalds1da177e2005-04-16 15:20:36 -07001038 return;
1039 }
Christoph Hellwig0b1b2132009-12-14 23:14:59 +00001040
Christoph Hellwigbfc60172011-01-07 13:02:23 +00001041 /*
1042 * If the write of the buffer was synchronous, we want to make
1043 * sure to return the error to the caller of xfs_bwrite().
1044 */
Christoph Hellwigc867cb62011-10-10 16:52:46 +00001045 xfs_buf_stale(bp);
Christoph Hellwigbfc60172011-01-07 13:02:23 +00001046 XFS_BUF_DONE(bp);
Christoph Hellwigbfc60172011-01-07 13:02:23 +00001047
1048 trace_xfs_buf_error_relse(bp, _RET_IP_);
Christoph Hellwigbfc60172011-01-07 13:02:23 +00001049
1050do_callbacks:
Dave Chinnerc90821a2010-12-03 17:00:52 +11001051 xfs_buf_do_callbacks(bp);
Christoph Hellwigadadbee2011-07-13 13:43:49 +02001052 bp->b_fspriv = NULL;
Christoph Hellwigcb669ca2011-07-13 13:43:49 +02001053 bp->b_iodone = NULL;
Christoph Hellwig1a1a3e92010-10-06 18:41:18 +00001054 xfs_buf_ioend(bp, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001055}
1056
1057/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001058 * This is the iodone() function for buffers which have been
1059 * logged. It is called when they are eventually flushed out.
1060 * It should remove the buf item from the AIL, and free the buf item.
1061 * It is called by xfs_buf_iodone_callbacks() above which will take
1062 * care of cleaning up the buffer itself.
1063 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001064void
1065xfs_buf_iodone(
Christoph Hellwigca30b2a2010-06-23 18:11:15 +10001066 struct xfs_buf *bp,
1067 struct xfs_log_item *lip)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001068{
Christoph Hellwigca30b2a2010-06-23 18:11:15 +10001069 struct xfs_ail *ailp = lip->li_ailp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001070
Christoph Hellwigca30b2a2010-06-23 18:11:15 +10001071 ASSERT(BUF_ITEM(lip)->bli_buf == bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001072
Lachlan McIlroye1f5dbd2008-09-17 16:52:13 +10001073 xfs_buf_rele(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001074
1075 /*
1076 * If we are forcibly shutting down, this may well be
1077 * off the AIL already. That's because we simulate the
1078 * log-committed callbacks to unpin these buffers. Or we may never
1079 * have put this item on AIL because of the transaction was
David Chinner783a2f62008-10-30 17:39:58 +11001080 * aborted forcibly. xfs_trans_ail_delete() takes care of these.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001081 *
1082 * Either way, AIL is useless if we're forcing a shutdown.
1083 */
David Chinnerfc1829f2008-10-30 17:39:46 +11001084 spin_lock(&ailp->xa_lock);
Dave Chinner04913fd2012-04-23 15:58:41 +10001085 xfs_trans_ail_delete(ailp, lip, SHUTDOWN_CORRUPT_INCORE);
Christoph Hellwigca30b2a2010-06-23 18:11:15 +10001086 xfs_buf_item_free(BUF_ITEM(lip));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001087}