blob: dcbc9b20474b88b9bfb9163e1c88ad56e1a5ecc2 [file] [log] [blame]
Ricardo Labiaga85e174b2010-10-20 00:17:58 -04001/*
2 * pNFS functions to call and manage layout drivers.
3 *
4 * Copyright (c) 2002 [year of first publication]
5 * The Regents of the University of Michigan
6 * All Rights Reserved
7 *
8 * Dean Hildebrand <dhildebz@umich.edu>
9 *
10 * Permission is granted to use, copy, create derivative works, and
11 * redistribute this software and such derivative works for any purpose,
12 * so long as the name of the University of Michigan is not used in
13 * any advertising or publicity pertaining to the use or distribution
14 * of this software without specific, written prior authorization. If
15 * the above copyright notice or any other identification of the
16 * University of Michigan is included in any copy of any portion of
17 * this software, then the disclaimer below must also be included.
18 *
19 * This software is provided as is, without representation or warranty
20 * of any kind either express or implied, including without limitation
21 * the implied warranties of merchantability, fitness for a particular
22 * purpose, or noninfringement. The Regents of the University of
23 * Michigan shall not be liable for any damages, including special,
24 * indirect, incidental, or consequential damages, with respect to any
25 * claim arising out of or in connection with the use of the software,
26 * even if it has been or is hereafter advised of the possibility of
27 * such damages.
28 */
29
30#include <linux/nfs_fs.h>
Trond Myklebust493292d2011-07-13 15:58:28 -040031#include <linux/nfs_page.h>
Paul Gortmaker143cb492011-07-01 14:23:34 -040032#include <linux/module.h>
Andy Adamson974cec82010-10-20 00:18:02 -040033#include "internal.h"
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040034#include "pnfs.h"
Andy Adamson64419a92011-03-01 01:34:16 +000035#include "iostat.h"
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040036
37#define NFSDBG_FACILITY NFSDBG_PNFS
Trond Myklebust25c75332012-09-18 17:01:12 -040038#define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ)
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040039
Fred Isaman02c35fc2010-10-20 00:17:59 -040040/* Locking:
41 *
42 * pnfs_spinlock:
43 * protects pnfs_modules_tbl.
44 */
45static DEFINE_SPINLOCK(pnfs_spinlock);
46
47/*
48 * pnfs_modules_tbl holds all pnfs modules
49 */
50static LIST_HEAD(pnfs_modules_tbl);
51
52/* Return the registered pnfs layout driver module matching given id */
53static struct pnfs_layoutdriver_type *
54find_pnfs_driver_locked(u32 id)
55{
56 struct pnfs_layoutdriver_type *local;
57
58 list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid)
59 if (local->id == id)
60 goto out;
61 local = NULL;
62out:
63 dprintk("%s: Searching for id %u, found %p\n", __func__, id, local);
64 return local;
65}
66
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040067static struct pnfs_layoutdriver_type *
68find_pnfs_driver(u32 id)
69{
Fred Isaman02c35fc2010-10-20 00:17:59 -040070 struct pnfs_layoutdriver_type *local;
71
72 spin_lock(&pnfs_spinlock);
73 local = find_pnfs_driver_locked(id);
Trond Myklebust0a9c63f2012-06-15 13:02:58 -040074 if (local != NULL && !try_module_get(local->owner)) {
75 dprintk("%s: Could not grab reference on module\n", __func__);
76 local = NULL;
77 }
Fred Isaman02c35fc2010-10-20 00:17:59 -040078 spin_unlock(&pnfs_spinlock);
79 return local;
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040080}
81
82void
83unset_pnfs_layoutdriver(struct nfs_server *nfss)
84{
Benny Halevy738fd0f32011-07-30 20:52:36 -040085 if (nfss->pnfs_curr_ld) {
86 if (nfss->pnfs_curr_ld->clear_layoutdriver)
87 nfss->pnfs_curr_ld->clear_layoutdriver(nfss);
Trond Myklebust2a4c8992012-06-14 13:08:38 -040088 /* Decrement the MDS count. Purge the deviceid cache if zero */
89 if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count))
90 nfs4_deviceid_purge_client(nfss->nfs_client);
Fred Isaman02c35fc2010-10-20 00:17:59 -040091 module_put(nfss->pnfs_curr_ld->owner);
Benny Halevy738fd0f32011-07-30 20:52:36 -040092 }
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040093 nfss->pnfs_curr_ld = NULL;
94}
95
96/*
97 * Try to set the server's pnfs module to the pnfs layout type specified by id.
98 * Currently only one pNFS layout driver per filesystem is supported.
99 *
100 * @id layout type. Zero (illegal layout type) indicates pNFS not in use.
101 */
102void
Benny Halevy738fd0f32011-07-30 20:52:36 -0400103set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh,
104 u32 id)
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400105{
106 struct pnfs_layoutdriver_type *ld_type = NULL;
107
108 if (id == 0)
109 goto out_no_driver;
110 if (!(server->nfs_client->cl_exchange_flags &
111 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500112 printk(KERN_ERR "NFS: %s: id %u cl_exchange_flags 0x%x\n",
113 __func__, id, server->nfs_client->cl_exchange_flags);
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400114 goto out_no_driver;
115 }
116 ld_type = find_pnfs_driver(id);
117 if (!ld_type) {
118 request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX, id);
119 ld_type = find_pnfs_driver(id);
120 if (!ld_type) {
121 dprintk("%s: No pNFS module found for %u.\n",
122 __func__, id);
123 goto out_no_driver;
124 }
125 }
126 server->pnfs_curr_ld = ld_type;
Benny Halevy738fd0f32011-07-30 20:52:36 -0400127 if (ld_type->set_layoutdriver
128 && ld_type->set_layoutdriver(server, mntfh)) {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500129 printk(KERN_ERR "NFS: %s: Error initializing pNFS layout "
130 "driver %u.\n", __func__, id);
Benny Halevy738fd0f32011-07-30 20:52:36 -0400131 module_put(ld_type->owner);
132 goto out_no_driver;
133 }
Trond Myklebust2a4c8992012-06-14 13:08:38 -0400134 /* Bump the MDS count */
135 atomic_inc(&server->nfs_client->cl_mds_count);
Christoph Hellwigea8eecd2011-03-01 01:34:21 +0000136
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400137 dprintk("%s: pNFS module for %u set\n", __func__, id);
138 return;
139
140out_no_driver:
141 dprintk("%s: Using NFSv4 I/O\n", __func__);
142 server->pnfs_curr_ld = NULL;
143}
Fred Isaman02c35fc2010-10-20 00:17:59 -0400144
145int
146pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
147{
148 int status = -EINVAL;
149 struct pnfs_layoutdriver_type *tmp;
150
151 if (ld_type->id == 0) {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500152 printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__);
Fred Isaman02c35fc2010-10-20 00:17:59 -0400153 return status;
154 }
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400155 if (!ld_type->alloc_lseg || !ld_type->free_lseg) {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500156 printk(KERN_ERR "NFS: %s Layout driver must provide "
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400157 "alloc_lseg and free_lseg.\n", __func__);
158 return status;
159 }
Fred Isaman02c35fc2010-10-20 00:17:59 -0400160
161 spin_lock(&pnfs_spinlock);
162 tmp = find_pnfs_driver_locked(ld_type->id);
163 if (!tmp) {
164 list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl);
165 status = 0;
166 dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id,
167 ld_type->name);
168 } else {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500169 printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n",
Fred Isaman02c35fc2010-10-20 00:17:59 -0400170 __func__, ld_type->id);
171 }
172 spin_unlock(&pnfs_spinlock);
173
174 return status;
175}
176EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver);
177
178void
179pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
180{
181 dprintk("%s Deregistering id:%u\n", __func__, ld_type->id);
182 spin_lock(&pnfs_spinlock);
183 list_del(&ld_type->pnfs_tblid);
184 spin_unlock(&pnfs_spinlock);
185}
186EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver);
Benny Halevye5e94012010-10-20 00:18:01 -0400187
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400188/*
189 * pNFS client layout cache
190 */
191
Fred Isamancc6e5342011-01-06 11:36:28 +0000192/* Need to hold i_lock if caller does not already hold reference */
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000193void
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400194pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo)
Benny Halevye5e94012010-10-20 00:18:01 -0400195{
Fred Isamancc6e5342011-01-06 11:36:28 +0000196 atomic_inc(&lo->plh_refcount);
197}
198
Benny Halevy636fb9c2011-05-22 19:51:33 +0300199static struct pnfs_layout_hdr *
200pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags)
201{
202 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
Trond Myklebust57934272012-09-20 20:37:23 -0400203 return ld->alloc_layout_hdr(ino, gfp_flags);
Benny Halevy636fb9c2011-05-22 19:51:33 +0300204}
205
206static void
207pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo)
208{
Trond Myklebust9c626382012-09-20 17:31:43 -0400209 struct nfs_server *server = NFS_SERVER(lo->plh_inode);
210 struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld;
211
212 if (!list_empty(&lo->plh_layouts)) {
213 struct nfs_client *clp = server->nfs_client;
214
215 spin_lock(&clp->cl_lock);
216 list_del_init(&lo->plh_layouts);
217 spin_unlock(&clp->cl_lock);
218 }
Peng Tao9fa40752011-07-30 20:52:32 -0400219 put_rpccred(lo->plh_lc_cred);
Trond Myklebust57934272012-09-20 20:37:23 -0400220 return ld->free_layout_hdr(lo);
Benny Halevy636fb9c2011-05-22 19:51:33 +0300221}
222
Fred Isamancc6e5342011-01-06 11:36:28 +0000223static void
Trond Myklebust6622c3e2012-09-20 17:23:11 -0400224pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo)
Fred Isamancc6e5342011-01-06 11:36:28 +0000225{
Trond Myklebustbb346f62012-09-20 15:52:13 -0400226 struct nfs_inode *nfsi = NFS_I(lo->plh_inode);
Fred Isamancc6e5342011-01-06 11:36:28 +0000227 dprintk("%s: freeing layout cache %p\n", __func__, lo);
Trond Myklebustbb346f62012-09-20 15:52:13 -0400228 nfsi->layout = NULL;
229 /* Reset MDS Threshold I/O counters */
230 nfsi->write_io = 0;
231 nfsi->read_io = 0;
Benny Halevye5e94012010-10-20 00:18:01 -0400232}
233
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400234void
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400235pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
Andy Adamson974cec82010-10-20 00:18:02 -0400236{
Fred Isamancc6e5342011-01-06 11:36:28 +0000237 struct inode *inode = lo->plh_inode;
238
239 if (atomic_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) {
Trond Myklebust6622c3e2012-09-20 17:23:11 -0400240 pnfs_detach_layout_hdr(lo);
Fred Isamancc6e5342011-01-06 11:36:28 +0000241 spin_unlock(&inode->i_lock);
Trond Myklebust6622c3e2012-09-20 17:23:11 -0400242 pnfs_free_layout_hdr(lo);
Fred Isamancc6e5342011-01-06 11:36:28 +0000243 }
Andy Adamson974cec82010-10-20 00:18:02 -0400244}
245
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400246static int
247pnfs_iomode_to_fail_bit(u32 iomode)
248{
249 return iomode == IOMODE_RW ?
250 NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED;
251}
252
253static void
Trond Myklebust3e621212012-09-24 13:07:16 -0400254pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400255{
Trond Myklebust25c75332012-09-18 17:01:12 -0400256 lo->plh_retry_timestamp = jiffies;
Trond Myklebust3e621212012-09-24 13:07:16 -0400257 if (test_and_set_bit(fail_bit, &lo->plh_flags))
258 atomic_inc(&lo->plh_refcount);
259}
260
261static void
262pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
263{
264 if (test_and_clear_bit(fail_bit, &lo->plh_flags))
265 atomic_dec(&lo->plh_refcount);
266}
267
268static void
269pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode)
270{
271 struct inode *inode = lo->plh_inode;
Trond Myklebust115ce572012-09-20 21:19:43 -0400272 struct pnfs_layout_range range = {
273 .iomode = iomode,
274 .offset = 0,
275 .length = NFS4_MAX_UINT64,
276 };
277 LIST_HEAD(head);
Trond Myklebust3e621212012-09-24 13:07:16 -0400278
279 spin_lock(&inode->i_lock);
280 pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
Trond Myklebust115ce572012-09-20 21:19:43 -0400281 pnfs_mark_matching_lsegs_invalid(lo, &head, &range);
Trond Myklebust3e621212012-09-24 13:07:16 -0400282 spin_unlock(&inode->i_lock);
Trond Myklebust115ce572012-09-20 21:19:43 -0400283 pnfs_free_lseg_list(&head);
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400284 dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__,
285 iomode == IOMODE_RW ? "RW" : "READ");
286}
287
288static bool
289pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)
290{
Trond Myklebust25c75332012-09-18 17:01:12 -0400291 unsigned long start, end;
Trond Myklebust3e621212012-09-24 13:07:16 -0400292 int fail_bit = pnfs_iomode_to_fail_bit(iomode);
293
294 if (test_bit(fail_bit, &lo->plh_flags) == 0)
Trond Myklebust25c75332012-09-18 17:01:12 -0400295 return false;
296 end = jiffies;
297 start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT;
298 if (!time_in_range(lo->plh_retry_timestamp, start, end)) {
299 /* It is time to retry the failed layoutgets */
Trond Myklebust3e621212012-09-24 13:07:16 -0400300 pnfs_layout_clear_fail_bit(lo, fail_bit);
Trond Myklebust25c75332012-09-18 17:01:12 -0400301 return false;
302 }
303 return true;
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400304}
305
Andy Adamson974cec82010-10-20 00:18:02 -0400306static void
307init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg)
308{
Fred Isaman566052c2011-01-06 11:36:20 +0000309 INIT_LIST_HEAD(&lseg->pls_list);
Peng Taoa9bae562011-07-30 20:52:33 -0400310 INIT_LIST_HEAD(&lseg->pls_lc_list);
Fred Isaman4541d162011-01-06 11:36:23 +0000311 atomic_set(&lseg->pls_refcount, 1);
312 smp_mb();
313 set_bit(NFS_LSEG_VALID, &lseg->pls_flags);
Fred Isaman566052c2011-01-06 11:36:20 +0000314 lseg->pls_layout = lo;
Andy Adamson974cec82010-10-20 00:18:02 -0400315}
316
Trond Myklebust905ca192012-09-20 20:46:49 -0400317static void pnfs_free_lseg(struct pnfs_layout_segment *lseg)
Andy Adamson974cec82010-10-20 00:18:02 -0400318{
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000319 struct inode *ino = lseg->pls_layout->plh_inode;
Andy Adamson974cec82010-10-20 00:18:02 -0400320
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400321 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
Andy Adamson974cec82010-10-20 00:18:02 -0400322}
323
Fred Isamand684d2a2011-03-01 01:34:13 +0000324static void
Trond Myklebust57036a32012-09-20 16:33:30 -0400325pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo,
326 struct pnfs_layout_segment *lseg)
Andy Adamson974cec82010-10-20 00:18:02 -0400327{
Trond Myklebust57036a32012-09-20 16:33:30 -0400328 struct inode *inode = lo->plh_inode;
Fred Isamand684d2a2011-03-01 01:34:13 +0000329
Benny Halevyd20581a2011-05-22 19:52:03 +0300330 WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
Fred Isamand684d2a2011-03-01 01:34:13 +0000331 list_del_init(&lseg->pls_list);
Trond Myklebust8f0d27d2012-09-20 20:57:11 -0400332 /* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */
333 atomic_dec(&lo->plh_refcount);
Trond Myklebust173f77e2012-09-21 15:49:42 -0400334 if (list_empty(&lo->plh_segs))
335 clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
Fred Isamand684d2a2011-03-01 01:34:13 +0000336 rpc_wake_up(&NFS_SERVER(inode)->roc_rpcwaitq);
337}
338
Fred Isamanbae724e2011-03-01 01:34:15 +0000339void
Trond Myklebust9369a432012-09-18 20:57:08 -0400340pnfs_put_lseg(struct pnfs_layout_segment *lseg)
Fred Isamand684d2a2011-03-01 01:34:13 +0000341{
Trond Myklebust57036a32012-09-20 16:33:30 -0400342 struct pnfs_layout_hdr *lo;
Fred Isamand684d2a2011-03-01 01:34:13 +0000343 struct inode *inode;
344
345 if (!lseg)
346 return;
347
Fred Isaman4541d162011-01-06 11:36:23 +0000348 dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
349 atomic_read(&lseg->pls_refcount),
350 test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
Trond Myklebust57036a32012-09-20 16:33:30 -0400351 lo = lseg->pls_layout;
352 inode = lo->plh_inode;
Fred Isamand684d2a2011-03-01 01:34:13 +0000353 if (atomic_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) {
Trond Myklebust8f0d27d2012-09-20 20:57:11 -0400354 pnfs_get_layout_hdr(lo);
Trond Myklebust57036a32012-09-20 16:33:30 -0400355 pnfs_layout_remove_lseg(lo, lseg);
Fred Isamand684d2a2011-03-01 01:34:13 +0000356 spin_unlock(&inode->i_lock);
Trond Myklebust905ca192012-09-20 20:46:49 -0400357 pnfs_free_lseg(lseg);
Trond Myklebust8f0d27d2012-09-20 20:57:11 -0400358 pnfs_put_layout_hdr(lo);
Fred Isaman4541d162011-01-06 11:36:23 +0000359 }
Andy Adamson974cec82010-10-20 00:18:02 -0400360}
Trond Myklebust9369a432012-09-18 20:57:08 -0400361EXPORT_SYMBOL_GPL(pnfs_put_lseg);
Andy Adamson974cec82010-10-20 00:18:02 -0400362
Benny Halevyfb3296e2011-05-22 19:47:26 +0300363static inline u64
364end_offset(u64 start, u64 len)
Fred Isaman4541d162011-01-06 11:36:23 +0000365{
Benny Halevyfb3296e2011-05-22 19:47:26 +0300366 u64 end;
367
368 end = start + len;
369 return end >= start ? end : NFS4_MAX_UINT64;
370}
371
Benny Halevyfb3296e2011-05-22 19:47:26 +0300372/*
373 * is l2 fully contained in l1?
374 * start1 end1
375 * [----------------------------------)
376 * start2 end2
377 * [----------------)
378 */
379static inline int
380lo_seg_contained(struct pnfs_layout_range *l1,
381 struct pnfs_layout_range *l2)
382{
383 u64 start1 = l1->offset;
384 u64 end1 = end_offset(start1, l1->length);
385 u64 start2 = l2->offset;
386 u64 end2 = end_offset(start2, l2->length);
387
388 return (start1 <= start2) && (end1 >= end2);
389}
390
391/*
392 * is l1 and l2 intersecting?
393 * start1 end1
394 * [----------------------------------)
395 * start2 end2
396 * [----------------)
397 */
398static inline int
399lo_seg_intersecting(struct pnfs_layout_range *l1,
400 struct pnfs_layout_range *l2)
401{
402 u64 start1 = l1->offset;
403 u64 end1 = end_offset(start1, l1->length);
404 u64 start2 = l2->offset;
405 u64 end2 = end_offset(start2, l2->length);
406
407 return (end1 == NFS4_MAX_UINT64 || end1 > start2) &&
408 (end2 == NFS4_MAX_UINT64 || end2 > start1);
409}
410
Fred Isaman4541d162011-01-06 11:36:23 +0000411static bool
Benny Halevy778b5502011-05-22 19:48:02 +0300412should_free_lseg(struct pnfs_layout_range *lseg_range,
413 struct pnfs_layout_range *recall_range)
Fred Isaman4541d162011-01-06 11:36:23 +0000414{
Benny Halevy778b5502011-05-22 19:48:02 +0300415 return (recall_range->iomode == IOMODE_ANY ||
416 lseg_range->iomode == recall_range->iomode) &&
417 lo_seg_intersecting(lseg_range, recall_range);
Fred Isaman4541d162011-01-06 11:36:23 +0000418}
419
420/* Returns 1 if lseg is removed from list, 0 otherwise */
421static int mark_lseg_invalid(struct pnfs_layout_segment *lseg,
422 struct list_head *tmp_list)
423{
424 int rv = 0;
425
426 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
427 /* Remove the reference keeping the lseg in the
428 * list. It will now be removed when all
429 * outstanding io is finished.
430 */
Fred Isamand684d2a2011-03-01 01:34:13 +0000431 dprintk("%s: lseg %p ref %d\n", __func__, lseg,
432 atomic_read(&lseg->pls_refcount));
433 if (atomic_dec_and_test(&lseg->pls_refcount)) {
Trond Myklebust57036a32012-09-20 16:33:30 -0400434 pnfs_layout_remove_lseg(lseg->pls_layout, lseg);
Fred Isamand684d2a2011-03-01 01:34:13 +0000435 list_add(&lseg->pls_list, tmp_list);
436 rv = 1;
437 }
Fred Isaman4541d162011-01-06 11:36:23 +0000438 }
439 return rv;
440}
441
442/* Returns count of number of matching invalid lsegs remaining in list
443 * after call.
444 */
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000445int
Trond Myklebust49a85062012-09-18 20:43:31 -0400446pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
Fred Isaman4541d162011-01-06 11:36:23 +0000447 struct list_head *tmp_list,
Benny Halevy778b5502011-05-22 19:48:02 +0300448 struct pnfs_layout_range *recall_range)
Andy Adamson974cec82010-10-20 00:18:02 -0400449{
450 struct pnfs_layout_segment *lseg, *next;
Fred Isaman4541d162011-01-06 11:36:23 +0000451 int invalid = 0, removed = 0;
Andy Adamson974cec82010-10-20 00:18:02 -0400452
453 dprintk("%s:Begin lo %p\n", __func__, lo);
454
Trond Myklebust8006bfb2012-09-21 14:48:04 -0400455 if (list_empty(&lo->plh_segs))
Fred Isaman38511722011-02-03 18:28:50 +0000456 return 0;
Fred Isaman4541d162011-01-06 11:36:23 +0000457 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
Benny Halevy778b5502011-05-22 19:48:02 +0300458 if (!recall_range ||
459 should_free_lseg(&lseg->pls_range, recall_range)) {
Fred Isaman4541d162011-01-06 11:36:23 +0000460 dprintk("%s: freeing lseg %p iomode %d "
461 "offset %llu length %llu\n", __func__,
462 lseg, lseg->pls_range.iomode, lseg->pls_range.offset,
463 lseg->pls_range.length);
464 invalid++;
465 removed += mark_lseg_invalid(lseg, tmp_list);
466 }
467 dprintk("%s:Return %i\n", __func__, invalid - removed);
468 return invalid - removed;
Andy Adamson974cec82010-10-20 00:18:02 -0400469}
470
Fred Isamanf49f9ba2011-02-03 18:28:52 +0000471/* note free_me must contain lsegs from a single layout_hdr */
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000472void
Fred Isaman4541d162011-01-06 11:36:23 +0000473pnfs_free_lseg_list(struct list_head *free_me)
Andy Adamson974cec82010-10-20 00:18:02 -0400474{
Fred Isaman4541d162011-01-06 11:36:23 +0000475 struct pnfs_layout_segment *lseg, *tmp;
Andy Adamson974cec82010-10-20 00:18:02 -0400476
Fred Isamanf49f9ba2011-02-03 18:28:52 +0000477 if (list_empty(free_me))
478 return;
479
Fred Isaman4541d162011-01-06 11:36:23 +0000480 list_for_each_entry_safe(lseg, tmp, free_me, pls_list) {
Fred Isaman566052c2011-01-06 11:36:20 +0000481 list_del(&lseg->pls_list);
Trond Myklebust905ca192012-09-20 20:46:49 -0400482 pnfs_free_lseg(lseg);
Andy Adamson974cec82010-10-20 00:18:02 -0400483 }
484}
485
Benny Halevye5e94012010-10-20 00:18:01 -0400486void
487pnfs_destroy_layout(struct nfs_inode *nfsi)
488{
489 struct pnfs_layout_hdr *lo;
Andy Adamson974cec82010-10-20 00:18:02 -0400490 LIST_HEAD(tmp_list);
Benny Halevye5e94012010-10-20 00:18:01 -0400491
492 spin_lock(&nfsi->vfs_inode.i_lock);
493 lo = nfsi->layout;
494 if (lo) {
Fred Isaman38511722011-02-03 18:28:50 +0000495 lo->plh_block_lgets++; /* permanently block new LAYOUTGETs */
Trond Myklebust49a85062012-09-18 20:43:31 -0400496 pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL);
Trond Myklebust3e621212012-09-24 13:07:16 -0400497 pnfs_get_layout_hdr(lo);
498 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED);
499 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED);
500 spin_unlock(&nfsi->vfs_inode.i_lock);
501 pnfs_free_lseg_list(&tmp_list);
502 pnfs_put_layout_hdr(lo);
503 } else
504 spin_unlock(&nfsi->vfs_inode.i_lock);
Benny Halevye5e94012010-10-20 00:18:01 -0400505}
Andy Adamson041245c2012-04-27 17:53:53 -0400506EXPORT_SYMBOL_GPL(pnfs_destroy_layout);
Benny Halevye5e94012010-10-20 00:18:01 -0400507
Andy Adamson974cec82010-10-20 00:18:02 -0400508/*
509 * Called by the state manger to remove all layouts established under an
510 * expired lease.
511 */
512void
513pnfs_destroy_all_layouts(struct nfs_client *clp)
514{
Weston Andros Adamson6382a442011-06-01 16:44:44 -0400515 struct nfs_server *server;
Andy Adamson974cec82010-10-20 00:18:02 -0400516 struct pnfs_layout_hdr *lo;
517 LIST_HEAD(tmp_list);
518
Andy Adamsonc47abcf2011-06-15 17:52:40 -0400519 nfs4_deviceid_mark_client_invalid(clp);
520 nfs4_deviceid_purge_client(clp);
521
Andy Adamson974cec82010-10-20 00:18:02 -0400522 spin_lock(&clp->cl_lock);
Weston Andros Adamson6382a442011-06-01 16:44:44 -0400523 rcu_read_lock();
524 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
525 if (!list_empty(&server->layouts))
526 list_splice_init(&server->layouts, &tmp_list);
527 }
528 rcu_read_unlock();
Andy Adamson974cec82010-10-20 00:18:02 -0400529 spin_unlock(&clp->cl_lock);
530
531 while (!list_empty(&tmp_list)) {
532 lo = list_entry(tmp_list.next, struct pnfs_layout_hdr,
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000533 plh_layouts);
Andy Adamson974cec82010-10-20 00:18:02 -0400534 dprintk("%s freeing layout for inode %lu\n", __func__,
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000535 lo->plh_inode->i_ino);
Andy Adamson2887fe42011-05-11 01:19:58 -0400536 list_del_init(&lo->plh_layouts);
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000537 pnfs_destroy_layout(NFS_I(lo->plh_inode));
Andy Adamson974cec82010-10-20 00:18:02 -0400538 }
539}
540
Trond Myklebust5a655032012-10-02 16:47:14 -0700541/*
542 * Compare 2 layout stateid sequence ids, to see which is newer,
543 * taking into account wraparound issues.
544 */
545static bool pnfs_seqid_is_newer(u32 s1, u32 s2)
546{
547 return (s32)s1 - (s32)s2 > 0;
548}
549
Fred Isamanfd6002e2011-01-06 11:36:22 +0000550/* update lo->plh_stateid with new if is more recent */
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000551void
552pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new,
553 bool update_barrier)
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400554{
Trond Myklebust22aaf712012-10-04 16:32:22 -0700555 u32 oldseq, newseq, new_barrier;
556 int empty = list_empty(&lo->plh_segs);
Andy Adamson974cec82010-10-20 00:18:02 -0400557
Trond Myklebust2d2f24a2012-03-04 18:13:57 -0500558 oldseq = be32_to_cpu(lo->plh_stateid.seqid);
559 newseq = be32_to_cpu(new->seqid);
Trond Myklebust22aaf712012-10-04 16:32:22 -0700560 if (empty || pnfs_seqid_is_newer(newseq, oldseq)) {
Trond Myklebustf597c532012-03-04 18:13:56 -0500561 nfs4_stateid_copy(&lo->plh_stateid, new);
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000562 if (update_barrier) {
Trond Myklebust22aaf712012-10-04 16:32:22 -0700563 new_barrier = be32_to_cpu(new->seqid);
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000564 } else {
565 /* Because of wraparound, we want to keep the barrier
Trond Myklebust22aaf712012-10-04 16:32:22 -0700566 * "close" to the current seqids.
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000567 */
Trond Myklebust22aaf712012-10-04 16:32:22 -0700568 new_barrier = newseq - atomic_read(&lo->plh_outstanding);
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000569 }
Trond Myklebust22aaf712012-10-04 16:32:22 -0700570 if (empty || pnfs_seqid_is_newer(new_barrier, lo->plh_barrier))
571 lo->plh_barrier = new_barrier;
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000572 }
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400573}
574
Trond Myklebust19c54ab2012-10-05 16:56:58 -0700575static bool
576pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo,
577 const nfs4_stateid *stateid)
578{
579 u32 seqid = be32_to_cpu(stateid->seqid);
580
581 return !pnfs_seqid_is_newer(seqid, lo->plh_barrier);
582}
583
Fred Isamancf7d63f2011-01-06 11:36:25 +0000584/* lget is set to 1 if called from inside send_layoutget call chain */
585static bool
Trond Myklebust19c54ab2012-10-05 16:56:58 -0700586pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo, int lget)
Fred Isamancf7d63f2011-01-06 11:36:25 +0000587{
Fred Isamanf7e89172011-01-06 11:36:32 +0000588 return lo->plh_block_lgets ||
589 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags) ||
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000590 (list_empty(&lo->plh_segs) &&
Fred Isamancf7d63f2011-01-06 11:36:25 +0000591 (atomic_read(&lo->plh_outstanding) > lget));
592}
593
Fred Isamanfd6002e2011-01-06 11:36:22 +0000594int
595pnfs_choose_layoutget_stateid(nfs4_stateid *dst, struct pnfs_layout_hdr *lo,
596 struct nfs4_state *open_state)
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400597{
Fred Isamanfd6002e2011-01-06 11:36:22 +0000598 int status = 0;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400599
600 dprintk("--> %s\n", __func__);
Fred Isamanfd6002e2011-01-06 11:36:22 +0000601 spin_lock(&lo->plh_inode->i_lock);
Trond Myklebust19c54ab2012-10-05 16:56:58 -0700602 if (pnfs_layoutgets_blocked(lo, 1)) {
Fred Isamancf7d63f2011-01-06 11:36:25 +0000603 status = -EAGAIN;
604 } else if (list_empty(&lo->plh_segs)) {
Fred Isamanfd6002e2011-01-06 11:36:22 +0000605 int seq;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400606
Fred Isamanfd6002e2011-01-06 11:36:22 +0000607 do {
608 seq = read_seqbegin(&open_state->seqlock);
Trond Myklebustf597c532012-03-04 18:13:56 -0500609 nfs4_stateid_copy(dst, &open_state->stateid);
Fred Isamanfd6002e2011-01-06 11:36:22 +0000610 } while (read_seqretry(&open_state->seqlock, seq));
611 } else
Trond Myklebustf597c532012-03-04 18:13:56 -0500612 nfs4_stateid_copy(dst, &lo->plh_stateid);
Fred Isamanfd6002e2011-01-06 11:36:22 +0000613 spin_unlock(&lo->plh_inode->i_lock);
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400614 dprintk("<-- %s\n", __func__);
Fred Isamanfd6002e2011-01-06 11:36:22 +0000615 return status;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400616}
617
618/*
619* Get layout from server.
620* for now, assume that whole file layouts are requested.
621* arg->offset: 0
622* arg->length: all ones
623*/
Benny Halevye5e94012010-10-20 00:18:01 -0400624static struct pnfs_layout_segment *
625send_layoutget(struct pnfs_layout_hdr *lo,
626 struct nfs_open_context *ctx,
Benny Halevyfb3296e2011-05-22 19:47:26 +0300627 struct pnfs_layout_range *range,
Trond Myklebusta75b9df2011-05-11 18:00:51 -0400628 gfp_t gfp_flags)
Benny Halevye5e94012010-10-20 00:18:01 -0400629{
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000630 struct inode *ino = lo->plh_inode;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400631 struct nfs_server *server = NFS_SERVER(ino);
632 struct nfs4_layoutget *lgp;
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -0400633 struct pnfs_layout_segment *lseg;
Benny Halevye5e94012010-10-20 00:18:01 -0400634
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400635 dprintk("--> %s\n", __func__);
636
637 BUG_ON(ctx == NULL);
Trond Myklebusta75b9df2011-05-11 18:00:51 -0400638 lgp = kzalloc(sizeof(*lgp), gfp_flags);
Fred Isamancf7d63f2011-01-06 11:36:25 +0000639 if (lgp == NULL)
Andy Adamson974cec82010-10-20 00:18:02 -0400640 return NULL;
Weston Andros Adamson35124a02011-03-24 16:48:21 -0400641
Benny Halevyfb3296e2011-05-22 19:47:26 +0300642 lgp->args.minlength = PAGE_CACHE_SIZE;
643 if (lgp->args.minlength > range->length)
644 lgp->args.minlength = range->length;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400645 lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE;
Benny Halevyfb3296e2011-05-22 19:47:26 +0300646 lgp->args.range = *range;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400647 lgp->args.type = server->pnfs_curr_ld->id;
648 lgp->args.inode = ino;
649 lgp->args.ctx = get_nfs_open_context(ctx);
Trond Myklebusta75b9df2011-05-11 18:00:51 -0400650 lgp->gfp_flags = gfp_flags;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400651
652 /* Synchronously retrieve layout information from server and
653 * store in lseg.
654 */
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -0400655 lseg = nfs4_proc_layoutget(lgp, gfp_flags);
656 if (IS_ERR(lseg)) {
657 switch (PTR_ERR(lseg)) {
658 case -ENOMEM:
659 case -ERESTARTSYS:
660 break;
661 default:
662 /* remember that LAYOUTGET failed and suspend trying */
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400663 pnfs_layout_io_set_failed(lo, range->iomode);
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -0400664 }
665 return NULL;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400666 }
Weston Andros Adamson35124a02011-03-24 16:48:21 -0400667
Andy Adamson974cec82010-10-20 00:18:02 -0400668 return lseg;
669}
670
Andy Adamson293b3b02012-06-20 15:03:34 -0400671/*
672 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr
673 * when the layout segment list is empty.
674 *
675 * Note that a pnfs_layout_hdr can exist with an empty layout segment
676 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the
677 * deviceid is marked invalid.
678 */
Benny Halevycbe82602011-05-22 19:52:37 +0300679int
680_pnfs_return_layout(struct inode *ino)
681{
682 struct pnfs_layout_hdr *lo = NULL;
683 struct nfs_inode *nfsi = NFS_I(ino);
684 LIST_HEAD(tmp_list);
685 struct nfs4_layoutreturn *lrp;
686 nfs4_stateid stateid;
Andy Adamson293b3b02012-06-20 15:03:34 -0400687 int status = 0, empty;
Benny Halevycbe82602011-05-22 19:52:37 +0300688
Andy Adamson366d5052012-06-20 15:03:33 -0400689 dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino);
Benny Halevycbe82602011-05-22 19:52:37 +0300690
691 spin_lock(&ino->i_lock);
692 lo = nfsi->layout;
Trond Myklebuste5929f32012-09-21 16:37:02 -0400693 if (!lo) {
Benny Halevycbe82602011-05-22 19:52:37 +0300694 spin_unlock(&ino->i_lock);
Andy Adamson293b3b02012-06-20 15:03:34 -0400695 dprintk("NFS: %s no layout to return\n", __func__);
696 goto out;
Benny Halevycbe82602011-05-22 19:52:37 +0300697 }
698 stateid = nfsi->layout->plh_stateid;
699 /* Reference matched in nfs4_layoutreturn_release */
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400700 pnfs_get_layout_hdr(lo);
Andy Adamson293b3b02012-06-20 15:03:34 -0400701 empty = list_empty(&lo->plh_segs);
Trond Myklebust49a85062012-09-18 20:43:31 -0400702 pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL);
Andy Adamson293b3b02012-06-20 15:03:34 -0400703 /* Don't send a LAYOUTRETURN if list was initially empty */
704 if (empty) {
705 spin_unlock(&ino->i_lock);
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400706 pnfs_put_layout_hdr(lo);
Andy Adamson293b3b02012-06-20 15:03:34 -0400707 dprintk("NFS: %s no layout segments to return\n", __func__);
708 goto out;
709 }
Fred Isamanea0ded72011-06-15 12:31:02 -0400710 lo->plh_block_lgets++;
Benny Halevycbe82602011-05-22 19:52:37 +0300711 spin_unlock(&ino->i_lock);
712 pnfs_free_lseg_list(&tmp_list);
713
714 WARN_ON(test_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags));
715
716 lrp = kzalloc(sizeof(*lrp), GFP_KERNEL);
717 if (unlikely(lrp == NULL)) {
718 status = -ENOMEM;
Trond Myklebust65857d52012-09-24 13:49:27 -0400719 spin_lock(&ino->i_lock);
720 lo->plh_block_lgets--;
721 spin_unlock(&ino->i_lock);
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400722 pnfs_put_layout_hdr(lo);
Benny Halevycbe82602011-05-22 19:52:37 +0300723 goto out;
724 }
725
726 lrp->args.stateid = stateid;
727 lrp->args.layout_type = NFS_SERVER(ino)->pnfs_curr_ld->id;
728 lrp->args.inode = ino;
Trond Myklebusta56aaa02011-06-15 11:59:10 -0400729 lrp->args.layout = lo;
Benny Halevycbe82602011-05-22 19:52:37 +0300730 lrp->clp = NFS_SERVER(ino)->nfs_client;
731
732 status = nfs4_proc_layoutreturn(lrp);
733out:
734 dprintk("<-- %s status: %d\n", __func__, status);
735 return status;
736}
Andy Adamson0a57cda2012-04-27 17:53:50 -0400737EXPORT_SYMBOL_GPL(_pnfs_return_layout);
Benny Halevycbe82602011-05-22 19:52:37 +0300738
Fred Isamanf7e89172011-01-06 11:36:32 +0000739bool pnfs_roc(struct inode *ino)
740{
741 struct pnfs_layout_hdr *lo;
742 struct pnfs_layout_segment *lseg, *tmp;
743 LIST_HEAD(tmp_list);
744 bool found = false;
745
746 spin_lock(&ino->i_lock);
747 lo = NFS_I(ino)->layout;
748 if (!lo || !test_and_clear_bit(NFS_LAYOUT_ROC, &lo->plh_flags) ||
749 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags))
750 goto out_nolayout;
751 list_for_each_entry_safe(lseg, tmp, &lo->plh_segs, pls_list)
752 if (test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) {
753 mark_lseg_invalid(lseg, &tmp_list);
754 found = true;
755 }
756 if (!found)
757 goto out_nolayout;
758 lo->plh_block_lgets++;
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400759 pnfs_get_layout_hdr(lo); /* matched in pnfs_roc_release */
Fred Isamanf7e89172011-01-06 11:36:32 +0000760 spin_unlock(&ino->i_lock);
761 pnfs_free_lseg_list(&tmp_list);
762 return true;
763
764out_nolayout:
765 spin_unlock(&ino->i_lock);
766 return false;
767}
768
769void pnfs_roc_release(struct inode *ino)
770{
771 struct pnfs_layout_hdr *lo;
772
773 spin_lock(&ino->i_lock);
774 lo = NFS_I(ino)->layout;
775 lo->plh_block_lgets--;
Trond Myklebust6622c3e2012-09-20 17:23:11 -0400776 if (atomic_dec_and_test(&lo->plh_refcount)) {
777 pnfs_detach_layout_hdr(lo);
778 spin_unlock(&ino->i_lock);
779 pnfs_free_layout_hdr(lo);
780 } else
781 spin_unlock(&ino->i_lock);
Fred Isamanf7e89172011-01-06 11:36:32 +0000782}
783
784void pnfs_roc_set_barrier(struct inode *ino, u32 barrier)
785{
786 struct pnfs_layout_hdr *lo;
787
788 spin_lock(&ino->i_lock);
789 lo = NFS_I(ino)->layout;
Trond Myklebust0f35ad62012-10-04 16:28:17 -0700790 if (pnfs_seqid_is_newer(barrier, lo->plh_barrier))
Fred Isamanf7e89172011-01-06 11:36:32 +0000791 lo->plh_barrier = barrier;
792 spin_unlock(&ino->i_lock);
793}
794
Trond Myklebust7fdab062012-09-20 20:15:57 -0400795bool pnfs_roc_drain(struct inode *ino, u32 *barrier, struct rpc_task *task)
Fred Isamanf7e89172011-01-06 11:36:32 +0000796{
797 struct nfs_inode *nfsi = NFS_I(ino);
Trond Myklebust7fdab062012-09-20 20:15:57 -0400798 struct pnfs_layout_hdr *lo;
Fred Isamanf7e89172011-01-06 11:36:32 +0000799 struct pnfs_layout_segment *lseg;
Trond Myklebust7fdab062012-09-20 20:15:57 -0400800 u32 current_seqid;
Fred Isamanf7e89172011-01-06 11:36:32 +0000801 bool found = false;
802
803 spin_lock(&ino->i_lock);
804 list_for_each_entry(lseg, &nfsi->layout->plh_segs, pls_list)
805 if (test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) {
Trond Myklebust7fdab062012-09-20 20:15:57 -0400806 rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL);
Fred Isamanf7e89172011-01-06 11:36:32 +0000807 found = true;
Trond Myklebust7fdab062012-09-20 20:15:57 -0400808 goto out;
Fred Isamanf7e89172011-01-06 11:36:32 +0000809 }
Trond Myklebust7fdab062012-09-20 20:15:57 -0400810 lo = nfsi->layout;
811 current_seqid = be32_to_cpu(lo->plh_stateid.seqid);
Fred Isamanf7e89172011-01-06 11:36:32 +0000812
Trond Myklebust7fdab062012-09-20 20:15:57 -0400813 /* Since close does not return a layout stateid for use as
814 * a barrier, we choose the worst-case barrier.
815 */
816 *barrier = current_seqid + atomic_read(&lo->plh_outstanding);
817out:
Fred Isamanf7e89172011-01-06 11:36:32 +0000818 spin_unlock(&ino->i_lock);
819 return found;
820}
821
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400822/*
823 * Compare two layout segments for sorting into layout cache.
824 * We want to preferentially return RW over RO layouts, so ensure those
825 * are seen first.
826 */
827static s64
Benny Halevyfb3296e2011-05-22 19:47:26 +0300828cmp_layout(struct pnfs_layout_range *l1,
829 struct pnfs_layout_range *l2)
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400830{
Benny Halevyfb3296e2011-05-22 19:47:26 +0300831 s64 d;
832
833 /* high offset > low offset */
834 d = l1->offset - l2->offset;
835 if (d)
836 return d;
837
838 /* short length > long length */
839 d = l2->length - l1->length;
840 if (d)
841 return d;
842
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400843 /* read > read/write */
Benny Halevyfb3296e2011-05-22 19:47:26 +0300844 return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ);
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400845}
846
Andy Adamson974cec82010-10-20 00:18:02 -0400847static void
Trond Myklebust57036a32012-09-20 16:33:30 -0400848pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo,
Andy Adamson974cec82010-10-20 00:18:02 -0400849 struct pnfs_layout_segment *lseg)
850{
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400851 struct pnfs_layout_segment *lp;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400852
Andy Adamson974cec82010-10-20 00:18:02 -0400853 dprintk("%s:Begin\n", __func__);
854
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000855 list_for_each_entry(lp, &lo->plh_segs, pls_list) {
Benny Halevyfb3296e2011-05-22 19:47:26 +0300856 if (cmp_layout(&lseg->pls_range, &lp->pls_range) > 0)
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400857 continue;
Fred Isaman566052c2011-01-06 11:36:20 +0000858 list_add_tail(&lseg->pls_list, &lp->pls_list);
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400859 dprintk("%s: inserted lseg %p "
860 "iomode %d offset %llu length %llu before "
861 "lp %p iomode %d offset %llu length %llu\n",
Fred Isaman566052c2011-01-06 11:36:20 +0000862 __func__, lseg, lseg->pls_range.iomode,
863 lseg->pls_range.offset, lseg->pls_range.length,
864 lp, lp->pls_range.iomode, lp->pls_range.offset,
865 lp->pls_range.length);
Benny Halevyfb3296e2011-05-22 19:47:26 +0300866 goto out;
Andy Adamson974cec82010-10-20 00:18:02 -0400867 }
Benny Halevyfb3296e2011-05-22 19:47:26 +0300868 list_add_tail(&lseg->pls_list, &lo->plh_segs);
869 dprintk("%s: inserted lseg %p "
870 "iomode %d offset %llu length %llu at tail\n",
871 __func__, lseg, lseg->pls_range.iomode,
872 lseg->pls_range.offset, lseg->pls_range.length);
873out:
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400874 pnfs_get_layout_hdr(lo);
Andy Adamson974cec82010-10-20 00:18:02 -0400875
876 dprintk("%s:Return\n", __func__);
Benny Halevye5e94012010-10-20 00:18:01 -0400877}
878
879static struct pnfs_layout_hdr *
Peng Tao9fa40752011-07-30 20:52:32 -0400880alloc_init_layout_hdr(struct inode *ino,
881 struct nfs_open_context *ctx,
882 gfp_t gfp_flags)
Benny Halevye5e94012010-10-20 00:18:01 -0400883{
884 struct pnfs_layout_hdr *lo;
885
Benny Halevy636fb9c2011-05-22 19:51:33 +0300886 lo = pnfs_alloc_layout_hdr(ino, gfp_flags);
Benny Halevye5e94012010-10-20 00:18:01 -0400887 if (!lo)
888 return NULL;
Fred Isamancc6e5342011-01-06 11:36:28 +0000889 atomic_set(&lo->plh_refcount, 1);
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000890 INIT_LIST_HEAD(&lo->plh_layouts);
891 INIT_LIST_HEAD(&lo->plh_segs);
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000892 INIT_LIST_HEAD(&lo->plh_bulk_recall);
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000893 lo->plh_inode = ino;
Peng Tao9fa40752011-07-30 20:52:32 -0400894 lo->plh_lc_cred = get_rpccred(ctx->state->owner->so_cred);
Benny Halevye5e94012010-10-20 00:18:01 -0400895 return lo;
896}
897
898static struct pnfs_layout_hdr *
Peng Tao9fa40752011-07-30 20:52:32 -0400899pnfs_find_alloc_layout(struct inode *ino,
900 struct nfs_open_context *ctx,
901 gfp_t gfp_flags)
Benny Halevye5e94012010-10-20 00:18:01 -0400902{
903 struct nfs_inode *nfsi = NFS_I(ino);
904 struct pnfs_layout_hdr *new = NULL;
905
906 dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout);
907
Trond Myklebust251ec412012-10-02 15:41:05 -0700908 if (nfsi->layout != NULL)
909 goto out_existing;
Benny Halevye5e94012010-10-20 00:18:01 -0400910 spin_unlock(&ino->i_lock);
Peng Tao9fa40752011-07-30 20:52:32 -0400911 new = alloc_init_layout_hdr(ino, ctx, gfp_flags);
Benny Halevye5e94012010-10-20 00:18:01 -0400912 spin_lock(&ino->i_lock);
913
Trond Myklebust251ec412012-10-02 15:41:05 -0700914 if (likely(nfsi->layout == NULL)) { /* Won the race? */
Benny Halevye5e94012010-10-20 00:18:01 -0400915 nfsi->layout = new;
Trond Myklebust251ec412012-10-02 15:41:05 -0700916 return new;
Yanchuan Nian7175fe92012-10-31 16:05:48 +0800917 } else if (new != NULL)
918 pnfs_free_layout_hdr(new);
Trond Myklebust251ec412012-10-02 15:41:05 -0700919out_existing:
920 pnfs_get_layout_hdr(nfsi->layout);
Benny Halevye5e94012010-10-20 00:18:01 -0400921 return nfsi->layout;
922}
923
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400924/*
925 * iomode matching rules:
926 * iomode lseg match
927 * ----- ----- -----
928 * ANY READ true
929 * ANY RW true
930 * RW READ false
931 * RW RW true
932 * READ READ true
933 * READ RW true
934 */
935static int
Benny Halevyfb3296e2011-05-22 19:47:26 +0300936is_matching_lseg(struct pnfs_layout_range *ls_range,
937 struct pnfs_layout_range *range)
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400938{
Benny Halevyfb3296e2011-05-22 19:47:26 +0300939 struct pnfs_layout_range range1;
940
941 if ((range->iomode == IOMODE_RW &&
942 ls_range->iomode != IOMODE_RW) ||
943 !lo_seg_intersecting(ls_range, range))
944 return 0;
945
946 /* range1 covers only the first byte in the range */
947 range1 = *range;
948 range1.length = 1;
949 return lo_seg_contained(ls_range, &range1);
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400950}
951
952/*
953 * lookup range in layout
954 */
Benny Halevye5e94012010-10-20 00:18:01 -0400955static struct pnfs_layout_segment *
Benny Halevyfb3296e2011-05-22 19:47:26 +0300956pnfs_find_lseg(struct pnfs_layout_hdr *lo,
957 struct pnfs_layout_range *range)
Benny Halevye5e94012010-10-20 00:18:01 -0400958{
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400959 struct pnfs_layout_segment *lseg, *ret = NULL;
960
961 dprintk("%s:Begin\n", __func__);
962
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000963 list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
Fred Isaman4541d162011-01-06 11:36:23 +0000964 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) &&
Benny Halevyfb3296e2011-05-22 19:47:26 +0300965 is_matching_lseg(&lseg->pls_range, range)) {
Trond Myklebust9369a432012-09-18 20:57:08 -0400966 ret = pnfs_get_lseg(lseg);
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400967 break;
968 }
Benny Halevyd771e3a2011-06-14 16:30:16 -0400969 if (lseg->pls_range.offset > range->offset)
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400970 break;
971 }
972
973 dprintk("%s:Return lseg %p ref %d\n",
Fred Isaman4541d162011-01-06 11:36:23 +0000974 __func__, ret, ret ? atomic_read(&ret->pls_refcount) : 0);
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400975 return ret;
Benny Halevye5e94012010-10-20 00:18:01 -0400976}
977
978/*
Andy Adamsond23d61c2012-05-23 05:02:37 -0400979 * Use mdsthreshold hints set at each OPEN to determine if I/O should go
980 * to the MDS or over pNFS
981 *
982 * The nfs_inode read_io and write_io fields are cumulative counters reset
983 * when there are no layout segments. Note that in pnfs_update_layout iomode
984 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a
985 * WRITE request.
986 *
987 * A return of true means use MDS I/O.
988 *
989 * From rfc 5661:
990 * If a file's size is smaller than the file size threshold, data accesses
991 * SHOULD be sent to the metadata server. If an I/O request has a length that
992 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata
993 * server. If both file size and I/O size are provided, the client SHOULD
994 * reach or exceed both thresholds before sending its read or write
995 * requests to the data server.
996 */
997static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx,
998 struct inode *ino, int iomode)
999{
1000 struct nfs4_threshold *t = ctx->mdsthreshold;
1001 struct nfs_inode *nfsi = NFS_I(ino);
1002 loff_t fsize = i_size_read(ino);
1003 bool size = false, size_set = false, io = false, io_set = false, ret = false;
1004
1005 if (t == NULL)
1006 return ret;
1007
1008 dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n",
1009 __func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz);
1010
1011 switch (iomode) {
1012 case IOMODE_READ:
1013 if (t->bm & THRESHOLD_RD) {
1014 dprintk("%s fsize %llu\n", __func__, fsize);
1015 size_set = true;
1016 if (fsize < t->rd_sz)
1017 size = true;
1018 }
1019 if (t->bm & THRESHOLD_RD_IO) {
1020 dprintk("%s nfsi->read_io %llu\n", __func__,
1021 nfsi->read_io);
1022 io_set = true;
1023 if (nfsi->read_io < t->rd_io_sz)
1024 io = true;
1025 }
1026 break;
1027 case IOMODE_RW:
1028 if (t->bm & THRESHOLD_WR) {
1029 dprintk("%s fsize %llu\n", __func__, fsize);
1030 size_set = true;
1031 if (fsize < t->wr_sz)
1032 size = true;
1033 }
1034 if (t->bm & THRESHOLD_WR_IO) {
1035 dprintk("%s nfsi->write_io %llu\n", __func__,
1036 nfsi->write_io);
1037 io_set = true;
1038 if (nfsi->write_io < t->wr_io_sz)
1039 io = true;
1040 }
1041 break;
1042 }
1043 if (size_set && io_set) {
1044 if (size && io)
1045 ret = true;
1046 } else if (size || io)
1047 ret = true;
1048
1049 dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret);
1050 return ret;
1051}
1052
1053/*
Benny Halevye5e94012010-10-20 00:18:01 -04001054 * Layout segment is retreived from the server if not cached.
1055 * The appropriate layout segment is referenced and returned to the caller.
1056 */
Andy Adamson7c24d942011-06-13 18:22:38 -04001057struct pnfs_layout_segment *
Benny Halevye5e94012010-10-20 00:18:01 -04001058pnfs_update_layout(struct inode *ino,
1059 struct nfs_open_context *ctx,
Benny Halevyfb3296e2011-05-22 19:47:26 +03001060 loff_t pos,
1061 u64 count,
Trond Myklebusta75b9df2011-05-11 18:00:51 -04001062 enum pnfs_iomode iomode,
1063 gfp_t gfp_flags)
Benny Halevye5e94012010-10-20 00:18:01 -04001064{
Benny Halevyfb3296e2011-05-22 19:47:26 +03001065 struct pnfs_layout_range arg = {
1066 .iomode = iomode,
1067 .offset = pos,
1068 .length = count,
1069 };
Benny Halevy707ed5f2011-05-22 19:47:46 +03001070 unsigned pg_offset;
Weston Andros Adamson6382a442011-06-01 16:44:44 -04001071 struct nfs_server *server = NFS_SERVER(ino);
1072 struct nfs_client *clp = server->nfs_client;
Benny Halevye5e94012010-10-20 00:18:01 -04001073 struct pnfs_layout_hdr *lo;
1074 struct pnfs_layout_segment *lseg = NULL;
Fred Isamanf49f9ba2011-02-03 18:28:52 +00001075 bool first = false;
Benny Halevye5e94012010-10-20 00:18:01 -04001076
1077 if (!pnfs_enabled_sb(NFS_SERVER(ino)))
Trond Myklebustf86bbcf2012-09-26 11:21:40 -04001078 goto out;
Andy Adamsond23d61c2012-05-23 05:02:37 -04001079
1080 if (pnfs_within_mdsthreshold(ctx, ino, iomode))
Trond Myklebustf86bbcf2012-09-26 11:21:40 -04001081 goto out;
Andy Adamsond23d61c2012-05-23 05:02:37 -04001082
Benny Halevye5e94012010-10-20 00:18:01 -04001083 spin_lock(&ino->i_lock);
Peng Tao9fa40752011-07-30 20:52:32 -04001084 lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags);
Trond Myklebust830ffb52012-09-20 21:25:19 -04001085 if (lo == NULL) {
1086 spin_unlock(&ino->i_lock);
1087 goto out;
1088 }
Benny Halevye5e94012010-10-20 00:18:01 -04001089
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001090 /* Do we even need to bother with this? */
Trond Myklebusta59c30a2012-03-01 11:17:47 -05001091 if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001092 dprintk("%s matches recall, use MDS\n", __func__);
Benny Halevye5e94012010-10-20 00:18:01 -04001093 goto out_unlock;
1094 }
1095
1096 /* if LAYOUTGET already failed once we don't try again */
Trond Myklebustb9e028f2012-09-18 16:41:18 -04001097 if (pnfs_layout_io_test_failed(lo, iomode))
Benny Halevye5e94012010-10-20 00:18:01 -04001098 goto out_unlock;
1099
Andy Adamson568e8c42011-03-01 01:34:22 +00001100 /* Check to see if the layout for the given range already exists */
Benny Halevyfb3296e2011-05-22 19:47:26 +03001101 lseg = pnfs_find_lseg(lo, &arg);
Andy Adamson568e8c42011-03-01 01:34:22 +00001102 if (lseg)
1103 goto out_unlock;
1104
Trond Myklebust19c54ab2012-10-05 16:56:58 -07001105 if (pnfs_layoutgets_blocked(lo, 0))
Fred Isamancf7d63f2011-01-06 11:36:25 +00001106 goto out_unlock;
1107 atomic_inc(&lo->plh_outstanding);
1108
Fred Isamanf49f9ba2011-02-03 18:28:52 +00001109 if (list_empty(&lo->plh_segs))
1110 first = true;
Andy Adamson366d5052012-06-20 15:03:33 -04001111
Fred Isamanf49f9ba2011-02-03 18:28:52 +00001112 spin_unlock(&ino->i_lock);
1113 if (first) {
Fred Isaman2130ff62011-01-06 11:36:26 +00001114 /* The lo must be on the clp list if there is any
1115 * chance of a CB_LAYOUTRECALL(FILE) coming in.
1116 */
1117 spin_lock(&clp->cl_lock);
1118 BUG_ON(!list_empty(&lo->plh_layouts));
Weston Andros Adamson6382a442011-06-01 16:44:44 -04001119 list_add_tail(&lo->plh_layouts, &server->layouts);
Fred Isaman2130ff62011-01-06 11:36:26 +00001120 spin_unlock(&clp->cl_lock);
1121 }
Benny Halevye5e94012010-10-20 00:18:01 -04001122
Benny Halevy707ed5f2011-05-22 19:47:46 +03001123 pg_offset = arg.offset & ~PAGE_CACHE_MASK;
1124 if (pg_offset) {
1125 arg.offset -= pg_offset;
1126 arg.length += pg_offset;
1127 }
Andy Adamson7c24d942011-06-13 18:22:38 -04001128 if (arg.length != NFS4_MAX_UINT64)
1129 arg.length = PAGE_CACHE_ALIGN(arg.length);
Benny Halevy707ed5f2011-05-22 19:47:46 +03001130
Benny Halevyfb3296e2011-05-22 19:47:26 +03001131 lseg = send_layoutget(lo, ctx, &arg, gfp_flags);
Fred Isamancf7d63f2011-01-06 11:36:25 +00001132 atomic_dec(&lo->plh_outstanding);
Trond Myklebust830ffb52012-09-20 21:25:19 -04001133out_put_layout_hdr:
Trond Myklebust70c3bd22012-09-18 20:51:13 -04001134 pnfs_put_layout_hdr(lo);
Benny Halevye5e94012010-10-20 00:18:01 -04001135out:
Trond Myklebustf86bbcf2012-09-26 11:21:40 -04001136 dprintk("%s: inode %s/%llu pNFS layout segment %s for "
1137 "(%s, offset: %llu, length: %llu)\n",
1138 __func__, ino->i_sb->s_id,
1139 (unsigned long long)NFS_FILEID(ino),
1140 lseg == NULL ? "not found" : "found",
1141 iomode==IOMODE_RW ? "read/write" : "read-only",
1142 (unsigned long long)pos,
1143 (unsigned long long)count);
Benny Halevye5e94012010-10-20 00:18:01 -04001144 return lseg;
1145out_unlock:
1146 spin_unlock(&ino->i_lock);
Trond Myklebust830ffb52012-09-20 21:25:19 -04001147 goto out_put_layout_hdr;
Benny Halevye5e94012010-10-20 00:18:01 -04001148}
Andy Adamson7c24d942011-06-13 18:22:38 -04001149EXPORT_SYMBOL_GPL(pnfs_update_layout);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001150
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -04001151struct pnfs_layout_segment *
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001152pnfs_layout_process(struct nfs4_layoutget *lgp)
1153{
1154 struct pnfs_layout_hdr *lo = NFS_I(lgp->args.inode)->layout;
1155 struct nfs4_layoutget_res *res = &lgp->res;
1156 struct pnfs_layout_segment *lseg;
Fred Isamanb7edfaa2011-01-06 11:36:21 +00001157 struct inode *ino = lo->plh_inode;
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001158 int status = 0;
1159
1160 /* Inject layout blob into I/O device driver */
Trond Myklebusta75b9df2011-05-11 18:00:51 -04001161 lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001162 if (!lseg || IS_ERR(lseg)) {
1163 if (!lseg)
1164 status = -ENOMEM;
1165 else
1166 status = PTR_ERR(lseg);
1167 dprintk("%s: Could not allocate layout: error %d\n",
1168 __func__, status);
1169 goto out;
1170 }
1171
1172 spin_lock(&ino->i_lock);
Trond Myklebusta59c30a2012-03-01 11:17:47 -05001173 if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001174 dprintk("%s forget reply due to recall\n", __func__);
1175 goto out_forget_reply;
1176 }
1177
Trond Myklebust19c54ab2012-10-05 16:56:58 -07001178 if (pnfs_layoutgets_blocked(lo, 1) ||
1179 pnfs_layout_stateid_blocked(lo, &res->stateid)) {
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001180 dprintk("%s forget reply due to state\n", __func__);
1181 goto out_forget_reply;
1182 }
Trond Myklebust038d6492012-10-02 16:38:41 -07001183
1184 /* Done processing layoutget. Set the layout stateid */
1185 pnfs_set_layout_stateid(lo, &res->stateid, false);
1186
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001187 init_lseg(lo, lseg);
Fred Isaman566052c2011-01-06 11:36:20 +00001188 lseg->pls_range = res->range;
Trond Myklebust9369a432012-09-18 20:57:08 -04001189 pnfs_get_lseg(lseg);
Trond Myklebust57036a32012-09-20 16:33:30 -04001190 pnfs_layout_insert_lseg(lo, lseg);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001191
Fred Isamanf7e89172011-01-06 11:36:32 +00001192 if (res->return_on_close) {
1193 set_bit(NFS_LSEG_ROC, &lseg->pls_flags);
1194 set_bit(NFS_LAYOUT_ROC, &lo->plh_flags);
1195 }
1196
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001197 spin_unlock(&ino->i_lock);
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -04001198 return lseg;
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001199out:
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -04001200 return ERR_PTR(status);
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001201
1202out_forget_reply:
1203 spin_unlock(&ino->i_lock);
1204 lseg->pls_layout = lo;
1205 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
1206 goto out;
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001207}
1208
Trond Myklebustd8007d42011-06-10 13:30:23 -04001209void
1210pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
1211{
Peng Tao1fd937b2012-09-25 14:55:57 +08001212 u64 rd_size = req->wb_bytes;
1213
Trond Myklebustd8007d42011-06-10 13:30:23 -04001214 BUG_ON(pgio->pg_lseg != NULL);
1215
Fred Isaman1825a0d2012-04-20 19:55:31 -04001216 if (req->wb_offset != req->wb_pgbase) {
1217 nfs_pageio_reset_read_mds(pgio);
1218 return;
1219 }
Peng Tao1fd937b2012-09-25 14:55:57 +08001220
1221 if (pgio->pg_dreq == NULL)
1222 rd_size = i_size_read(pgio->pg_inode) - req_offset(req);
1223 else
1224 rd_size = nfs_dreq_bytes_left(pgio->pg_dreq);
1225
Trond Myklebustd8007d42011-06-10 13:30:23 -04001226 pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
1227 req->wb_context,
1228 req_offset(req),
Peng Tao1fd937b2012-09-25 14:55:57 +08001229 rd_size,
Trond Myklebustd8007d42011-06-10 13:30:23 -04001230 IOMODE_READ,
1231 GFP_KERNEL);
Trond Myklebuste885de12011-06-10 13:30:23 -04001232 /* If no lseg, fall back to read through mds */
1233 if (pgio->pg_lseg == NULL)
Trond Myklebust1f945352011-07-13 15:59:57 -04001234 nfs_pageio_reset_read_mds(pgio);
Trond Myklebuste885de12011-06-10 13:30:23 -04001235
Trond Myklebustd8007d42011-06-10 13:30:23 -04001236}
1237EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read);
1238
1239void
Peng Tao62965562012-09-25 14:55:57 +08001240pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio,
1241 struct nfs_page *req, u64 wb_size)
Trond Myklebustd8007d42011-06-10 13:30:23 -04001242{
1243 BUG_ON(pgio->pg_lseg != NULL);
1244
Fred Isaman1825a0d2012-04-20 19:55:31 -04001245 if (req->wb_offset != req->wb_pgbase) {
1246 nfs_pageio_reset_write_mds(pgio);
1247 return;
1248 }
Peng Tao62965562012-09-25 14:55:57 +08001249
Trond Myklebustd8007d42011-06-10 13:30:23 -04001250 pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
1251 req->wb_context,
1252 req_offset(req),
Peng Tao62965562012-09-25 14:55:57 +08001253 wb_size,
Trond Myklebustd8007d42011-06-10 13:30:23 -04001254 IOMODE_RW,
1255 GFP_NOFS);
Trond Myklebuste885de12011-06-10 13:30:23 -04001256 /* If no lseg, fall back to write through mds */
1257 if (pgio->pg_lseg == NULL)
Trond Myklebust1f945352011-07-13 15:59:57 -04001258 nfs_pageio_reset_write_mds(pgio);
Trond Myklebustd8007d42011-06-10 13:30:23 -04001259}
1260EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write);
1261
Bryan Schumaker1abb50882012-06-20 15:53:47 -04001262void
Fred Isaman061ae2e2012-04-20 14:47:48 -04001263pnfs_pageio_init_read(struct nfs_pageio_descriptor *pgio, struct inode *inode,
1264 const struct nfs_pgio_completion_ops *compl_ops)
Trond Myklebust1751c362011-06-10 13:30:23 -04001265{
1266 struct nfs_server *server = NFS_SERVER(inode);
1267 struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld;
1268
1269 if (ld == NULL)
Bryan Schumaker1abb50882012-06-20 15:53:47 -04001270 nfs_pageio_init_read(pgio, inode, compl_ops);
1271 else
1272 nfs_pageio_init(pgio, inode, ld->pg_read_ops, compl_ops, server->rsize, 0);
Trond Myklebust1751c362011-06-10 13:30:23 -04001273}
1274
Bryan Schumaker57208fa2012-06-20 15:53:48 -04001275void
Fred Isaman061ae2e2012-04-20 14:47:48 -04001276pnfs_pageio_init_write(struct nfs_pageio_descriptor *pgio, struct inode *inode,
1277 int ioflags,
1278 const struct nfs_pgio_completion_ops *compl_ops)
Trond Myklebust1751c362011-06-10 13:30:23 -04001279{
1280 struct nfs_server *server = NFS_SERVER(inode);
1281 struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld;
1282
1283 if (ld == NULL)
Bryan Schumaker57208fa2012-06-20 15:53:48 -04001284 nfs_pageio_init_write(pgio, inode, ioflags, compl_ops);
1285 else
1286 nfs_pageio_init(pgio, inode, ld->pg_write_ops, compl_ops, server->wsize, ioflags);
Trond Myklebust1751c362011-06-10 13:30:23 -04001287}
1288
1289bool
Benny Halevydfed2062011-05-25 20:25:22 +03001290pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev,
1291 struct nfs_page *req)
Fred Isamanbae724e2011-03-01 01:34:15 +00001292{
Trond Myklebustd8007d42011-06-10 13:30:23 -04001293 if (pgio->pg_lseg == NULL)
1294 return nfs_generic_pg_test(pgio, prev, req);
Benny Halevy89a58e32011-05-25 20:54:40 +03001295
Trond Myklebust19982ba2011-06-10 13:30:23 -04001296 /*
1297 * Test if a nfs_page is fully contained in the pnfs_layout_range.
1298 * Note that this test makes several assumptions:
1299 * - that the previous nfs_page in the struct nfs_pageio_descriptor
1300 * is known to lie within the range.
1301 * - that the nfs_page being tested is known to be contiguous with the
1302 * previous nfs_page.
1303 * - Layout ranges are page aligned, so we only have to test the
1304 * start offset of the request.
1305 *
1306 * Please also note that 'end_offset' is actually the offset of the
1307 * first byte that lies outside the pnfs_layout_range. FIXME?
1308 *
1309 */
1310 return req_offset(req) < end_offset(pgio->pg_lseg->pls_range.offset,
1311 pgio->pg_lseg->pls_range.length);
Fred Isamanbae724e2011-03-01 01:34:15 +00001312}
Benny Halevy89a58e32011-05-25 20:54:40 +03001313EXPORT_SYMBOL_GPL(pnfs_generic_pg_test);
Fred Isamanbae724e2011-03-01 01:34:15 +00001314
Andy Adamsone7dd79af2012-04-27 17:53:46 -04001315int pnfs_write_done_resend_to_mds(struct inode *inode,
Fred Isaman061ae2e2012-04-20 14:47:48 -04001316 struct list_head *head,
1317 const struct nfs_pgio_completion_ops *compl_ops)
Trond Myklebuste2fecb22012-01-06 08:57:46 -05001318{
1319 struct nfs_pageio_descriptor pgio;
1320 LIST_HEAD(failed);
1321
1322 /* Resend all requests through the MDS */
Bryan Schumaker57208fa2012-06-20 15:53:48 -04001323 nfs_pageio_init_write(&pgio, inode, FLUSH_STABLE, compl_ops);
Trond Myklebuste2fecb22012-01-06 08:57:46 -05001324 while (!list_empty(head)) {
1325 struct nfs_page *req = nfs_list_entry(head->next);
1326
1327 nfs_list_remove_request(req);
1328 if (!nfs_pageio_add_request(&pgio, req))
1329 nfs_list_add_request(req, &failed);
1330 }
1331 nfs_pageio_complete(&pgio);
1332
1333 if (!list_empty(&failed)) {
1334 /* For some reason our attempt to resend pages. Mark the
1335 * overall send request as having failed, and let
1336 * nfs_writeback_release_full deal with the error.
1337 */
1338 list_move(&failed, head);
1339 return -EIO;
1340 }
1341 return 0;
1342}
Andy Adamsone7dd79af2012-04-27 17:53:46 -04001343EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds);
Trond Myklebuste2fecb22012-01-06 08:57:46 -05001344
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001345static void pnfs_ld_handle_write_error(struct nfs_write_data *data)
1346{
Fred Isamancd841602012-04-20 14:47:44 -04001347 struct nfs_pgio_header *hdr = data->header;
1348
1349 dprintk("pnfs write error = %d\n", hdr->pnfs_error);
1350 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001351 PNFS_LAYOUTRET_ON_ERROR) {
Fred Isamancd841602012-04-20 14:47:44 -04001352 clear_bit(NFS_INO_LAYOUTCOMMIT, &NFS_I(hdr->inode)->flags);
1353 pnfs_return_layout(hdr->inode);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001354 }
Fred Isaman6c75dc02012-04-20 14:47:47 -04001355 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
1356 data->task.tk_status = pnfs_write_done_resend_to_mds(hdr->inode,
Fred Isaman061ae2e2012-04-20 14:47:48 -04001357 &hdr->pages,
1358 hdr->completion_ops);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001359}
1360
Benny Halevyd20581a2011-05-22 19:52:03 +03001361/*
1362 * Called by non rpc-based layout drivers
1363 */
Peng Tao8ce160c2011-09-22 21:50:14 -04001364void pnfs_ld_write_done(struct nfs_write_data *data)
Fred Isaman94ad1c82011-03-01 01:34:14 +00001365{
Fred Isamancd841602012-04-20 14:47:44 -04001366 struct nfs_pgio_header *hdr = data->header;
1367
1368 if (!hdr->pnfs_error) {
Benny Halevyd20581a2011-05-22 19:52:03 +03001369 pnfs_set_layoutcommit(data);
Fred Isamancd841602012-04-20 14:47:44 -04001370 hdr->mds_ops->rpc_call_done(&data->task, data);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001371 } else
1372 pnfs_ld_handle_write_error(data);
Fred Isamancd841602012-04-20 14:47:44 -04001373 hdr->mds_ops->rpc_release(data);
Fred Isaman44b83792011-03-03 15:13:44 +00001374}
Benny Halevyd20581a2011-05-22 19:52:03 +03001375EXPORT_SYMBOL_GPL(pnfs_ld_write_done);
Fred Isaman44b83792011-03-03 15:13:44 +00001376
Trond Myklebustdce81292011-07-13 15:59:19 -04001377static void
1378pnfs_write_through_mds(struct nfs_pageio_descriptor *desc,
1379 struct nfs_write_data *data)
1380{
Fred Isamancd841602012-04-20 14:47:44 -04001381 struct nfs_pgio_header *hdr = data->header;
1382
Fred Isaman6c75dc02012-04-20 14:47:47 -04001383 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
1384 list_splice_tail_init(&hdr->pages, &desc->pg_list);
1385 nfs_pageio_reset_write_mds(desc);
1386 desc->pg_recoalesce = 1;
1387 }
Trond Myklebustdce81292011-07-13 15:59:19 -04001388 nfs_writedata_release(data);
1389}
1390
1391static enum pnfs_try_status
Andy Adamson0382b742011-03-03 15:13:45 +00001392pnfs_try_to_write_data(struct nfs_write_data *wdata,
Trond Myklebustdce81292011-07-13 15:59:19 -04001393 const struct rpc_call_ops *call_ops,
1394 struct pnfs_layout_segment *lseg,
1395 int how)
Andy Adamson0382b742011-03-03 15:13:45 +00001396{
Fred Isamancd841602012-04-20 14:47:44 -04001397 struct nfs_pgio_header *hdr = wdata->header;
1398 struct inode *inode = hdr->inode;
Andy Adamson0382b742011-03-03 15:13:45 +00001399 enum pnfs_try_status trypnfs;
1400 struct nfs_server *nfss = NFS_SERVER(inode);
1401
Fred Isamancd841602012-04-20 14:47:44 -04001402 hdr->mds_ops = call_ops;
Andy Adamson0382b742011-03-03 15:13:45 +00001403
1404 dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__,
1405 inode->i_ino, wdata->args.count, wdata->args.offset, how);
Andy Adamson0382b742011-03-03 15:13:45 +00001406 trypnfs = nfss->pnfs_curr_ld->write_pagelist(wdata, how);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001407 if (trypnfs != PNFS_NOT_ATTEMPTED)
Andy Adamson0382b742011-03-03 15:13:45 +00001408 nfs_inc_stats(inode, NFSIOS_PNFS_WRITE);
Andy Adamson0382b742011-03-03 15:13:45 +00001409 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
1410 return trypnfs;
1411}
1412
Trond Myklebustdce81292011-07-13 15:59:19 -04001413static void
1414pnfs_do_multiple_writes(struct nfs_pageio_descriptor *desc, struct list_head *head, int how)
1415{
1416 struct nfs_write_data *data;
1417 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
1418 struct pnfs_layout_segment *lseg = desc->pg_lseg;
1419
1420 desc->pg_lseg = NULL;
1421 while (!list_empty(head)) {
1422 enum pnfs_try_status trypnfs;
1423
Fred Isaman6c75dc02012-04-20 14:47:47 -04001424 data = list_first_entry(head, struct nfs_write_data, list);
Trond Myklebustdce81292011-07-13 15:59:19 -04001425 list_del_init(&data->list);
1426
1427 trypnfs = pnfs_try_to_write_data(data, call_ops, lseg, how);
1428 if (trypnfs == PNFS_NOT_ATTEMPTED)
1429 pnfs_write_through_mds(desc, data);
1430 }
Trond Myklebust9369a432012-09-18 20:57:08 -04001431 pnfs_put_lseg(lseg);
Trond Myklebustdce81292011-07-13 15:59:19 -04001432}
1433
Fred Isaman6c75dc02012-04-20 14:47:47 -04001434static void pnfs_writehdr_free(struct nfs_pgio_header *hdr)
1435{
Trond Myklebust9369a432012-09-18 20:57:08 -04001436 pnfs_put_lseg(hdr->lseg);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001437 nfs_writehdr_free(hdr);
1438}
Bryan Schumaker89d77c82012-07-30 16:05:25 -04001439EXPORT_SYMBOL_GPL(pnfs_writehdr_free);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001440
Trond Myklebustdce81292011-07-13 15:59:19 -04001441int
1442pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc)
1443{
Fred Isaman6c75dc02012-04-20 14:47:47 -04001444 struct nfs_write_header *whdr;
1445 struct nfs_pgio_header *hdr;
Trond Myklebustdce81292011-07-13 15:59:19 -04001446 int ret;
1447
Fred Isaman6c75dc02012-04-20 14:47:47 -04001448 whdr = nfs_writehdr_alloc();
1449 if (!whdr) {
Trond Myklebust9b5415b2012-04-27 14:31:47 -04001450 desc->pg_completion_ops->error_cleanup(&desc->pg_list);
Trond Myklebust9369a432012-09-18 20:57:08 -04001451 pnfs_put_lseg(desc->pg_lseg);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001452 desc->pg_lseg = NULL;
1453 return -ENOMEM;
1454 }
1455 hdr = &whdr->header;
1456 nfs_pgheader_init(desc, hdr, pnfs_writehdr_free);
Trond Myklebust9369a432012-09-18 20:57:08 -04001457 hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001458 atomic_inc(&hdr->refcnt);
1459 ret = nfs_generic_flush(desc, hdr);
Trond Myklebustdce81292011-07-13 15:59:19 -04001460 if (ret != 0) {
Trond Myklebust9369a432012-09-18 20:57:08 -04001461 pnfs_put_lseg(desc->pg_lseg);
Trond Myklebustdce81292011-07-13 15:59:19 -04001462 desc->pg_lseg = NULL;
Fred Isaman6c75dc02012-04-20 14:47:47 -04001463 } else
1464 pnfs_do_multiple_writes(desc, &hdr->rpc_list, desc->pg_ioflags);
1465 if (atomic_dec_and_test(&hdr->refcnt))
Fred Isaman061ae2e2012-04-20 14:47:48 -04001466 hdr->completion_ops->completion(hdr);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001467 return ret;
Trond Myklebustdce81292011-07-13 15:59:19 -04001468}
1469EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages);
1470
Andy Adamsone7dd79af2012-04-27 17:53:46 -04001471int pnfs_read_done_resend_to_mds(struct inode *inode,
Fred Isaman061ae2e2012-04-20 14:47:48 -04001472 struct list_head *head,
1473 const struct nfs_pgio_completion_ops *compl_ops)
Trond Myklebust62e4a762011-11-10 14:30:37 -05001474{
1475 struct nfs_pageio_descriptor pgio;
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001476 LIST_HEAD(failed);
Trond Myklebust62e4a762011-11-10 14:30:37 -05001477
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001478 /* Resend all requests through the MDS */
Bryan Schumaker1abb50882012-06-20 15:53:47 -04001479 nfs_pageio_init_read(&pgio, inode, compl_ops);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001480 while (!list_empty(head)) {
1481 struct nfs_page *req = nfs_list_entry(head->next);
Trond Myklebust62e4a762011-11-10 14:30:37 -05001482
1483 nfs_list_remove_request(req);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001484 if (!nfs_pageio_add_request(&pgio, req))
1485 nfs_list_add_request(req, &failed);
Trond Myklebust62e4a762011-11-10 14:30:37 -05001486 }
1487 nfs_pageio_complete(&pgio);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001488
1489 if (!list_empty(&failed)) {
1490 list_move(&failed, head);
1491 return -EIO;
1492 }
1493 return 0;
1494}
Andy Adamsone7dd79af2012-04-27 17:53:46 -04001495EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001496
1497static void pnfs_ld_handle_read_error(struct nfs_read_data *data)
1498{
Fred Isamancd841602012-04-20 14:47:44 -04001499 struct nfs_pgio_header *hdr = data->header;
1500
1501 dprintk("pnfs read error = %d\n", hdr->pnfs_error);
1502 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001503 PNFS_LAYOUTRET_ON_ERROR) {
Fred Isamancd841602012-04-20 14:47:44 -04001504 clear_bit(NFS_INO_LAYOUTCOMMIT, &NFS_I(hdr->inode)->flags);
1505 pnfs_return_layout(hdr->inode);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001506 }
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001507 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
1508 data->task.tk_status = pnfs_read_done_resend_to_mds(hdr->inode,
Fred Isaman061ae2e2012-04-20 14:47:48 -04001509 &hdr->pages,
1510 hdr->completion_ops);
Trond Myklebust62e4a762011-11-10 14:30:37 -05001511}
1512
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001513/*
Benny Halevyd20581a2011-05-22 19:52:03 +03001514 * Called by non rpc-based layout drivers
1515 */
Peng Tao9b7eecd2011-09-22 21:50:15 -04001516void pnfs_ld_read_done(struct nfs_read_data *data)
Benny Halevyd20581a2011-05-22 19:52:03 +03001517{
Fred Isamancd841602012-04-20 14:47:44 -04001518 struct nfs_pgio_header *hdr = data->header;
1519
1520 if (likely(!hdr->pnfs_error)) {
Benny Halevyd20581a2011-05-22 19:52:03 +03001521 __nfs4_read_done_cb(data);
Fred Isamancd841602012-04-20 14:47:44 -04001522 hdr->mds_ops->rpc_call_done(&data->task, data);
Trond Myklebust62e4a762011-11-10 14:30:37 -05001523 } else
1524 pnfs_ld_handle_read_error(data);
Fred Isamancd841602012-04-20 14:47:44 -04001525 hdr->mds_ops->rpc_release(data);
Benny Halevyd20581a2011-05-22 19:52:03 +03001526}
1527EXPORT_SYMBOL_GPL(pnfs_ld_read_done);
1528
Trond Myklebust493292d2011-07-13 15:58:28 -04001529static void
1530pnfs_read_through_mds(struct nfs_pageio_descriptor *desc,
1531 struct nfs_read_data *data)
1532{
Fred Isamancd841602012-04-20 14:47:44 -04001533 struct nfs_pgio_header *hdr = data->header;
1534
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001535 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
1536 list_splice_tail_init(&hdr->pages, &desc->pg_list);
1537 nfs_pageio_reset_read_mds(desc);
1538 desc->pg_recoalesce = 1;
1539 }
Trond Myklebust493292d2011-07-13 15:58:28 -04001540 nfs_readdata_release(data);
1541}
1542
Benny Halevyd20581a2011-05-22 19:52:03 +03001543/*
Andy Adamson64419a92011-03-01 01:34:16 +00001544 * Call the appropriate parallel I/O subsystem read function.
1545 */
Trond Myklebust493292d2011-07-13 15:58:28 -04001546static enum pnfs_try_status
Andy Adamson64419a92011-03-01 01:34:16 +00001547pnfs_try_to_read_data(struct nfs_read_data *rdata,
Trond Myklebust493292d2011-07-13 15:58:28 -04001548 const struct rpc_call_ops *call_ops,
1549 struct pnfs_layout_segment *lseg)
Andy Adamson64419a92011-03-01 01:34:16 +00001550{
Fred Isamancd841602012-04-20 14:47:44 -04001551 struct nfs_pgio_header *hdr = rdata->header;
1552 struct inode *inode = hdr->inode;
Andy Adamson64419a92011-03-01 01:34:16 +00001553 struct nfs_server *nfss = NFS_SERVER(inode);
1554 enum pnfs_try_status trypnfs;
1555
Fred Isamancd841602012-04-20 14:47:44 -04001556 hdr->mds_ops = call_ops;
Andy Adamson64419a92011-03-01 01:34:16 +00001557
1558 dprintk("%s: Reading ino:%lu %u@%llu\n",
1559 __func__, inode->i_ino, rdata->args.count, rdata->args.offset);
1560
1561 trypnfs = nfss->pnfs_curr_ld->read_pagelist(rdata);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001562 if (trypnfs != PNFS_NOT_ATTEMPTED)
Andy Adamson64419a92011-03-01 01:34:16 +00001563 nfs_inc_stats(inode, NFSIOS_PNFS_READ);
Andy Adamson64419a92011-03-01 01:34:16 +00001564 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
1565 return trypnfs;
1566}
Andy Adamson863a3c62011-03-23 13:27:54 +00001567
Trond Myklebust493292d2011-07-13 15:58:28 -04001568static void
1569pnfs_do_multiple_reads(struct nfs_pageio_descriptor *desc, struct list_head *head)
1570{
1571 struct nfs_read_data *data;
1572 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
1573 struct pnfs_layout_segment *lseg = desc->pg_lseg;
1574
1575 desc->pg_lseg = NULL;
1576 while (!list_empty(head)) {
1577 enum pnfs_try_status trypnfs;
1578
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001579 data = list_first_entry(head, struct nfs_read_data, list);
Trond Myklebust493292d2011-07-13 15:58:28 -04001580 list_del_init(&data->list);
1581
1582 trypnfs = pnfs_try_to_read_data(data, call_ops, lseg);
1583 if (trypnfs == PNFS_NOT_ATTEMPTED)
1584 pnfs_read_through_mds(desc, data);
1585 }
Trond Myklebust9369a432012-09-18 20:57:08 -04001586 pnfs_put_lseg(lseg);
Trond Myklebust493292d2011-07-13 15:58:28 -04001587}
1588
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001589static void pnfs_readhdr_free(struct nfs_pgio_header *hdr)
1590{
Trond Myklebust9369a432012-09-18 20:57:08 -04001591 pnfs_put_lseg(hdr->lseg);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001592 nfs_readhdr_free(hdr);
1593}
Bryan Schumaker89d77c82012-07-30 16:05:25 -04001594EXPORT_SYMBOL_GPL(pnfs_readhdr_free);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001595
Trond Myklebust493292d2011-07-13 15:58:28 -04001596int
1597pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc)
1598{
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001599 struct nfs_read_header *rhdr;
1600 struct nfs_pgio_header *hdr;
Trond Myklebust493292d2011-07-13 15:58:28 -04001601 int ret;
1602
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001603 rhdr = nfs_readhdr_alloc();
1604 if (!rhdr) {
Fred Isaman061ae2e2012-04-20 14:47:48 -04001605 desc->pg_completion_ops->error_cleanup(&desc->pg_list);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001606 ret = -ENOMEM;
Trond Myklebust9369a432012-09-18 20:57:08 -04001607 pnfs_put_lseg(desc->pg_lseg);
Trond Myklebust493292d2011-07-13 15:58:28 -04001608 desc->pg_lseg = NULL;
1609 return ret;
1610 }
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001611 hdr = &rhdr->header;
1612 nfs_pgheader_init(desc, hdr, pnfs_readhdr_free);
Trond Myklebust9369a432012-09-18 20:57:08 -04001613 hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001614 atomic_inc(&hdr->refcnt);
1615 ret = nfs_generic_pagein(desc, hdr);
1616 if (ret != 0) {
Trond Myklebust9369a432012-09-18 20:57:08 -04001617 pnfs_put_lseg(desc->pg_lseg);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001618 desc->pg_lseg = NULL;
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001619 } else
1620 pnfs_do_multiple_reads(desc, &hdr->rpc_list);
1621 if (atomic_dec_and_test(&hdr->refcnt))
Fred Isaman061ae2e2012-04-20 14:47:48 -04001622 hdr->completion_ops->completion(hdr);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04001623 return ret;
Trond Myklebust493292d2011-07-13 15:58:28 -04001624}
1625EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages);
1626
Andy Adamson863a3c62011-03-23 13:27:54 +00001627/*
Peng Taoa9bae562011-07-30 20:52:33 -04001628 * There can be multiple RW segments.
Andy Adamson863a3c62011-03-23 13:27:54 +00001629 */
Peng Taoa9bae562011-07-30 20:52:33 -04001630static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp)
Andy Adamson863a3c62011-03-23 13:27:54 +00001631{
Peng Taoa9bae562011-07-30 20:52:33 -04001632 struct pnfs_layout_segment *lseg;
Andy Adamson863a3c62011-03-23 13:27:54 +00001633
Peng Taoa9bae562011-07-30 20:52:33 -04001634 list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) {
1635 if (lseg->pls_range.iomode == IOMODE_RW &&
1636 test_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
1637 list_add(&lseg->pls_lc_list, listp);
1638 }
Andy Adamson863a3c62011-03-23 13:27:54 +00001639}
1640
Peng Tao1b0ae062011-09-22 21:50:12 -04001641void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg)
1642{
Trond Myklebustb9e028f2012-09-18 16:41:18 -04001643 pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode);
Peng Tao1b0ae062011-09-22 21:50:12 -04001644}
1645EXPORT_SYMBOL_GPL(pnfs_set_lo_fail);
1646
Andy Adamson863a3c62011-03-23 13:27:54 +00001647void
1648pnfs_set_layoutcommit(struct nfs_write_data *wdata)
1649{
Fred Isamancd841602012-04-20 14:47:44 -04001650 struct nfs_pgio_header *hdr = wdata->header;
1651 struct inode *inode = hdr->inode;
1652 struct nfs_inode *nfsi = NFS_I(inode);
Vitaliy Gusev4b8ee2b2011-05-20 01:34:46 +04001653 loff_t end_pos = wdata->mds_offset + wdata->res.count;
Weston Andros Adamson79a48a12011-04-13 10:53:51 -04001654 bool mark_as_dirty = false;
Andy Adamson863a3c62011-03-23 13:27:54 +00001655
Fred Isamancd841602012-04-20 14:47:44 -04001656 spin_lock(&inode->i_lock);
Andy Adamson863a3c62011-03-23 13:27:54 +00001657 if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
Weston Andros Adamson79a48a12011-04-13 10:53:51 -04001658 mark_as_dirty = true;
Andy Adamson863a3c62011-03-23 13:27:54 +00001659 dprintk("%s: Set layoutcommit for inode %lu ",
Fred Isamancd841602012-04-20 14:47:44 -04001660 __func__, inode->i_ino);
Andy Adamson863a3c62011-03-23 13:27:54 +00001661 }
Fred Isamancd841602012-04-20 14:47:44 -04001662 if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &hdr->lseg->pls_flags)) {
Peng Taoa9bae562011-07-30 20:52:33 -04001663 /* references matched in nfs4_layoutcommit_release */
Trond Myklebust9369a432012-09-18 20:57:08 -04001664 pnfs_get_lseg(hdr->lseg);
Peng Taoa9bae562011-07-30 20:52:33 -04001665 }
Peng Taoacff58802011-07-30 20:52:31 -04001666 if (end_pos > nfsi->layout->plh_lwb)
1667 nfsi->layout->plh_lwb = end_pos;
Fred Isamancd841602012-04-20 14:47:44 -04001668 spin_unlock(&inode->i_lock);
Peng Taoacff58802011-07-30 20:52:31 -04001669 dprintk("%s: lseg %p end_pos %llu\n",
Fred Isamancd841602012-04-20 14:47:44 -04001670 __func__, hdr->lseg, nfsi->layout->plh_lwb);
Weston Andros Adamson79a48a12011-04-13 10:53:51 -04001671
1672 /* if pnfs_layoutcommit_inode() runs between inode locks, the next one
1673 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */
1674 if (mark_as_dirty)
Fred Isamancd841602012-04-20 14:47:44 -04001675 mark_inode_dirty_sync(inode);
Andy Adamson863a3c62011-03-23 13:27:54 +00001676}
1677EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit);
1678
Andy Adamsondb29c082011-07-30 20:52:38 -04001679void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data)
1680{
1681 struct nfs_server *nfss = NFS_SERVER(data->args.inode);
1682
1683 if (nfss->pnfs_curr_ld->cleanup_layoutcommit)
1684 nfss->pnfs_curr_ld->cleanup_layoutcommit(data);
1685}
1686
Andy Adamsonde4b15c2011-03-12 02:58:09 -05001687/*
1688 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and
1689 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough
1690 * data to disk to allow the server to recover the data if it crashes.
1691 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag
1692 * is off, and a COMMIT is sent to a data server, or
1693 * if WRITEs to a data server return NFS_DATA_SYNC.
1694 */
Andy Adamson863a3c62011-03-23 13:27:54 +00001695int
Andy Adamsonef311532011-03-12 02:58:10 -05001696pnfs_layoutcommit_inode(struct inode *inode, bool sync)
Andy Adamson863a3c62011-03-23 13:27:54 +00001697{
1698 struct nfs4_layoutcommit_data *data;
1699 struct nfs_inode *nfsi = NFS_I(inode);
Andy Adamson863a3c62011-03-23 13:27:54 +00001700 loff_t end_pos;
1701 int status = 0;
1702
1703 dprintk("--> %s inode %lu\n", __func__, inode->i_ino);
1704
Andy Adamsonde4b15c2011-03-12 02:58:09 -05001705 if (!test_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
1706 return 0;
1707
Andy Adamson863a3c62011-03-23 13:27:54 +00001708 /* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */
1709 data = kzalloc(sizeof(*data), GFP_NOFS);
Andy Adamsonde4b15c2011-03-12 02:58:09 -05001710 if (!data) {
Andy Adamsonde4b15c2011-03-12 02:58:09 -05001711 status = -ENOMEM;
1712 goto out;
1713 }
Andy Adamson863a3c62011-03-23 13:27:54 +00001714
Peng Tao92407e72011-10-23 20:21:17 -07001715 if (!test_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
1716 goto out_free;
1717
1718 if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) {
1719 if (!sync) {
1720 status = -EAGAIN;
1721 goto out_free;
1722 }
1723 status = wait_on_bit_lock(&nfsi->flags, NFS_INO_LAYOUTCOMMITTING,
1724 nfs_wait_bit_killable, TASK_KILLABLE);
1725 if (status)
1726 goto out_free;
1727 }
1728
Peng Taoa9bae562011-07-30 20:52:33 -04001729 INIT_LIST_HEAD(&data->lseg_list);
Andy Adamsonde4b15c2011-03-12 02:58:09 -05001730 spin_lock(&inode->i_lock);
Andy Adamson863a3c62011-03-23 13:27:54 +00001731 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
Peng Tao92407e72011-10-23 20:21:17 -07001732 clear_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags);
Andy Adamson863a3c62011-03-23 13:27:54 +00001733 spin_unlock(&inode->i_lock);
Peng Tao92407e72011-10-23 20:21:17 -07001734 wake_up_bit(&nfsi->flags, NFS_INO_LAYOUTCOMMITTING);
1735 goto out_free;
Andy Adamson863a3c62011-03-23 13:27:54 +00001736 }
Peng Taoa9bae562011-07-30 20:52:33 -04001737
1738 pnfs_list_write_lseg(inode, &data->lseg_list);
Andy Adamson863a3c62011-03-23 13:27:54 +00001739
Peng Taoacff58802011-07-30 20:52:31 -04001740 end_pos = nfsi->layout->plh_lwb;
Peng Taoacff58802011-07-30 20:52:31 -04001741 nfsi->layout->plh_lwb = 0;
Andy Adamson863a3c62011-03-23 13:27:54 +00001742
Trond Myklebustf597c532012-03-04 18:13:56 -05001743 nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid);
Andy Adamson863a3c62011-03-23 13:27:54 +00001744 spin_unlock(&inode->i_lock);
1745
1746 data->args.inode = inode;
Peng Tao9fa40752011-07-30 20:52:32 -04001747 data->cred = get_rpccred(nfsi->layout->plh_lc_cred);
Andy Adamson863a3c62011-03-23 13:27:54 +00001748 nfs_fattr_init(&data->fattr);
1749 data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask;
1750 data->res.fattr = &data->fattr;
1751 data->args.lastbytewritten = end_pos - 1;
1752 data->res.server = NFS_SERVER(inode);
1753
1754 status = nfs4_proc_layoutcommit(data, sync);
1755out:
Peng Tao92407e72011-10-23 20:21:17 -07001756 if (status)
1757 mark_inode_dirty_sync(inode);
Andy Adamson863a3c62011-03-23 13:27:54 +00001758 dprintk("<-- %s status %d\n", __func__, status);
1759 return status;
Peng Tao92407e72011-10-23 20:21:17 -07001760out_free:
1761 kfree(data);
1762 goto out;
Andy Adamson863a3c62011-03-23 13:27:54 +00001763}
Andy Adamson82be4172012-05-23 05:02:35 -04001764
1765struct nfs4_threshold *pnfs_mdsthreshold_alloc(void)
1766{
1767 struct nfs4_threshold *thp;
1768
1769 thp = kzalloc(sizeof(*thp), GFP_NOFS);
1770 if (!thp) {
1771 dprintk("%s mdsthreshold allocation failed\n", __func__);
1772 return NULL;
1773 }
1774 return thp;
1775}