blob: dd3ede86bb2e3d5b67dcb57269c5647717834717 [file] [log] [blame]
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001/*******************************************************************************
2 * This file contains iSCSI extentions for RDMA (iSER) Verbs
3 *
4 * (c) Copyright 2013 RisingTide Systems LLC.
5 *
6 * Nicholas A. Bellinger <nab@linux-iscsi.org>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 ****************************************************************************/
18
19#include <linux/string.h>
20#include <linux/module.h>
21#include <linux/scatterlist.h>
22#include <linux/socket.h>
23#include <linux/in.h>
24#include <linux/in6.h>
25#include <rdma/ib_verbs.h>
26#include <rdma/rdma_cm.h>
27#include <target/target_core_base.h>
28#include <target/target_core_fabric.h>
29#include <target/iscsi/iscsi_transport.h>
Sagi Grimberg8a2629a2014-04-29 13:13:45 +030030#include <linux/semaphore.h>
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -080031
32#include "isert_proto.h"
33#include "ib_isert.h"
34
35#define ISERT_MAX_CONN 8
36#define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN)
37#define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN)
38
39static DEFINE_MUTEX(device_list_mutex);
40static LIST_HEAD(device_list);
41static struct workqueue_struct *isert_rx_wq;
42static struct workqueue_struct *isert_comp_wq;
43static struct kmem_cache *isert_cmd_cache;
44
45static void
46isert_qp_event_callback(struct ib_event *e, void *context)
47{
48 struct isert_conn *isert_conn = (struct isert_conn *)context;
49
50 pr_err("isert_qp_event_callback event: %d\n", e->event);
51 switch (e->event) {
52 case IB_EVENT_COMM_EST:
53 rdma_notify(isert_conn->conn_cm_id, IB_EVENT_COMM_EST);
54 break;
55 case IB_EVENT_QP_LAST_WQE_REACHED:
56 pr_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED:\n");
57 break;
58 default:
59 break;
60 }
61}
62
63static int
64isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr)
65{
66 int ret;
67
68 ret = ib_query_device(ib_dev, devattr);
69 if (ret) {
70 pr_err("ib_query_device() failed: %d\n", ret);
71 return ret;
72 }
73 pr_debug("devattr->max_sge: %d\n", devattr->max_sge);
74 pr_debug("devattr->max_sge_rd: %d\n", devattr->max_sge_rd);
75
76 return 0;
77}
78
79static int
80isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id)
81{
82 struct isert_device *device = isert_conn->conn_device;
83 struct ib_qp_init_attr attr;
84 struct ib_device_attr devattr;
85 int ret, index, min_index = 0;
86
87 memset(&devattr, 0, sizeof(struct ib_device_attr));
88 ret = isert_query_device(cma_id->device, &devattr);
89 if (ret)
90 return ret;
91
92 mutex_lock(&device_list_mutex);
93 for (index = 0; index < device->cqs_used; index++)
94 if (device->cq_active_qps[index] <
95 device->cq_active_qps[min_index])
96 min_index = index;
97 device->cq_active_qps[min_index]++;
98 pr_debug("isert_conn_setup_qp: Using min_index: %d\n", min_index);
99 mutex_unlock(&device_list_mutex);
100
101 memset(&attr, 0, sizeof(struct ib_qp_init_attr));
102 attr.event_handler = isert_qp_event_callback;
103 attr.qp_context = isert_conn;
104 attr.send_cq = device->dev_tx_cq[min_index];
105 attr.recv_cq = device->dev_rx_cq[min_index];
106 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS;
107 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS;
108 /*
109 * FIXME: Use devattr.max_sge - 2 for max_send_sge as
Or Gerlitzf2da3c22015-01-30 22:17:21 +0000110 * work-around for RDMA_READs with ConnectX-2.
111 *
112 * Also, still make sure to have at least two SGEs for
113 * outgoing control PDU responses.
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800114 */
Or Gerlitzf2da3c22015-01-30 22:17:21 +0000115 attr.cap.max_send_sge = max(2, devattr.max_sge - 2);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800116 isert_conn->max_sge = attr.cap.max_send_sge;
117
118 attr.cap.max_recv_sge = 1;
119 attr.sq_sig_type = IB_SIGNAL_REQ_WR;
120 attr.qp_type = IB_QPT_RC;
121
122 pr_debug("isert_conn_setup_qp cma_id->device: %p\n",
123 cma_id->device);
124 pr_debug("isert_conn_setup_qp conn_pd->device: %p\n",
125 isert_conn->conn_pd->device);
126
127 ret = rdma_create_qp(cma_id, isert_conn->conn_pd, &attr);
128 if (ret) {
129 pr_err("rdma_create_qp failed for cma_id %d\n", ret);
130 return ret;
131 }
132 isert_conn->conn_qp = cma_id->qp;
133 pr_debug("rdma_create_qp() returned success >>>>>>>>>>>>>>>>>>>>>>>>>.\n");
134
135 return 0;
136}
137
138static void
139isert_cq_event_callback(struct ib_event *e, void *context)
140{
141 pr_debug("isert_cq_event_callback event: %d\n", e->event);
142}
143
144static int
145isert_alloc_rx_descriptors(struct isert_conn *isert_conn)
146{
147 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
148 struct iser_rx_desc *rx_desc;
149 struct ib_sge *rx_sg;
150 u64 dma_addr;
151 int i, j;
152
153 isert_conn->conn_rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS *
154 sizeof(struct iser_rx_desc), GFP_KERNEL);
155 if (!isert_conn->conn_rx_descs)
156 goto fail;
157
158 rx_desc = isert_conn->conn_rx_descs;
159
160 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) {
161 dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc,
162 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
163 if (ib_dma_mapping_error(ib_dev, dma_addr))
164 goto dma_map_fail;
165
166 rx_desc->dma_addr = dma_addr;
167
168 rx_sg = &rx_desc->rx_sg;
169 rx_sg->addr = rx_desc->dma_addr;
170 rx_sg->length = ISER_RX_PAYLOAD_SIZE;
171 rx_sg->lkey = isert_conn->conn_mr->lkey;
172 }
173
174 isert_conn->conn_rx_desc_head = 0;
175 return 0;
176
177dma_map_fail:
178 rx_desc = isert_conn->conn_rx_descs;
179 for (j = 0; j < i; j++, rx_desc++) {
180 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
181 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
182 }
183 kfree(isert_conn->conn_rx_descs);
184 isert_conn->conn_rx_descs = NULL;
185fail:
186 return -ENOMEM;
187}
188
189static void
190isert_free_rx_descriptors(struct isert_conn *isert_conn)
191{
192 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
193 struct iser_rx_desc *rx_desc;
194 int i;
195
196 if (!isert_conn->conn_rx_descs)
197 return;
198
199 rx_desc = isert_conn->conn_rx_descs;
200 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) {
201 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
202 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
203 }
204
205 kfree(isert_conn->conn_rx_descs);
206 isert_conn->conn_rx_descs = NULL;
207}
208
209static void isert_cq_tx_callback(struct ib_cq *, void *);
210static void isert_cq_rx_callback(struct ib_cq *, void *);
211
212static int
213isert_create_device_ib_res(struct isert_device *device)
214{
215 struct ib_device *ib_dev = device->ib_device;
216 struct isert_cq_desc *cq_desc;
217 int ret = 0, i, j;
Chris Moorec50ad632015-01-30 22:17:20 +0000218 int max_rx_cqe, max_tx_cqe;
219 struct ib_device_attr dev_attr;
220
221 memset(&dev_attr, 0, sizeof(struct ib_device_attr));
222 ret = isert_query_device(device->ib_device, &dev_attr);
223 if (ret)
224 return ret;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800225
226 device->cqs_used = min_t(int, num_online_cpus(),
227 device->ib_device->num_comp_vectors);
228 device->cqs_used = min(ISERT_MAX_CQ, device->cqs_used);
229 pr_debug("Using %d CQs, device %s supports %d vectors\n",
230 device->cqs_used, device->ib_device->name,
231 device->ib_device->num_comp_vectors);
232 device->cq_desc = kzalloc(sizeof(struct isert_cq_desc) *
233 device->cqs_used, GFP_KERNEL);
234 if (!device->cq_desc) {
235 pr_err("Unable to allocate device->cq_desc\n");
236 return -ENOMEM;
237 }
238 cq_desc = device->cq_desc;
239
240 device->dev_pd = ib_alloc_pd(ib_dev);
241 if (IS_ERR(device->dev_pd)) {
242 ret = PTR_ERR(device->dev_pd);
243 pr_err("ib_alloc_pd failed for dev_pd: %d\n", ret);
244 goto out_cq_desc;
245 }
246
Chris Moorec50ad632015-01-30 22:17:20 +0000247 max_rx_cqe = min(ISER_MAX_RX_CQ_LEN, dev_attr.max_cqe);
248 max_tx_cqe = min(ISER_MAX_TX_CQ_LEN, dev_attr.max_cqe);
249
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800250 for (i = 0; i < device->cqs_used; i++) {
251 cq_desc[i].device = device;
252 cq_desc[i].cq_index = i;
253
254 device->dev_rx_cq[i] = ib_create_cq(device->ib_device,
255 isert_cq_rx_callback,
256 isert_cq_event_callback,
257 (void *)&cq_desc[i],
Chris Moorec50ad632015-01-30 22:17:20 +0000258 max_rx_cqe, i);
Wei Yongjun0629b402013-10-29 09:56:34 +0800259 if (IS_ERR(device->dev_rx_cq[i])) {
260 ret = PTR_ERR(device->dev_rx_cq[i]);
261 device->dev_rx_cq[i] = NULL;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800262 goto out_cq;
Wei Yongjun0629b402013-10-29 09:56:34 +0800263 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800264
265 device->dev_tx_cq[i] = ib_create_cq(device->ib_device,
266 isert_cq_tx_callback,
267 isert_cq_event_callback,
268 (void *)&cq_desc[i],
Chris Moorec50ad632015-01-30 22:17:20 +0000269 max_tx_cqe, i);
Wei Yongjun0629b402013-10-29 09:56:34 +0800270 if (IS_ERR(device->dev_tx_cq[i])) {
271 ret = PTR_ERR(device->dev_tx_cq[i]);
272 device->dev_tx_cq[i] = NULL;
273 goto out_cq;
274 }
275
276 ret = ib_req_notify_cq(device->dev_rx_cq[i], IB_CQ_NEXT_COMP);
277 if (ret)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800278 goto out_cq;
279
Wei Yongjun0629b402013-10-29 09:56:34 +0800280 ret = ib_req_notify_cq(device->dev_tx_cq[i], IB_CQ_NEXT_COMP);
281 if (ret)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800282 goto out_cq;
283 }
284
285 device->dev_mr = ib_get_dma_mr(device->dev_pd, IB_ACCESS_LOCAL_WRITE);
286 if (IS_ERR(device->dev_mr)) {
287 ret = PTR_ERR(device->dev_mr);
288 pr_err("ib_get_dma_mr failed for dev_mr: %d\n", ret);
289 goto out_cq;
290 }
291
292 return 0;
293
294out_cq:
295 for (j = 0; j < i; j++) {
296 cq_desc = &device->cq_desc[j];
297
298 if (device->dev_rx_cq[j]) {
299 cancel_work_sync(&cq_desc->cq_rx_work);
300 ib_destroy_cq(device->dev_rx_cq[j]);
301 }
302 if (device->dev_tx_cq[j]) {
303 cancel_work_sync(&cq_desc->cq_tx_work);
304 ib_destroy_cq(device->dev_tx_cq[j]);
305 }
306 }
307 ib_dealloc_pd(device->dev_pd);
308
309out_cq_desc:
310 kfree(device->cq_desc);
311
312 return ret;
313}
314
315static void
316isert_free_device_ib_res(struct isert_device *device)
317{
318 struct isert_cq_desc *cq_desc;
319 int i;
320
321 for (i = 0; i < device->cqs_used; i++) {
322 cq_desc = &device->cq_desc[i];
323
324 cancel_work_sync(&cq_desc->cq_rx_work);
325 cancel_work_sync(&cq_desc->cq_tx_work);
326 ib_destroy_cq(device->dev_rx_cq[i]);
327 ib_destroy_cq(device->dev_tx_cq[i]);
328 device->dev_rx_cq[i] = NULL;
329 device->dev_tx_cq[i] = NULL;
330 }
331
332 ib_dereg_mr(device->dev_mr);
333 ib_dealloc_pd(device->dev_pd);
334 kfree(device->cq_desc);
335}
336
337static void
338isert_device_try_release(struct isert_device *device)
339{
340 mutex_lock(&device_list_mutex);
341 device->refcount--;
342 if (!device->refcount) {
343 isert_free_device_ib_res(device);
344 list_del(&device->dev_node);
345 kfree(device);
346 }
347 mutex_unlock(&device_list_mutex);
348}
349
350static struct isert_device *
351isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id)
352{
353 struct isert_device *device;
354 int ret;
355
356 mutex_lock(&device_list_mutex);
357 list_for_each_entry(device, &device_list, dev_node) {
358 if (device->ib_device->node_guid == cma_id->device->node_guid) {
359 device->refcount++;
360 mutex_unlock(&device_list_mutex);
361 return device;
362 }
363 }
364
365 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL);
366 if (!device) {
367 mutex_unlock(&device_list_mutex);
368 return ERR_PTR(-ENOMEM);
369 }
370
371 INIT_LIST_HEAD(&device->dev_node);
372
373 device->ib_device = cma_id->device;
374 ret = isert_create_device_ib_res(device);
375 if (ret) {
376 kfree(device);
377 mutex_unlock(&device_list_mutex);
378 return ERR_PTR(ret);
379 }
380
381 device->refcount++;
382 list_add_tail(&device->dev_node, &device_list);
383 mutex_unlock(&device_list_mutex);
384
385 return device;
386}
387
388static int
389isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
390{
391 struct iscsi_np *np = cma_id->context;
392 struct isert_np *isert_np = np->np_context;
393 struct isert_conn *isert_conn;
394 struct isert_device *device;
395 struct ib_device *ib_dev = cma_id->device;
396 int ret = 0;
397
Sagi Grimbergc2334082014-04-29 13:13:47 +0300398 spin_lock_bh(&np->np_thread_lock);
399 if (!np->enabled) {
400 spin_unlock_bh(&np->np_thread_lock);
401 pr_debug("iscsi_np is not enabled, reject connect request\n");
402 return rdma_reject(cma_id, NULL, 0);
403 }
404 spin_unlock_bh(&np->np_thread_lock);
405
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800406 pr_debug("Entering isert_connect_request cma_id: %p, context: %p\n",
407 cma_id, cma_id->context);
408
409 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL);
410 if (!isert_conn) {
411 pr_err("Unable to allocate isert_conn\n");
412 return -ENOMEM;
413 }
414 isert_conn->state = ISER_CONN_INIT;
415 INIT_LIST_HEAD(&isert_conn->conn_accept_node);
416 init_completion(&isert_conn->conn_login_comp);
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -0800417 init_completion(&isert_conn->conn_wait);
418 init_completion(&isert_conn->conn_wait_comp_err);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800419 kref_init(&isert_conn->conn_kref);
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -0700420 mutex_init(&isert_conn->conn_mutex);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800421
422 cma_id->context = isert_conn;
423 isert_conn->conn_cm_id = cma_id;
424 isert_conn->responder_resources = event->param.conn.responder_resources;
425 isert_conn->initiator_depth = event->param.conn.initiator_depth;
426 pr_debug("Using responder_resources: %u initiator_depth: %u\n",
427 isert_conn->responder_resources, isert_conn->initiator_depth);
428
429 isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN +
430 ISER_RX_LOGIN_SIZE, GFP_KERNEL);
431 if (!isert_conn->login_buf) {
432 pr_err("Unable to allocate isert_conn->login_buf\n");
433 ret = -ENOMEM;
434 goto out;
435 }
436
437 isert_conn->login_req_buf = isert_conn->login_buf;
438 isert_conn->login_rsp_buf = isert_conn->login_buf +
439 ISCSI_DEF_MAX_RECV_SEG_LEN;
440 pr_debug("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n",
441 isert_conn->login_buf, isert_conn->login_req_buf,
442 isert_conn->login_rsp_buf);
443
444 isert_conn->login_req_dma = ib_dma_map_single(ib_dev,
445 (void *)isert_conn->login_req_buf,
446 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE);
447
448 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma);
449 if (ret) {
450 pr_err("ib_dma_mapping_error failed for login_req_dma: %d\n",
451 ret);
452 isert_conn->login_req_dma = 0;
453 goto out_login_buf;
454 }
455
456 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev,
457 (void *)isert_conn->login_rsp_buf,
458 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
459
460 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma);
461 if (ret) {
462 pr_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n",
463 ret);
464 isert_conn->login_rsp_dma = 0;
465 goto out_req_dma_map;
466 }
467
468 device = isert_device_find_by_ib_dev(cma_id);
469 if (IS_ERR(device)) {
470 ret = PTR_ERR(device);
471 goto out_rsp_dma_map;
472 }
473
474 isert_conn->conn_device = device;
475 isert_conn->conn_pd = device->dev_pd;
476 isert_conn->conn_mr = device->dev_mr;
477
478 ret = isert_conn_setup_qp(isert_conn, cma_id);
479 if (ret)
480 goto out_conn_dev;
481
482 mutex_lock(&isert_np->np_accept_mutex);
Sagi Grimberg5de94f82014-04-29 13:13:44 +0300483 list_add_tail(&isert_conn->conn_accept_node, &isert_np->np_accept_list);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800484 mutex_unlock(&isert_np->np_accept_mutex);
485
Sagi Grimberg8a2629a2014-04-29 13:13:45 +0300486 pr_debug("isert_connect_request() up np_sem np: %p\n", np);
487 up(&isert_np->np_sem);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800488 return 0;
489
490out_conn_dev:
491 isert_device_try_release(device);
492out_rsp_dma_map:
493 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma,
494 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
495out_req_dma_map:
496 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
497 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE);
498out_login_buf:
499 kfree(isert_conn->login_buf);
500out:
501 kfree(isert_conn);
502 return ret;
503}
504
505static void
506isert_connect_release(struct isert_conn *isert_conn)
507{
508 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
509 struct isert_device *device = isert_conn->conn_device;
510 int cq_index;
511
512 pr_debug("Entering isert_connect_release(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
513
514 if (isert_conn->conn_qp) {
515 cq_index = ((struct isert_cq_desc *)
516 isert_conn->conn_qp->recv_cq->cq_context)->cq_index;
517 pr_debug("isert_connect_release: cq_index: %d\n", cq_index);
518 isert_conn->conn_device->cq_active_qps[cq_index]--;
519
520 rdma_destroy_qp(isert_conn->conn_cm_id);
521 }
522
523 isert_free_rx_descriptors(isert_conn);
524 rdma_destroy_id(isert_conn->conn_cm_id);
525
526 if (isert_conn->login_buf) {
527 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma,
528 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
529 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
530 ISCSI_DEF_MAX_RECV_SEG_LEN,
531 DMA_FROM_DEVICE);
532 kfree(isert_conn->login_buf);
533 }
534 kfree(isert_conn);
535
536 if (device)
537 isert_device_try_release(device);
538
539 pr_debug("Leaving isert_connect_release >>>>>>>>>>>>\n");
540}
541
542static void
543isert_connected_handler(struct rdma_cm_id *cma_id)
544{
Sagi Grimberg058ab452014-07-02 16:19:24 +0300545 struct isert_conn *isert_conn = cma_id->context;
546
547 kref_get(&isert_conn->conn_kref);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800548}
549
550static void
551isert_release_conn_kref(struct kref *kref)
552{
553 struct isert_conn *isert_conn = container_of(kref,
554 struct isert_conn, conn_kref);
555
556 pr_debug("Calling isert_connect_release for final kref %s/%d\n",
557 current->comm, current->pid);
558
559 isert_connect_release(isert_conn);
560}
561
562static void
563isert_put_conn(struct isert_conn *isert_conn)
564{
565 kref_put(&isert_conn->conn_kref, isert_release_conn_kref);
566}
567
Sagi Grimberg839eac52015-01-30 22:17:25 +0000568/**
569 * isert_conn_terminate() - Initiate connection termination
570 * @isert_conn: isert connection struct
571 *
572 * Notes:
573 * In case the connection state is UP, move state
574 * to TEMINATING and start teardown sequence (rdma_disconnect).
575 *
576 * This routine must be called with conn_mutex held. Thus it is
577 * safe to call multiple times.
578 */
579static void
580isert_conn_terminate(struct isert_conn *isert_conn)
581{
582 int err;
583
584 if (isert_conn->state == ISER_CONN_UP) {
585 isert_conn->state = ISER_CONN_TERMINATING;
586 pr_info("Terminating conn %p state %d\n",
587 isert_conn, isert_conn->state);
588 err = rdma_disconnect(isert_conn->conn_cm_id);
589 if (err)
590 pr_warn("Failed rdma_disconnect isert_conn %p\n",
591 isert_conn);
592 }
593}
594
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800595static void
596isert_disconnect_work(struct work_struct *work)
597{
598 struct isert_conn *isert_conn = container_of(work,
599 struct isert_conn, conn_logout_work);
600
601 pr_debug("isert_disconnect_work(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -0700602 mutex_lock(&isert_conn->conn_mutex);
Sagi Grimberg839eac52015-01-30 22:17:25 +0000603 isert_conn_terminate(isert_conn);
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -0700604 mutex_unlock(&isert_conn->conn_mutex);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800605
Sagi Grimberg839eac52015-01-30 22:17:25 +0000606 pr_info("conn %p completing conn_wait\n", isert_conn);
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -0800607 complete(&isert_conn->conn_wait);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800608}
609
Sagi Grimberg11b926d2014-10-28 13:45:03 -0700610static int
Sagi Grimberg839eac52015-01-30 22:17:25 +0000611isert_disconnected_handler(struct rdma_cm_id *cma_id)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800612{
Sagi Grimberg11b926d2014-10-28 13:45:03 -0700613 struct isert_conn *isert_conn;
614
615 if (!cma_id->qp) {
616 struct isert_np *isert_np = cma_id->context;
617
618 isert_np->np_cm_id = NULL;
619 return -1;
620 }
621
622 isert_conn = (struct isert_conn *)cma_id->context;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800623
624 INIT_WORK(&isert_conn->conn_logout_work, isert_disconnect_work);
625 schedule_work(&isert_conn->conn_logout_work);
Sagi Grimberg11b926d2014-10-28 13:45:03 -0700626
627 return 0;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800628}
629
Sagi Grimberg839eac52015-01-30 22:17:25 +0000630static void
631isert_connect_error(struct rdma_cm_id *cma_id)
632{
633 struct isert_conn *isert_conn = (struct isert_conn *)cma_id->context;
634
635 isert_put_conn(isert_conn);
636}
637
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800638static int
639isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
640{
641 int ret = 0;
642
643 pr_debug("isert_cma_handler: event %d status %d conn %p id %p\n",
644 event->event, event->status, cma_id->context, cma_id);
645
646 switch (event->event) {
647 case RDMA_CM_EVENT_CONNECT_REQUEST:
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800648 ret = isert_connect_request(cma_id, event);
Sagi Grimberg11b926d2014-10-28 13:45:03 -0700649 if (ret)
650 pr_err("isert_cma_handler failed RDMA_CM_EVENT: 0x%08x %d\n",
651 event->event, ret);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800652 break;
653 case RDMA_CM_EVENT_ESTABLISHED:
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800654 isert_connected_handler(cma_id);
655 break;
Sagi Grimberg674a2362014-05-19 17:44:24 +0300656 case RDMA_CM_EVENT_ADDR_CHANGE: /* FALLTHRU */
657 case RDMA_CM_EVENT_DISCONNECTED: /* FALLTHRU */
658 case RDMA_CM_EVENT_DEVICE_REMOVAL: /* FALLTHRU */
Sagi Grimberg674a2362014-05-19 17:44:24 +0300659 case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */
Sagi Grimberg839eac52015-01-30 22:17:25 +0000660 ret = isert_disconnected_handler(cma_id);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800661 break;
Sagi Grimberg839eac52015-01-30 22:17:25 +0000662 case RDMA_CM_EVENT_REJECTED: /* FALLTHRU */
663 case RDMA_CM_EVENT_UNREACHABLE: /* FALLTHRU */
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800664 case RDMA_CM_EVENT_CONNECT_ERROR:
Sagi Grimberg839eac52015-01-30 22:17:25 +0000665 isert_connect_error(cma_id);
666 break;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800667 default:
Sagi Grimberg674a2362014-05-19 17:44:24 +0300668 pr_err("Unhandled RDMA CMA event: %d\n", event->event);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800669 break;
670 }
671
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800672 return ret;
673}
674
675static int
676isert_post_recv(struct isert_conn *isert_conn, u32 count)
677{
678 struct ib_recv_wr *rx_wr, *rx_wr_failed;
679 int i, ret;
680 unsigned int rx_head = isert_conn->conn_rx_desc_head;
681 struct iser_rx_desc *rx_desc;
682
683 for (rx_wr = isert_conn->conn_rx_wr, i = 0; i < count; i++, rx_wr++) {
684 rx_desc = &isert_conn->conn_rx_descs[rx_head];
685 rx_wr->wr_id = (unsigned long)rx_desc;
686 rx_wr->sg_list = &rx_desc->rx_sg;
687 rx_wr->num_sge = 1;
688 rx_wr->next = rx_wr + 1;
689 rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1);
690 }
691
692 rx_wr--;
693 rx_wr->next = NULL; /* mark end of work requests list */
694
695 isert_conn->post_recv_buf_count += count;
696 ret = ib_post_recv(isert_conn->conn_qp, isert_conn->conn_rx_wr,
697 &rx_wr_failed);
698 if (ret) {
699 pr_err("ib_post_recv() failed with ret: %d\n", ret);
700 isert_conn->post_recv_buf_count -= count;
701 } else {
702 pr_debug("isert_post_recv(): Posted %d RX buffers\n", count);
703 isert_conn->conn_rx_desc_head = rx_head;
704 }
705 return ret;
706}
707
708static int
709isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc)
710{
711 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
712 struct ib_send_wr send_wr, *send_wr_failed;
713 int ret;
714
715 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr,
716 ISER_HEADERS_LEN, DMA_TO_DEVICE);
717
718 send_wr.next = NULL;
719 send_wr.wr_id = (unsigned long)tx_desc;
720 send_wr.sg_list = tx_desc->tx_sg;
721 send_wr.num_sge = tx_desc->num_sge;
722 send_wr.opcode = IB_WR_SEND;
723 send_wr.send_flags = IB_SEND_SIGNALED;
724
725 atomic_inc(&isert_conn->post_send_buf_count);
726
727 ret = ib_post_send(isert_conn->conn_qp, &send_wr, &send_wr_failed);
728 if (ret) {
729 pr_err("ib_post_send() failed, ret: %d\n", ret);
730 atomic_dec(&isert_conn->post_send_buf_count);
731 }
732
733 return ret;
734}
735
736static void
737isert_create_send_desc(struct isert_conn *isert_conn,
738 struct isert_cmd *isert_cmd,
739 struct iser_tx_desc *tx_desc)
740{
741 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
742
743 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr,
744 ISER_HEADERS_LEN, DMA_TO_DEVICE);
745
746 memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr));
747 tx_desc->iser_header.flags = ISER_VER;
748
749 tx_desc->num_sge = 1;
750 tx_desc->isert_cmd = isert_cmd;
751
752 if (tx_desc->tx_sg[0].lkey != isert_conn->conn_mr->lkey) {
753 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey;
754 pr_debug("tx_desc %p lkey mismatch, fixing\n", tx_desc);
755 }
756}
757
758static int
759isert_init_tx_hdrs(struct isert_conn *isert_conn,
760 struct iser_tx_desc *tx_desc)
761{
762 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
763 u64 dma_addr;
764
765 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc,
766 ISER_HEADERS_LEN, DMA_TO_DEVICE);
767 if (ib_dma_mapping_error(ib_dev, dma_addr)) {
768 pr_err("ib_dma_mapping_error() failed\n");
769 return -ENOMEM;
770 }
771
772 tx_desc->dma_addr = dma_addr;
773 tx_desc->tx_sg[0].addr = tx_desc->dma_addr;
774 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN;
775 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey;
776
777 pr_debug("isert_init_tx_hdrs: Setup tx_sg[0].addr: 0x%llx length: %u"
778 " lkey: 0x%08x\n", tx_desc->tx_sg[0].addr,
779 tx_desc->tx_sg[0].length, tx_desc->tx_sg[0].lkey);
780
781 return 0;
782}
783
784static void
785isert_init_send_wr(struct isert_cmd *isert_cmd, struct ib_send_wr *send_wr)
786{
787 isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND;
788 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc;
789 send_wr->opcode = IB_WR_SEND;
790 send_wr->send_flags = IB_SEND_SIGNALED;
791 send_wr->sg_list = &isert_cmd->tx_desc.tx_sg[0];
792 send_wr->num_sge = isert_cmd->tx_desc.num_sge;
793}
794
795static int
796isert_rdma_post_recvl(struct isert_conn *isert_conn)
797{
798 struct ib_recv_wr rx_wr, *rx_wr_fail;
799 struct ib_sge sge;
800 int ret;
801
802 memset(&sge, 0, sizeof(struct ib_sge));
803 sge.addr = isert_conn->login_req_dma;
804 sge.length = ISER_RX_LOGIN_SIZE;
805 sge.lkey = isert_conn->conn_mr->lkey;
806
807 pr_debug("Setup sge: addr: %llx length: %d 0x%08x\n",
808 sge.addr, sge.length, sge.lkey);
809
810 memset(&rx_wr, 0, sizeof(struct ib_recv_wr));
811 rx_wr.wr_id = (unsigned long)isert_conn->login_req_buf;
812 rx_wr.sg_list = &sge;
813 rx_wr.num_sge = 1;
814
815 isert_conn->post_recv_buf_count++;
816 ret = ib_post_recv(isert_conn->conn_qp, &rx_wr, &rx_wr_fail);
817 if (ret) {
818 pr_err("ib_post_recv() failed: %d\n", ret);
819 isert_conn->post_recv_buf_count--;
820 }
821
822 pr_debug("ib_post_recv(): returned success >>>>>>>>>>>>>>>>>>>>>>>>\n");
823 return ret;
824}
825
826static int
827isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
828 u32 length)
829{
830 struct isert_conn *isert_conn = conn->context;
831 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
832 struct iser_tx_desc *tx_desc = &isert_conn->conn_login_tx_desc;
833 int ret;
834
835 isert_create_send_desc(isert_conn, NULL, tx_desc);
836
837 memcpy(&tx_desc->iscsi_header, &login->rsp[0],
838 sizeof(struct iscsi_hdr));
839
840 isert_init_tx_hdrs(isert_conn, tx_desc);
841
842 if (length > 0) {
843 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1];
844
845 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma,
846 length, DMA_TO_DEVICE);
847
848 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length);
849
850 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma,
851 length, DMA_TO_DEVICE);
852
853 tx_dsg->addr = isert_conn->login_rsp_dma;
854 tx_dsg->length = length;
855 tx_dsg->lkey = isert_conn->conn_mr->lkey;
856 tx_desc->num_sge = 2;
857 }
858 if (!login->login_failed) {
859 if (login->login_complete) {
860 ret = isert_alloc_rx_descriptors(isert_conn);
861 if (ret)
862 return ret;
863
864 ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX);
865 if (ret)
866 return ret;
867
868 isert_conn->state = ISER_CONN_UP;
869 goto post_send;
870 }
871
872 ret = isert_rdma_post_recvl(isert_conn);
873 if (ret)
874 return ret;
875 }
876post_send:
877 ret = isert_post_send(isert_conn, tx_desc);
878 if (ret)
879 return ret;
880
881 return 0;
882}
883
884static void
885isert_rx_login_req(struct iser_rx_desc *rx_desc, int rx_buflen,
886 struct isert_conn *isert_conn)
887{
888 struct iscsi_conn *conn = isert_conn->conn;
889 struct iscsi_login *login = conn->conn_login;
890 int size;
891
892 if (!login) {
893 pr_err("conn->conn_login is NULL\n");
894 dump_stack();
895 return;
896 }
897
898 if (login->first_request) {
899 struct iscsi_login_req *login_req =
900 (struct iscsi_login_req *)&rx_desc->iscsi_header;
901 /*
902 * Setup the initial iscsi_login values from the leading
903 * login request PDU.
904 */
905 login->leading_connection = (!login_req->tsih) ? 1 : 0;
906 login->current_stage =
907 (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK)
908 >> 2;
909 login->version_min = login_req->min_version;
910 login->version_max = login_req->max_version;
911 memcpy(login->isid, login_req->isid, 6);
912 login->cmd_sn = be32_to_cpu(login_req->cmdsn);
913 login->init_task_tag = login_req->itt;
914 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
915 login->cid = be16_to_cpu(login_req->cid);
916 login->tsih = be16_to_cpu(login_req->tsih);
917 }
918
919 memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN);
920
921 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS);
922 pr_debug("Using login payload size: %d, rx_buflen: %d MAX_KEY_VALUE_PAIRS: %d\n",
923 size, rx_buflen, MAX_KEY_VALUE_PAIRS);
924 memcpy(login->req_buf, &rx_desc->data[0], size);
925
926 complete(&isert_conn->conn_login_comp);
927}
928
929static void
930isert_release_cmd(struct iscsi_cmd *cmd)
931{
932 struct isert_cmd *isert_cmd = container_of(cmd, struct isert_cmd,
933 iscsi_cmd);
934
935 pr_debug("Entering isert_release_cmd %p >>>>>>>>>>>>>>>.\n", isert_cmd);
936
937 kfree(cmd->buf_ptr);
938 kfree(cmd->tmr_req);
939
940 kmem_cache_free(isert_cmd_cache, isert_cmd);
941}
942
943static struct iscsi_cmd
944*isert_alloc_cmd(struct iscsi_conn *conn, gfp_t gfp)
945{
946 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
947 struct isert_cmd *isert_cmd;
948
949 isert_cmd = kmem_cache_zalloc(isert_cmd_cache, gfp);
950 if (!isert_cmd) {
951 pr_err("Unable to allocate isert_cmd\n");
952 return NULL;
953 }
954 isert_cmd->conn = isert_conn;
955 isert_cmd->iscsi_cmd.release_cmd = &isert_release_cmd;
956
957 return &isert_cmd->iscsi_cmd;
958}
959
960static int
961isert_handle_scsi_cmd(struct isert_conn *isert_conn,
962 struct isert_cmd *isert_cmd, struct iser_rx_desc *rx_desc,
963 unsigned char *buf)
964{
965 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
966 struct iscsi_conn *conn = isert_conn->conn;
967 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf;
968 struct scatterlist *sg;
969 int imm_data, imm_data_len, unsol_data, sg_nents, rc;
970 bool dump_payload = false;
971
972 rc = iscsit_setup_scsi_cmd(conn, cmd, buf);
973 if (rc < 0)
974 return rc;
975
976 imm_data = cmd->immediate_data;
977 imm_data_len = cmd->first_burst_len;
978 unsol_data = cmd->unsolicited_data;
979
980 rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
981 if (rc < 0) {
982 return 0;
983 } else if (rc > 0) {
984 dump_payload = true;
985 goto sequence_cmd;
986 }
987
988 if (!imm_data)
989 return 0;
990
991 sg = &cmd->se_cmd.t_data_sg[0];
992 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE));
993
994 pr_debug("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n",
995 sg, sg_nents, &rx_desc->data[0], imm_data_len);
996
997 sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len);
998
999 cmd->write_data_done += imm_data_len;
1000
1001 if (cmd->write_data_done == cmd->se_cmd.data_length) {
1002 spin_lock_bh(&cmd->istate_lock);
1003 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1004 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1005 spin_unlock_bh(&cmd->istate_lock);
1006 }
1007
1008sequence_cmd:
Nicholas Bellingeradb97c22013-07-30 04:04:02 +00001009 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001010
1011 if (!rc && dump_payload == false && unsol_data)
1012 iscsit_set_unsoliticed_dataout(cmd);
Nicholas Bellinger553e4c52014-05-23 00:48:35 -07001013 else if (dump_payload && imm_data)
1014 target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001015
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001016 return 0;
1017}
1018
1019static int
1020isert_handle_iscsi_dataout(struct isert_conn *isert_conn,
1021 struct iser_rx_desc *rx_desc, unsigned char *buf)
1022{
1023 struct scatterlist *sg_start;
1024 struct iscsi_conn *conn = isert_conn->conn;
1025 struct iscsi_cmd *cmd = NULL;
1026 struct iscsi_data *hdr = (struct iscsi_data *)buf;
1027 u32 unsol_data_len = ntoh24(hdr->dlength);
1028 int rc, sg_nents, sg_off, page_off;
1029
1030 rc = iscsit_check_dataout_hdr(conn, buf, &cmd);
1031 if (rc < 0)
1032 return rc;
1033 else if (!cmd)
1034 return 0;
1035 /*
1036 * FIXME: Unexpected unsolicited_data out
1037 */
1038 if (!cmd->unsolicited_data) {
1039 pr_err("Received unexpected solicited data payload\n");
1040 dump_stack();
1041 return -1;
1042 }
1043
1044 pr_debug("Unsolicited DataOut unsol_data_len: %u, write_data_done: %u, data_length: %u\n",
1045 unsol_data_len, cmd->write_data_done, cmd->se_cmd.data_length);
1046
1047 sg_off = cmd->write_data_done / PAGE_SIZE;
1048 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1049 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE));
1050 page_off = cmd->write_data_done % PAGE_SIZE;
1051 /*
1052 * FIXME: Non page-aligned unsolicited_data out
1053 */
1054 if (page_off) {
1055 pr_err("Received unexpected non-page aligned data payload\n");
1056 dump_stack();
1057 return -1;
1058 }
1059 pr_debug("Copying DataOut: sg_start: %p, sg_off: %u sg_nents: %u from %p %u\n",
1060 sg_start, sg_off, sg_nents, &rx_desc->data[0], unsol_data_len);
1061
1062 sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0],
1063 unsol_data_len);
1064
1065 rc = iscsit_check_dataout_payload(cmd, hdr, false);
1066 if (rc < 0)
1067 return rc;
1068
1069 return 0;
1070}
1071
1072static int
1073isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc,
1074 uint32_t read_stag, uint64_t read_va,
1075 uint32_t write_stag, uint64_t write_va)
1076{
1077 struct iscsi_hdr *hdr = &rx_desc->iscsi_header;
1078 struct iscsi_conn *conn = isert_conn->conn;
1079 struct iscsi_cmd *cmd;
1080 struct isert_cmd *isert_cmd;
1081 int ret = -EINVAL;
1082 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1083
1084 switch (opcode) {
1085 case ISCSI_OP_SCSI_CMD:
1086 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1087 if (!cmd)
1088 break;
1089
1090 isert_cmd = container_of(cmd, struct isert_cmd, iscsi_cmd);
1091 isert_cmd->read_stag = read_stag;
1092 isert_cmd->read_va = read_va;
1093 isert_cmd->write_stag = write_stag;
1094 isert_cmd->write_va = write_va;
1095
1096 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd,
1097 rx_desc, (unsigned char *)hdr);
1098 break;
1099 case ISCSI_OP_NOOP_OUT:
1100 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1101 if (!cmd)
1102 break;
1103
1104 ret = iscsit_handle_nop_out(conn, cmd, (unsigned char *)hdr);
1105 break;
1106 case ISCSI_OP_SCSI_DATA_OUT:
1107 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc,
1108 (unsigned char *)hdr);
1109 break;
1110 case ISCSI_OP_SCSI_TMFUNC:
1111 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1112 if (!cmd)
1113 break;
1114
1115 ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1116 (unsigned char *)hdr);
1117 break;
1118 case ISCSI_OP_LOGOUT:
1119 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1120 if (!cmd)
1121 break;
1122
1123 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
1124 if (ret > 0)
1125 wait_for_completion_timeout(&conn->conn_logout_comp,
1126 SECONDS_FOR_LOGOUT_COMP *
1127 HZ);
1128 break;
1129 default:
1130 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1131 dump_stack();
1132 break;
1133 }
1134
1135 return ret;
1136}
1137
1138static void
1139isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn)
1140{
1141 struct iser_hdr *iser_hdr = &rx_desc->iser_header;
1142 uint64_t read_va = 0, write_va = 0;
1143 uint32_t read_stag = 0, write_stag = 0;
1144 int rc;
1145
1146 switch (iser_hdr->flags & 0xF0) {
1147 case ISCSI_CTRL:
1148 if (iser_hdr->flags & ISER_RSV) {
1149 read_stag = be32_to_cpu(iser_hdr->read_stag);
1150 read_va = be64_to_cpu(iser_hdr->read_va);
1151 pr_debug("ISER_RSV: read_stag: 0x%08x read_va: 0x%16llx\n",
1152 read_stag, (unsigned long long)read_va);
1153 }
1154 if (iser_hdr->flags & ISER_WSV) {
1155 write_stag = be32_to_cpu(iser_hdr->write_stag);
1156 write_va = be64_to_cpu(iser_hdr->write_va);
1157 pr_debug("ISER_WSV: write__stag: 0x%08x write_va: 0x%16llx\n",
1158 write_stag, (unsigned long long)write_va);
1159 }
1160
1161 pr_debug("ISER ISCSI_CTRL PDU\n");
1162 break;
1163 case ISER_HELLO:
1164 pr_err("iSER Hello message\n");
1165 break;
1166 default:
1167 pr_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags);
1168 break;
1169 }
1170
1171 rc = isert_rx_opcode(isert_conn, rx_desc,
1172 read_stag, read_va, write_stag, write_va);
1173}
1174
1175static void
1176isert_rx_completion(struct iser_rx_desc *desc, struct isert_conn *isert_conn,
1177 unsigned long xfer_len)
1178{
1179 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1180 struct iscsi_hdr *hdr;
1181 u64 rx_dma;
1182 int rx_buflen, outstanding;
1183
1184 if ((char *)desc == isert_conn->login_req_buf) {
1185 rx_dma = isert_conn->login_req_dma;
1186 rx_buflen = ISER_RX_LOGIN_SIZE;
1187 pr_debug("ISER login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n",
1188 rx_dma, rx_buflen);
1189 } else {
1190 rx_dma = desc->dma_addr;
1191 rx_buflen = ISER_RX_PAYLOAD_SIZE;
1192 pr_debug("ISER req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n",
1193 rx_dma, rx_buflen);
1194 }
1195
1196 ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE);
1197
1198 hdr = &desc->iscsi_header;
1199 pr_debug("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n",
1200 hdr->opcode, hdr->itt, hdr->flags,
1201 (int)(xfer_len - ISER_HEADERS_LEN));
1202
1203 if ((char *)desc == isert_conn->login_req_buf)
1204 isert_rx_login_req(desc, xfer_len - ISER_HEADERS_LEN,
1205 isert_conn);
1206 else
1207 isert_rx_do_work(desc, isert_conn);
1208
1209 ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen,
1210 DMA_FROM_DEVICE);
1211
1212 isert_conn->post_recv_buf_count--;
1213 pr_debug("iSERT: Decremented post_recv_buf_count: %d\n",
1214 isert_conn->post_recv_buf_count);
1215
1216 if ((char *)desc == isert_conn->login_req_buf)
1217 return;
1218
1219 outstanding = isert_conn->post_recv_buf_count;
1220 if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) {
1221 int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding,
1222 ISERT_MIN_POSTED_RX);
1223 err = isert_post_recv(isert_conn, count);
1224 if (err) {
1225 pr_err("isert_post_recv() count: %d failed, %d\n",
1226 count, err);
1227 }
1228 }
1229}
1230
1231static void
1232isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn)
1233{
1234 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
1235 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1236
1237 pr_debug("isert_unmap_cmd >>>>>>>>>>>>>>>>>>>>>>>\n");
1238
1239 if (wr->sge) {
1240 ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, DMA_TO_DEVICE);
1241 wr->sge = NULL;
1242 }
1243
1244 kfree(wr->send_wr);
1245 wr->send_wr = NULL;
1246
1247 kfree(isert_cmd->ib_sge);
1248 isert_cmd->ib_sge = NULL;
1249}
1250
1251static void
Nicholas Bellingerd3066062014-03-30 15:50:03 -07001252isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001253{
1254 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
1255 struct isert_conn *isert_conn = isert_cmd->conn;
Nicholas Bellingerc6ccbb92013-07-03 03:11:48 -07001256 struct iscsi_conn *conn = isert_conn->conn;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001257
1258 pr_debug("Entering isert_put_cmd: %p\n", isert_cmd);
1259
1260 switch (cmd->iscsi_opcode) {
1261 case ISCSI_OP_SCSI_CMD:
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001262 spin_lock_bh(&conn->cmd_lock);
1263 if (!list_empty(&cmd->i_conn_node))
Nicholas Bellingeraf737f62014-02-03 12:53:51 -08001264 list_del_init(&cmd->i_conn_node);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001265 spin_unlock_bh(&conn->cmd_lock);
1266
Nicholas Bellingerd3066062014-03-30 15:50:03 -07001267 if (cmd->data_direction == DMA_TO_DEVICE) {
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001268 iscsit_stop_dataout_timer(cmd);
Nicholas Bellingerd3066062014-03-30 15:50:03 -07001269 /*
1270 * Check for special case during comp_err where
1271 * WRITE_PENDING has been handed off from core,
1272 * but requires an extra target_put_sess_cmd()
1273 * before transport_generic_free_cmd() below.
1274 */
1275 if (comp_err &&
1276 cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) {
1277 struct se_cmd *se_cmd = &cmd->se_cmd;
1278
1279 target_put_sess_cmd(se_cmd->se_sess, se_cmd);
1280 }
1281 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001282
1283 isert_unmap_cmd(isert_cmd, isert_conn);
Nicholas Bellingerc6ccbb92013-07-03 03:11:48 -07001284 transport_generic_free_cmd(&cmd->se_cmd, 0);
1285 break;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001286 case ISCSI_OP_SCSI_TMFUNC:
Nicholas Bellingerc6ccbb92013-07-03 03:11:48 -07001287 spin_lock_bh(&conn->cmd_lock);
1288 if (!list_empty(&cmd->i_conn_node))
Nicholas Bellingeraf737f62014-02-03 12:53:51 -08001289 list_del_init(&cmd->i_conn_node);
Nicholas Bellingerc6ccbb92013-07-03 03:11:48 -07001290 spin_unlock_bh(&conn->cmd_lock);
1291
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001292 transport_generic_free_cmd(&cmd->se_cmd, 0);
1293 break;
1294 case ISCSI_OP_REJECT:
1295 case ISCSI_OP_NOOP_OUT:
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001296 spin_lock_bh(&conn->cmd_lock);
1297 if (!list_empty(&cmd->i_conn_node))
Nicholas Bellingeraf737f62014-02-03 12:53:51 -08001298 list_del_init(&cmd->i_conn_node);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001299 spin_unlock_bh(&conn->cmd_lock);
1300
1301 /*
1302 * Handle special case for REJECT when iscsi_add_reject*() has
1303 * overwritten the original iscsi_opcode assignment, and the
1304 * associated cmd->se_cmd needs to be released.
1305 */
1306 if (cmd->se_cmd.se_tfo != NULL) {
Nicholas Bellingerfff98872013-06-26 02:31:42 -07001307 pr_debug("Calling transport_generic_free_cmd from"
1308 " isert_put_cmd for 0x%02x\n",
1309 cmd->iscsi_opcode);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001310 transport_generic_free_cmd(&cmd->se_cmd, 0);
1311 break;
1312 }
1313 /*
1314 * Fall-through
1315 */
1316 default:
1317 isert_release_cmd(cmd);
1318 break;
1319 }
1320}
1321
1322static void
1323isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev)
1324{
1325 if (tx_desc->dma_addr != 0) {
1326 pr_debug("Calling ib_dma_unmap_single for tx_desc->dma_addr\n");
1327 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr,
1328 ISER_HEADERS_LEN, DMA_TO_DEVICE);
1329 tx_desc->dma_addr = 0;
1330 }
1331}
1332
1333static void
1334isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd,
Nicholas Bellingerd3066062014-03-30 15:50:03 -07001335 struct ib_device *ib_dev, bool comp_err)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001336{
1337 if (isert_cmd->sense_buf_dma != 0) {
1338 pr_debug("Calling ib_dma_unmap_single for isert_cmd->sense_buf_dma\n");
1339 ib_dma_unmap_single(ib_dev, isert_cmd->sense_buf_dma,
1340 isert_cmd->sense_buf_len, DMA_TO_DEVICE);
1341 isert_cmd->sense_buf_dma = 0;
1342 }
1343
1344 isert_unmap_tx_desc(tx_desc, ib_dev);
Nicholas Bellingerd3066062014-03-30 15:50:03 -07001345 isert_put_cmd(isert_cmd, comp_err);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001346}
1347
1348static void
1349isert_completion_rdma_read(struct iser_tx_desc *tx_desc,
1350 struct isert_cmd *isert_cmd)
1351{
1352 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
1353 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
1354 struct se_cmd *se_cmd = &cmd->se_cmd;
1355 struct ib_device *ib_dev = isert_cmd->conn->conn_cm_id->device;
1356
1357 iscsit_stop_dataout_timer(cmd);
1358
1359 if (wr->sge) {
1360 pr_debug("isert_do_rdma_read_comp: Unmapping wr->sge from t_data_sg\n");
1361 ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, DMA_TO_DEVICE);
1362 wr->sge = NULL;
1363 }
1364
1365 if (isert_cmd->ib_sge) {
1366 pr_debug("isert_do_rdma_read_comp: Freeing isert_cmd->ib_sge\n");
1367 kfree(isert_cmd->ib_sge);
1368 isert_cmd->ib_sge = NULL;
1369 }
1370
1371 cmd->write_data_done = se_cmd->data_length;
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001372 wr->send_wr_num = 0;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001373
1374 pr_debug("isert_do_rdma_read_comp, calling target_execute_cmd\n");
1375 spin_lock_bh(&cmd->istate_lock);
1376 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1377 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1378 spin_unlock_bh(&cmd->istate_lock);
1379
1380 target_execute_cmd(se_cmd);
1381}
1382
1383static void
1384isert_do_control_comp(struct work_struct *work)
1385{
1386 struct isert_cmd *isert_cmd = container_of(work,
1387 struct isert_cmd, comp_work);
1388 struct isert_conn *isert_conn = isert_cmd->conn;
1389 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1390 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
1391
1392 switch (cmd->i_state) {
1393 case ISTATE_SEND_TASKMGTRSP:
1394 pr_debug("Calling iscsit_tmr_post_handler >>>>>>>>>>>>>>>>>\n");
1395
1396 atomic_dec(&isert_conn->post_send_buf_count);
1397 iscsit_tmr_post_handler(cmd, cmd->conn);
1398
1399 cmd->i_state = ISTATE_SENT_STATUS;
Nicholas Bellingerd3066062014-03-30 15:50:03 -07001400 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001401 break;
1402 case ISTATE_SEND_REJECT:
1403 pr_debug("Got isert_do_control_comp ISTATE_SEND_REJECT: >>>\n");
1404 atomic_dec(&isert_conn->post_send_buf_count);
1405
1406 cmd->i_state = ISTATE_SENT_STATUS;
Nicholas Bellingerd3066062014-03-30 15:50:03 -07001407 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false);
Nicholas Bellingerfff98872013-06-26 02:31:42 -07001408 break;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001409 case ISTATE_SEND_LOGOUTRSP:
1410 pr_debug("Calling iscsit_logout_post_handler >>>>>>>>>>>>>>\n");
Sagi Grimberg1813e802014-05-19 17:44:23 +03001411
1412 atomic_dec(&isert_conn->post_send_buf_count);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001413 iscsit_logout_post_handler(cmd, cmd->conn);
1414 break;
1415 default:
1416 pr_err("Unknown do_control_comp i_state %d\n", cmd->i_state);
1417 dump_stack();
1418 break;
1419 }
1420}
1421
1422static void
1423isert_response_completion(struct iser_tx_desc *tx_desc,
1424 struct isert_cmd *isert_cmd,
1425 struct isert_conn *isert_conn,
1426 struct ib_device *ib_dev)
1427{
1428 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001429 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001430
1431 if (cmd->i_state == ISTATE_SEND_TASKMGTRSP ||
Nicholas Bellingerfff98872013-06-26 02:31:42 -07001432 cmd->i_state == ISTATE_SEND_LOGOUTRSP ||
1433 cmd->i_state == ISTATE_SEND_REJECT) {
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001434 isert_unmap_tx_desc(tx_desc, ib_dev);
1435
1436 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp);
1437 queue_work(isert_comp_wq, &isert_cmd->comp_work);
1438 return;
1439 }
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001440 atomic_sub(wr->send_wr_num + 1, &isert_conn->post_send_buf_count);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001441
1442 cmd->i_state = ISTATE_SENT_STATUS;
Nicholas Bellingerd3066062014-03-30 15:50:03 -07001443 isert_completion_put(tx_desc, isert_cmd, ib_dev, false);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001444}
1445
1446static void
1447isert_send_completion(struct iser_tx_desc *tx_desc,
1448 struct isert_conn *isert_conn)
1449{
1450 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1451 struct isert_cmd *isert_cmd = tx_desc->isert_cmd;
1452 struct isert_rdma_wr *wr;
1453
1454 if (!isert_cmd) {
1455 atomic_dec(&isert_conn->post_send_buf_count);
1456 isert_unmap_tx_desc(tx_desc, ib_dev);
1457 return;
1458 }
1459 wr = &isert_cmd->rdma_wr;
1460
1461 switch (wr->iser_ib_op) {
1462 case ISER_IB_RECV:
1463 pr_err("isert_send_completion: Got ISER_IB_RECV\n");
1464 dump_stack();
1465 break;
1466 case ISER_IB_SEND:
1467 pr_debug("isert_send_completion: Got ISER_IB_SEND\n");
1468 isert_response_completion(tx_desc, isert_cmd,
1469 isert_conn, ib_dev);
1470 break;
1471 case ISER_IB_RDMA_WRITE:
1472 pr_err("isert_send_completion: Got ISER_IB_RDMA_WRITE\n");
1473 dump_stack();
1474 break;
1475 case ISER_IB_RDMA_READ:
1476 pr_debug("isert_send_completion: Got ISER_IB_RDMA_READ:\n");
1477
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001478 atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001479 isert_completion_rdma_read(tx_desc, isert_cmd);
1480 break;
1481 default:
1482 pr_err("Unknown wr->iser_ib_op: 0x%02x\n", wr->iser_ib_op);
1483 dump_stack();
1484 break;
1485 }
1486}
1487
1488static void
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001489isert_cq_tx_comp_err(struct iser_tx_desc *tx_desc, struct isert_conn *isert_conn)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001490{
1491 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001492 struct isert_cmd *isert_cmd = tx_desc->isert_cmd;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001493
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001494 if (!isert_cmd)
1495 isert_unmap_tx_desc(tx_desc, ib_dev);
1496 else
Nicholas Bellingerd3066062014-03-30 15:50:03 -07001497 isert_completion_put(tx_desc, isert_cmd, ib_dev, true);
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001498}
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001499
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001500static void
1501isert_cq_rx_comp_err(struct isert_conn *isert_conn)
1502{
1503 struct iscsi_conn *conn = isert_conn->conn;
1504
1505 if (isert_conn->post_recv_buf_count)
1506 return;
1507
1508 if (conn->sess) {
1509 target_sess_cmd_list_set_waiting(conn->sess->se_sess);
1510 target_wait_for_sess_cmds(conn->sess->se_sess);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001511 }
1512
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001513 while (atomic_read(&isert_conn->post_send_buf_count))
1514 msleep(3000);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001515
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001516 mutex_lock(&isert_conn->conn_mutex);
Sagi Grimberg839eac52015-01-30 22:17:25 +00001517 isert_conn_terminate(isert_conn);
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001518 mutex_unlock(&isert_conn->conn_mutex);
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -07001519
Sagi Grimberg1813e802014-05-19 17:44:23 +03001520 iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
1521
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001522 complete(&isert_conn->conn_wait_comp_err);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001523}
1524
1525static void
1526isert_cq_tx_work(struct work_struct *work)
1527{
1528 struct isert_cq_desc *cq_desc = container_of(work,
1529 struct isert_cq_desc, cq_tx_work);
1530 struct isert_device *device = cq_desc->device;
1531 int cq_index = cq_desc->cq_index;
1532 struct ib_cq *tx_cq = device->dev_tx_cq[cq_index];
1533 struct isert_conn *isert_conn;
1534 struct iser_tx_desc *tx_desc;
1535 struct ib_wc wc;
1536
1537 while (ib_poll_cq(tx_cq, 1, &wc) == 1) {
1538 tx_desc = (struct iser_tx_desc *)(unsigned long)wc.wr_id;
1539 isert_conn = wc.qp->qp_context;
1540
1541 if (wc.status == IB_WC_SUCCESS) {
1542 isert_send_completion(tx_desc, isert_conn);
1543 } else {
1544 pr_debug("TX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n");
1545 pr_debug("TX wc.status: 0x%08x\n", wc.status);
1546 atomic_dec(&isert_conn->post_send_buf_count);
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001547 isert_cq_tx_comp_err(tx_desc, isert_conn);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001548 }
1549 }
1550
1551 ib_req_notify_cq(tx_cq, IB_CQ_NEXT_COMP);
1552}
1553
1554static void
1555isert_cq_tx_callback(struct ib_cq *cq, void *context)
1556{
1557 struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context;
1558
1559 INIT_WORK(&cq_desc->cq_tx_work, isert_cq_tx_work);
1560 queue_work(isert_comp_wq, &cq_desc->cq_tx_work);
1561}
1562
1563static void
1564isert_cq_rx_work(struct work_struct *work)
1565{
1566 struct isert_cq_desc *cq_desc = container_of(work,
1567 struct isert_cq_desc, cq_rx_work);
1568 struct isert_device *device = cq_desc->device;
1569 int cq_index = cq_desc->cq_index;
1570 struct ib_cq *rx_cq = device->dev_rx_cq[cq_index];
1571 struct isert_conn *isert_conn;
1572 struct iser_rx_desc *rx_desc;
1573 struct ib_wc wc;
1574 unsigned long xfer_len;
1575
1576 while (ib_poll_cq(rx_cq, 1, &wc) == 1) {
1577 rx_desc = (struct iser_rx_desc *)(unsigned long)wc.wr_id;
1578 isert_conn = wc.qp->qp_context;
1579
1580 if (wc.status == IB_WC_SUCCESS) {
1581 xfer_len = (unsigned long)wc.byte_len;
1582 isert_rx_completion(rx_desc, isert_conn, xfer_len);
1583 } else {
1584 pr_debug("RX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n");
1585 if (wc.status != IB_WC_WR_FLUSH_ERR)
1586 pr_debug("RX wc.status: 0x%08x\n", wc.status);
1587
1588 isert_conn->post_recv_buf_count--;
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001589 isert_cq_rx_comp_err(isert_conn);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001590 }
1591 }
1592
1593 ib_req_notify_cq(rx_cq, IB_CQ_NEXT_COMP);
1594}
1595
1596static void
1597isert_cq_rx_callback(struct ib_cq *cq, void *context)
1598{
1599 struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context;
1600
1601 INIT_WORK(&cq_desc->cq_rx_work, isert_cq_rx_work);
1602 queue_work(isert_rx_wq, &cq_desc->cq_rx_work);
1603}
1604
1605static int
1606isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd)
1607{
1608 struct ib_send_wr *wr_failed;
1609 int ret;
1610
1611 atomic_inc(&isert_conn->post_send_buf_count);
1612
1613 ret = ib_post_send(isert_conn->conn_qp, &isert_cmd->tx_desc.send_wr,
1614 &wr_failed);
1615 if (ret) {
1616 pr_err("ib_post_send failed with %d\n", ret);
1617 atomic_dec(&isert_conn->post_send_buf_count);
1618 return ret;
1619 }
1620 return ret;
1621}
1622
1623static int
1624isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
1625{
1626 struct isert_cmd *isert_cmd = container_of(cmd,
1627 struct isert_cmd, iscsi_cmd);
1628 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1629 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1630 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)
1631 &isert_cmd->tx_desc.iscsi_header;
1632
1633 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1634 iscsit_build_rsp_pdu(cmd, conn, true, hdr);
1635 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1636 /*
1637 * Attach SENSE DATA payload to iSCSI Response PDU
1638 */
1639 if (cmd->se_cmd.sense_buffer &&
1640 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
1641 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
1642 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1643 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
1644 u32 padding, sense_len;
1645
1646 put_unaligned_be16(cmd->se_cmd.scsi_sense_length,
1647 cmd->sense_buffer);
1648 cmd->se_cmd.scsi_sense_length += sizeof(__be16);
1649
1650 padding = -(cmd->se_cmd.scsi_sense_length) & 3;
1651 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length);
1652 sense_len = cmd->se_cmd.scsi_sense_length + padding;
1653
1654 isert_cmd->sense_buf_dma = ib_dma_map_single(ib_dev,
1655 (void *)cmd->sense_buffer, sense_len,
1656 DMA_TO_DEVICE);
1657
1658 isert_cmd->sense_buf_len = sense_len;
1659 tx_dsg->addr = isert_cmd->sense_buf_dma;
1660 tx_dsg->length = sense_len;
1661 tx_dsg->lkey = isert_conn->conn_mr->lkey;
1662 isert_cmd->tx_desc.num_sge = 2;
1663 }
1664
1665 isert_init_send_wr(isert_cmd, send_wr);
1666
1667 pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1668
1669 return isert_post_response(isert_conn, isert_cmd);
1670}
1671
1672static int
1673isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
1674 bool nopout_response)
1675{
1676 struct isert_cmd *isert_cmd = container_of(cmd,
1677 struct isert_cmd, iscsi_cmd);
1678 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1679 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1680
1681 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1682 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *)
1683 &isert_cmd->tx_desc.iscsi_header,
1684 nopout_response);
1685 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1686 isert_init_send_wr(isert_cmd, send_wr);
1687
1688 pr_debug("Posting NOPIN Reponse IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1689
1690 return isert_post_response(isert_conn, isert_cmd);
1691}
1692
1693static int
1694isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1695{
1696 struct isert_cmd *isert_cmd = container_of(cmd,
1697 struct isert_cmd, iscsi_cmd);
1698 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1699 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1700
1701 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1702 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *)
1703 &isert_cmd->tx_desc.iscsi_header);
1704 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1705 isert_init_send_wr(isert_cmd, send_wr);
1706
1707 pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1708
1709 return isert_post_response(isert_conn, isert_cmd);
1710}
1711
1712static int
1713isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1714{
1715 struct isert_cmd *isert_cmd = container_of(cmd,
1716 struct isert_cmd, iscsi_cmd);
1717 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1718 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1719
1720 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1721 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *)
1722 &isert_cmd->tx_desc.iscsi_header);
1723 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1724 isert_init_send_wr(isert_cmd, send_wr);
1725
1726 pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1727
1728 return isert_post_response(isert_conn, isert_cmd);
1729}
1730
1731static int
1732isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1733{
1734 struct isert_cmd *isert_cmd = container_of(cmd,
1735 struct isert_cmd, iscsi_cmd);
1736 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1737 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
Nicholas Bellingerfff98872013-06-26 02:31:42 -07001738 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1739 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
1740 struct iscsi_reject *hdr =
1741 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001742
1743 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
Nicholas Bellingerfff98872013-06-26 02:31:42 -07001744 iscsit_build_reject(cmd, conn, hdr);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001745 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
Nicholas Bellingerfff98872013-06-26 02:31:42 -07001746
1747 hton24(hdr->dlength, ISCSI_HDR_LEN);
1748 isert_cmd->sense_buf_dma = ib_dma_map_single(ib_dev,
1749 (void *)cmd->buf_ptr, ISCSI_HDR_LEN,
1750 DMA_TO_DEVICE);
1751 isert_cmd->sense_buf_len = ISCSI_HDR_LEN;
1752 tx_dsg->addr = isert_cmd->sense_buf_dma;
1753 tx_dsg->length = ISCSI_HDR_LEN;
1754 tx_dsg->lkey = isert_conn->conn_mr->lkey;
1755 isert_cmd->tx_desc.num_sge = 2;
1756
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001757 isert_init_send_wr(isert_cmd, send_wr);
1758
1759 pr_debug("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1760
1761 return isert_post_response(isert_conn, isert_cmd);
1762}
1763
1764static int
1765isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
1766 struct ib_sge *ib_sge, struct ib_send_wr *send_wr,
1767 u32 data_left, u32 offset)
1768{
1769 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
1770 struct scatterlist *sg_start, *tmp_sg;
1771 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1772 u32 sg_off, page_off;
1773 int i = 0, sg_nents;
1774
1775 sg_off = offset / PAGE_SIZE;
1776 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1777 sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge);
1778 page_off = offset % PAGE_SIZE;
1779
1780 send_wr->sg_list = ib_sge;
1781 send_wr->num_sge = sg_nents;
1782 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc;
1783 /*
1784 * Perform mapping of TCM scatterlist memory ib_sge dma_addr.
1785 */
1786 for_each_sg(sg_start, tmp_sg, sg_nents, i) {
1787 pr_debug("ISER RDMA from SGL dma_addr: 0x%16llx dma_len: %u, page_off: %u\n",
1788 (unsigned long long)tmp_sg->dma_address,
1789 tmp_sg->length, page_off);
1790
1791 ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off;
1792 ib_sge->length = min_t(u32, data_left,
1793 ib_sg_dma_len(ib_dev, tmp_sg) - page_off);
1794 ib_sge->lkey = isert_conn->conn_mr->lkey;
1795
1796 pr_debug("RDMA ib_sge: addr: 0x%16llx length: %u\n",
1797 ib_sge->addr, ib_sge->length);
1798 page_off = 0;
1799 data_left -= ib_sge->length;
1800 ib_sge++;
1801 pr_debug("Incrementing ib_sge pointer to %p\n", ib_sge);
1802 }
1803
1804 pr_debug("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n",
1805 send_wr->sg_list, send_wr->num_sge);
1806
1807 return sg_nents;
1808}
1809
1810static int
1811isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
1812{
1813 struct se_cmd *se_cmd = &cmd->se_cmd;
1814 struct isert_cmd *isert_cmd = container_of(cmd,
1815 struct isert_cmd, iscsi_cmd);
1816 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
1817 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1818 struct ib_send_wr *wr_failed, *send_wr;
1819 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1820 struct ib_sge *ib_sge;
1821 struct scatterlist *sg;
1822 u32 offset = 0, data_len, data_left, rdma_write_max;
1823 int rc, ret = 0, count, sg_nents, i, ib_sge_cnt;
1824
1825 pr_debug("RDMA_WRITE: data_length: %u\n", se_cmd->data_length);
1826
1827 sg = &se_cmd->t_data_sg[0];
1828 sg_nents = se_cmd->t_data_nents;
1829
1830 count = ib_dma_map_sg(ib_dev, sg, sg_nents, DMA_TO_DEVICE);
1831 if (unlikely(!count)) {
1832 pr_err("Unable to map put_datain SGs\n");
1833 return -EINVAL;
1834 }
1835 wr->sge = sg;
1836 wr->num_sge = sg_nents;
1837 pr_debug("Mapped IB count: %u sg: %p sg_nents: %u for RDMA_WRITE\n",
1838 count, sg, sg_nents);
1839
1840 ib_sge = kzalloc(sizeof(struct ib_sge) * sg_nents, GFP_KERNEL);
1841 if (!ib_sge) {
1842 pr_warn("Unable to allocate datain ib_sge\n");
1843 ret = -ENOMEM;
1844 goto unmap_sg;
1845 }
1846 isert_cmd->ib_sge = ib_sge;
1847
1848 pr_debug("Allocated ib_sge: %p from t_data_ents: %d for RDMA_WRITE\n",
1849 ib_sge, se_cmd->t_data_nents);
1850
1851 wr->send_wr_num = DIV_ROUND_UP(sg_nents, isert_conn->max_sge);
1852 wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num,
1853 GFP_KERNEL);
1854 if (!wr->send_wr) {
1855 pr_err("Unable to allocate wr->send_wr\n");
1856 ret = -ENOMEM;
1857 goto unmap_sg;
1858 }
1859 pr_debug("Allocated wr->send_wr: %p wr->send_wr_num: %u\n",
1860 wr->send_wr, wr->send_wr_num);
1861
1862 iscsit_increment_maxcmdsn(cmd, conn->sess);
1863 cmd->stat_sn = conn->stat_sn++;
1864
1865 wr->isert_cmd = isert_cmd;
1866 rdma_write_max = isert_conn->max_sge * PAGE_SIZE;
1867 data_left = se_cmd->data_length;
1868
1869 for (i = 0; i < wr->send_wr_num; i++) {
1870 send_wr = &isert_cmd->rdma_wr.send_wr[i];
1871 data_len = min(data_left, rdma_write_max);
1872
1873 send_wr->opcode = IB_WR_RDMA_WRITE;
1874 send_wr->send_flags = 0;
1875 send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset;
1876 send_wr->wr.rdma.rkey = isert_cmd->read_stag;
1877
1878 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge,
1879 send_wr, data_len, offset);
1880 ib_sge += ib_sge_cnt;
1881
1882 if (i + 1 == wr->send_wr_num)
1883 send_wr->next = &isert_cmd->tx_desc.send_wr;
1884 else
1885 send_wr->next = &wr->send_wr[i + 1];
1886
1887 offset += data_len;
1888 data_left -= data_len;
1889 }
1890 /*
1891 * Build isert_conn->tx_desc for iSCSI response PDU and attach
1892 */
1893 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1894 iscsit_build_rsp_pdu(cmd, conn, false, (struct iscsi_scsi_rsp *)
1895 &isert_cmd->tx_desc.iscsi_header);
1896 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1897 isert_init_send_wr(isert_cmd, &isert_cmd->tx_desc.send_wr);
1898
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001899 atomic_add(wr->send_wr_num + 1, &isert_conn->post_send_buf_count);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001900
1901 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed);
1902 if (rc) {
1903 pr_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n");
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001904 atomic_sub(wr->send_wr_num + 1, &isert_conn->post_send_buf_count);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001905 }
1906 pr_debug("Posted RDMA_WRITE + Response for iSER Data READ\n");
1907 return 1;
1908
1909unmap_sg:
1910 ib_dma_unmap_sg(ib_dev, sg, sg_nents, DMA_TO_DEVICE);
1911 return ret;
1912}
1913
1914static int
1915isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery)
1916{
1917 struct se_cmd *se_cmd = &cmd->se_cmd;
1918 struct isert_cmd *isert_cmd = container_of(cmd,
1919 struct isert_cmd, iscsi_cmd);
1920 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
1921 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1922 struct ib_send_wr *wr_failed, *send_wr;
1923 struct ib_sge *ib_sge;
1924 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1925 struct scatterlist *sg_start;
1926 u32 sg_off, sg_nents, page_off, va_offset = 0;
1927 u32 offset = 0, data_len, data_left, rdma_write_max;
1928 int rc, ret = 0, count, i, ib_sge_cnt;
1929
1930 pr_debug("RDMA_READ: data_length: %u write_data_done: %u\n",
1931 se_cmd->data_length, cmd->write_data_done);
1932
1933 sg_off = cmd->write_data_done / PAGE_SIZE;
1934 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1935 page_off = cmd->write_data_done % PAGE_SIZE;
1936
1937 pr_debug("RDMA_READ: sg_off: %d, sg_start: %p page_off: %d\n",
1938 sg_off, sg_start, page_off);
1939
1940 data_left = se_cmd->data_length - cmd->write_data_done;
1941 sg_nents = se_cmd->t_data_nents - sg_off;
1942
1943 pr_debug("RDMA_READ: data_left: %d, sg_nents: %d\n",
1944 data_left, sg_nents);
1945
1946 count = ib_dma_map_sg(ib_dev, sg_start, sg_nents, DMA_FROM_DEVICE);
1947 if (unlikely(!count)) {
1948 pr_err("Unable to map get_dataout SGs\n");
1949 return -EINVAL;
1950 }
1951 wr->sge = sg_start;
1952 wr->num_sge = sg_nents;
1953 pr_debug("Mapped IB count: %u sg_start: %p sg_nents: %u for RDMA_READ\n",
1954 count, sg_start, sg_nents);
1955
1956 ib_sge = kzalloc(sizeof(struct ib_sge) * sg_nents, GFP_KERNEL);
1957 if (!ib_sge) {
1958 pr_warn("Unable to allocate dataout ib_sge\n");
1959 ret = -ENOMEM;
1960 goto unmap_sg;
1961 }
1962 isert_cmd->ib_sge = ib_sge;
1963
1964 pr_debug("Using ib_sge: %p from sg_ents: %d for RDMA_READ\n",
1965 ib_sge, sg_nents);
1966
1967 wr->send_wr_num = DIV_ROUND_UP(sg_nents, isert_conn->max_sge);
1968 wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num,
1969 GFP_KERNEL);
1970 if (!wr->send_wr) {
1971 pr_debug("Unable to allocate wr->send_wr\n");
1972 ret = -ENOMEM;
1973 goto unmap_sg;
1974 }
1975 pr_debug("Allocated wr->send_wr: %p wr->send_wr_num: %u\n",
1976 wr->send_wr, wr->send_wr_num);
1977
1978 isert_cmd->tx_desc.isert_cmd = isert_cmd;
1979
1980 wr->iser_ib_op = ISER_IB_RDMA_READ;
1981 wr->isert_cmd = isert_cmd;
1982 rdma_write_max = isert_conn->max_sge * PAGE_SIZE;
1983 offset = cmd->write_data_done;
1984
1985 for (i = 0; i < wr->send_wr_num; i++) {
1986 send_wr = &isert_cmd->rdma_wr.send_wr[i];
1987 data_len = min(data_left, rdma_write_max);
1988
1989 send_wr->opcode = IB_WR_RDMA_READ;
1990 send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset;
1991 send_wr->wr.rdma.rkey = isert_cmd->write_stag;
1992
1993 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge,
1994 send_wr, data_len, offset);
1995 ib_sge += ib_sge_cnt;
1996
1997 if (i + 1 == wr->send_wr_num)
1998 send_wr->send_flags = IB_SEND_SIGNALED;
1999 else
2000 send_wr->next = &wr->send_wr[i + 1];
2001
2002 offset += data_len;
2003 va_offset += data_len;
2004 data_left -= data_len;
2005 }
2006
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08002007 atomic_add(wr->send_wr_num, &isert_conn->post_send_buf_count);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002008
2009 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed);
2010 if (rc) {
2011 pr_warn("ib_post_send() failed for IB_WR_RDMA_READ\n");
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08002012 atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002013 }
2014 pr_debug("Posted RDMA_READ memory for ISER Data WRITE\n");
2015 return 0;
2016
2017unmap_sg:
2018 ib_dma_unmap_sg(ib_dev, sg_start, sg_nents, DMA_FROM_DEVICE);
2019 return ret;
2020}
2021
2022static int
2023isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
2024{
2025 int ret;
2026
2027 switch (state) {
2028 case ISTATE_SEND_NOPIN_WANT_RESPONSE:
2029 ret = isert_put_nopin(cmd, conn, false);
2030 break;
2031 default:
2032 pr_err("Unknown immediate state: 0x%02x\n", state);
2033 ret = -EINVAL;
2034 break;
2035 }
2036
2037 return ret;
2038}
2039
2040static int
2041isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
2042{
2043 int ret;
2044
2045 switch (state) {
2046 case ISTATE_SEND_LOGOUTRSP:
2047 ret = isert_put_logout_rsp(cmd, conn);
2048 if (!ret) {
2049 pr_debug("Returning iSER Logout -EAGAIN\n");
2050 ret = -EAGAIN;
2051 }
2052 break;
2053 case ISTATE_SEND_NOPIN:
2054 ret = isert_put_nopin(cmd, conn, true);
2055 break;
2056 case ISTATE_SEND_TASKMGTRSP:
2057 ret = isert_put_tm_rsp(cmd, conn);
2058 break;
2059 case ISTATE_SEND_REJECT:
2060 ret = isert_put_reject(cmd, conn);
2061 break;
2062 case ISTATE_SEND_STATUS:
2063 /*
2064 * Special case for sending non GOOD SCSI status from TX thread
2065 * context during pre se_cmd excecution failure.
2066 */
2067 ret = isert_put_response(conn, cmd);
2068 break;
2069 default:
2070 pr_err("Unknown response state: 0x%02x\n", state);
2071 ret = -EINVAL;
2072 break;
2073 }
2074
2075 return ret;
2076}
2077
2078static int
2079isert_setup_np(struct iscsi_np *np,
2080 struct __kernel_sockaddr_storage *ksockaddr)
2081{
2082 struct isert_np *isert_np;
2083 struct rdma_cm_id *isert_lid;
2084 struct sockaddr *sa;
2085 int ret;
2086
2087 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL);
2088 if (!isert_np) {
2089 pr_err("Unable to allocate struct isert_np\n");
2090 return -ENOMEM;
2091 }
Sagi Grimberg8a2629a2014-04-29 13:13:45 +03002092 sema_init(&isert_np->np_sem, 0);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002093 mutex_init(&isert_np->np_accept_mutex);
2094 INIT_LIST_HEAD(&isert_np->np_accept_list);
2095 init_completion(&isert_np->np_login_comp);
2096
2097 sa = (struct sockaddr *)ksockaddr;
2098 pr_debug("ksockaddr: %p, sa: %p\n", ksockaddr, sa);
2099 /*
2100 * Setup the np->np_sockaddr from the passed sockaddr setup
2101 * in iscsi_target_configfs.c code..
2102 */
2103 memcpy(&np->np_sockaddr, ksockaddr,
2104 sizeof(struct __kernel_sockaddr_storage));
2105
2106 isert_lid = rdma_create_id(isert_cma_handler, np, RDMA_PS_TCP,
2107 IB_QPT_RC);
2108 if (IS_ERR(isert_lid)) {
2109 pr_err("rdma_create_id() for isert_listen_handler failed: %ld\n",
2110 PTR_ERR(isert_lid));
2111 ret = PTR_ERR(isert_lid);
2112 goto out;
2113 }
2114
2115 ret = rdma_bind_addr(isert_lid, sa);
2116 if (ret) {
2117 pr_err("rdma_bind_addr() for isert_lid failed: %d\n", ret);
2118 goto out_lid;
2119 }
2120
2121 ret = rdma_listen(isert_lid, ISERT_RDMA_LISTEN_BACKLOG);
2122 if (ret) {
2123 pr_err("rdma_listen() for isert_lid failed: %d\n", ret);
2124 goto out_lid;
2125 }
2126
2127 isert_np->np_cm_id = isert_lid;
2128 np->np_context = isert_np;
2129 pr_debug("Setup isert_lid->context: %p\n", isert_lid->context);
2130
2131 return 0;
2132
2133out_lid:
2134 rdma_destroy_id(isert_lid);
2135out:
2136 kfree(isert_np);
2137 return ret;
2138}
2139
2140static int
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002141isert_rdma_accept(struct isert_conn *isert_conn)
2142{
2143 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id;
2144 struct rdma_conn_param cp;
2145 int ret;
2146
2147 memset(&cp, 0, sizeof(struct rdma_conn_param));
2148 cp.responder_resources = isert_conn->responder_resources;
2149 cp.initiator_depth = isert_conn->initiator_depth;
2150 cp.retry_count = 7;
2151 cp.rnr_retry_count = 7;
2152
2153 pr_debug("Before rdma_accept >>>>>>>>>>>>>>>>>>>>.\n");
2154
2155 ret = rdma_accept(cm_id, &cp);
2156 if (ret) {
2157 pr_err("rdma_accept() failed with: %d\n", ret);
2158 return ret;
2159 }
2160
2161 pr_debug("After rdma_accept >>>>>>>>>>>>>>>>>>>>>.\n");
2162
2163 return 0;
2164}
2165
2166static int
2167isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
2168{
2169 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
2170 int ret;
2171
2172 pr_debug("isert_get_login_rx before conn_login_comp conn: %p\n", conn);
2173
2174 ret = wait_for_completion_interruptible(&isert_conn->conn_login_comp);
2175 if (ret)
2176 return ret;
2177
2178 pr_debug("isert_get_login_rx processing login->req: %p\n", login->req);
2179 return 0;
2180}
2181
2182static void
2183isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn,
2184 struct isert_conn *isert_conn)
2185{
2186 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id;
2187 struct rdma_route *cm_route = &cm_id->route;
2188 struct sockaddr_in *sock_in;
2189 struct sockaddr_in6 *sock_in6;
2190
2191 conn->login_family = np->np_sockaddr.ss_family;
2192
2193 if (np->np_sockaddr.ss_family == AF_INET6) {
2194 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr;
2195 snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c",
2196 &sock_in6->sin6_addr.in6_u);
2197 conn->login_port = ntohs(sock_in6->sin6_port);
2198
2199 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr;
2200 snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c",
2201 &sock_in6->sin6_addr.in6_u);
2202 conn->local_port = ntohs(sock_in6->sin6_port);
2203 } else {
2204 sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr;
2205 sprintf(conn->login_ip, "%pI4",
2206 &sock_in->sin_addr.s_addr);
2207 conn->login_port = ntohs(sock_in->sin_port);
2208
2209 sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr;
2210 sprintf(conn->local_ip, "%pI4",
2211 &sock_in->sin_addr.s_addr);
2212 conn->local_port = ntohs(sock_in->sin_port);
2213 }
2214}
2215
2216static int
2217isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn)
2218{
2219 struct isert_np *isert_np = (struct isert_np *)np->np_context;
2220 struct isert_conn *isert_conn;
2221 int max_accept = 0, ret;
2222
2223accept_wait:
Sagi Grimberg8a2629a2014-04-29 13:13:45 +03002224 ret = down_interruptible(&isert_np->np_sem);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002225 if (max_accept > 5)
2226 return -ENODEV;
2227
2228 spin_lock_bh(&np->np_thread_lock);
Sagi Grimberg3ddb8752014-05-19 17:44:22 +03002229 if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) {
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002230 spin_unlock_bh(&np->np_thread_lock);
Sagi Grimberg3ddb8752014-05-19 17:44:22 +03002231 pr_debug("np_thread_state %d for isert_accept_np\n",
2232 np->np_thread_state);
2233 /**
2234 * No point in stalling here when np_thread
2235 * is in state RESET/SHUTDOWN/EXIT - bail
2236 **/
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002237 return -ENODEV;
2238 }
2239 spin_unlock_bh(&np->np_thread_lock);
2240
2241 mutex_lock(&isert_np->np_accept_mutex);
2242 if (list_empty(&isert_np->np_accept_list)) {
2243 mutex_unlock(&isert_np->np_accept_mutex);
2244 max_accept++;
2245 goto accept_wait;
2246 }
2247 isert_conn = list_first_entry(&isert_np->np_accept_list,
2248 struct isert_conn, conn_accept_node);
2249 list_del_init(&isert_conn->conn_accept_node);
2250 mutex_unlock(&isert_np->np_accept_mutex);
2251
2252 conn->context = isert_conn;
2253 isert_conn->conn = conn;
2254 max_accept = 0;
2255
2256 ret = isert_rdma_post_recvl(isert_conn);
2257 if (ret)
2258 return ret;
2259
2260 ret = isert_rdma_accept(isert_conn);
2261 if (ret)
2262 return ret;
2263
2264 isert_set_conn_info(np, conn, isert_conn);
2265
2266 pr_debug("Processing isert_accept_np: isert_conn: %p\n", isert_conn);
2267 return 0;
2268}
2269
2270static void
2271isert_free_np(struct iscsi_np *np)
2272{
2273 struct isert_np *isert_np = (struct isert_np *)np->np_context;
2274
Sagi Grimberg11b926d2014-10-28 13:45:03 -07002275 if (isert_np->np_cm_id)
2276 rdma_destroy_id(isert_np->np_cm_id);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002277
2278 np->np_context = NULL;
2279 kfree(isert_np);
2280}
2281
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08002282static void isert_wait_conn(struct iscsi_conn *conn)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002283{
2284 struct isert_conn *isert_conn = conn->context;
2285
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08002286 pr_debug("isert_wait_conn: Starting \n");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002287
Sagi Grimberg1813e802014-05-19 17:44:23 +03002288 mutex_lock(&isert_conn->conn_mutex);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002289 /*
2290 * Only wait for conn_wait_comp_err if the isert_conn made it
2291 * into full feature phase..
2292 */
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -07002293 if (isert_conn->state == ISER_CONN_INIT) {
2294 mutex_unlock(&isert_conn->conn_mutex);
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -07002295 return;
2296 }
Sagi Grimberg839eac52015-01-30 22:17:25 +00002297 isert_conn_terminate(isert_conn);
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -07002298 mutex_unlock(&isert_conn->conn_mutex);
2299
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08002300 wait_for_completion(&isert_conn->conn_wait_comp_err);
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08002301 wait_for_completion(&isert_conn->conn_wait);
Sagi Grimberg839eac52015-01-30 22:17:25 +00002302
2303 mutex_lock(&isert_conn->conn_mutex);
2304 isert_conn->state = ISER_CONN_DOWN;
2305 mutex_unlock(&isert_conn->conn_mutex);
2306
2307 pr_info("Destroying conn %p\n", isert_conn);
Sagi Grimbergb9979822014-07-02 16:19:25 +03002308 isert_put_conn(isert_conn);
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08002309}
2310
2311static void isert_free_conn(struct iscsi_conn *conn)
2312{
2313 struct isert_conn *isert_conn = conn->context;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002314
2315 isert_put_conn(isert_conn);
2316}
2317
2318static struct iscsit_transport iser_target_transport = {
2319 .name = "IB/iSER",
2320 .transport_type = ISCSI_INFINIBAND,
2321 .owner = THIS_MODULE,
2322 .iscsit_setup_np = isert_setup_np,
2323 .iscsit_accept_np = isert_accept_np,
2324 .iscsit_free_np = isert_free_np,
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08002325 .iscsit_wait_conn = isert_wait_conn,
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002326 .iscsit_free_conn = isert_free_conn,
2327 .iscsit_alloc_cmd = isert_alloc_cmd,
2328 .iscsit_get_login_rx = isert_get_login_rx,
2329 .iscsit_put_login_tx = isert_put_login_tx,
2330 .iscsit_immediate_queue = isert_immediate_queue,
2331 .iscsit_response_queue = isert_response_queue,
2332 .iscsit_get_dataout = isert_get_dataout,
2333 .iscsit_queue_data_in = isert_put_datain,
2334 .iscsit_queue_status = isert_put_response,
2335};
2336
2337static int __init isert_init(void)
2338{
2339 int ret;
2340
2341 isert_rx_wq = alloc_workqueue("isert_rx_wq", 0, 0);
2342 if (!isert_rx_wq) {
2343 pr_err("Unable to allocate isert_rx_wq\n");
2344 return -ENOMEM;
2345 }
2346
2347 isert_comp_wq = alloc_workqueue("isert_comp_wq", 0, 0);
2348 if (!isert_comp_wq) {
2349 pr_err("Unable to allocate isert_comp_wq\n");
2350 ret = -ENOMEM;
2351 goto destroy_rx_wq;
2352 }
2353
2354 isert_cmd_cache = kmem_cache_create("isert_cmd_cache",
2355 sizeof(struct isert_cmd), __alignof__(struct isert_cmd),
2356 0, NULL);
2357 if (!isert_cmd_cache) {
2358 pr_err("Unable to create isert_cmd_cache\n");
2359 ret = -ENOMEM;
2360 goto destroy_tx_cq;
2361 }
2362
2363 iscsit_register_transport(&iser_target_transport);
2364 pr_debug("iSER_TARGET[0] - Loaded iser_target_transport\n");
2365 return 0;
2366
2367destroy_tx_cq:
2368 destroy_workqueue(isert_comp_wq);
2369destroy_rx_wq:
2370 destroy_workqueue(isert_rx_wq);
2371 return ret;
2372}
2373
2374static void __exit isert_exit(void)
2375{
Sagi Grimberg029a8942014-05-19 17:44:25 +03002376 flush_scheduled_work();
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002377 kmem_cache_destroy(isert_cmd_cache);
2378 destroy_workqueue(isert_comp_wq);
2379 destroy_workqueue(isert_rx_wq);
2380 iscsit_unregister_transport(&iser_target_transport);
2381 pr_debug("iSER_TARGET[0] - Released iser_target_transport\n");
2382}
2383
2384MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure");
2385MODULE_VERSION("0.1");
2386MODULE_AUTHOR("nab@Linux-iSCSI.org");
2387MODULE_LICENSE("GPL");
2388
2389module_init(isert_init);
2390module_exit(isert_exit);