blob: 588a5eca63d84603ee89cce40636d7a87d6fbf80 [file] [log] [blame]
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001/*******************************************************************************
2 * This file contains iSCSI extentions for RDMA (iSER) Verbs
3 *
4 * (c) Copyright 2013 RisingTide Systems LLC.
5 *
6 * Nicholas A. Bellinger <nab@linux-iscsi.org>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 ****************************************************************************/
18
19#include <linux/string.h>
20#include <linux/module.h>
21#include <linux/scatterlist.h>
22#include <linux/socket.h>
23#include <linux/in.h>
24#include <linux/in6.h>
25#include <rdma/ib_verbs.h>
26#include <rdma/rdma_cm.h>
27#include <target/target_core_base.h>
28#include <target/target_core_fabric.h>
29#include <target/iscsi/iscsi_transport.h>
30
31#include "isert_proto.h"
32#include "ib_isert.h"
33
34#define ISERT_MAX_CONN 8
35#define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN)
36#define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN)
37
38static DEFINE_MUTEX(device_list_mutex);
39static LIST_HEAD(device_list);
40static struct workqueue_struct *isert_rx_wq;
41static struct workqueue_struct *isert_comp_wq;
42static struct kmem_cache *isert_cmd_cache;
43
44static void
45isert_qp_event_callback(struct ib_event *e, void *context)
46{
47 struct isert_conn *isert_conn = (struct isert_conn *)context;
48
49 pr_err("isert_qp_event_callback event: %d\n", e->event);
50 switch (e->event) {
51 case IB_EVENT_COMM_EST:
52 rdma_notify(isert_conn->conn_cm_id, IB_EVENT_COMM_EST);
53 break;
54 case IB_EVENT_QP_LAST_WQE_REACHED:
55 pr_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED:\n");
56 break;
57 default:
58 break;
59 }
60}
61
62static int
63isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr)
64{
65 int ret;
66
67 ret = ib_query_device(ib_dev, devattr);
68 if (ret) {
69 pr_err("ib_query_device() failed: %d\n", ret);
70 return ret;
71 }
72 pr_debug("devattr->max_sge: %d\n", devattr->max_sge);
73 pr_debug("devattr->max_sge_rd: %d\n", devattr->max_sge_rd);
74
75 return 0;
76}
77
78static int
79isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id)
80{
81 struct isert_device *device = isert_conn->conn_device;
82 struct ib_qp_init_attr attr;
83 struct ib_device_attr devattr;
84 int ret, index, min_index = 0;
85
86 memset(&devattr, 0, sizeof(struct ib_device_attr));
87 ret = isert_query_device(cma_id->device, &devattr);
88 if (ret)
89 return ret;
90
91 mutex_lock(&device_list_mutex);
92 for (index = 0; index < device->cqs_used; index++)
93 if (device->cq_active_qps[index] <
94 device->cq_active_qps[min_index])
95 min_index = index;
96 device->cq_active_qps[min_index]++;
97 pr_debug("isert_conn_setup_qp: Using min_index: %d\n", min_index);
98 mutex_unlock(&device_list_mutex);
99
100 memset(&attr, 0, sizeof(struct ib_qp_init_attr));
101 attr.event_handler = isert_qp_event_callback;
102 attr.qp_context = isert_conn;
103 attr.send_cq = device->dev_tx_cq[min_index];
104 attr.recv_cq = device->dev_rx_cq[min_index];
105 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS;
106 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS;
107 /*
108 * FIXME: Use devattr.max_sge - 2 for max_send_sge as
109 * work-around for RDMA_READ..
110 */
111 attr.cap.max_send_sge = devattr.max_sge - 2;
112 isert_conn->max_sge = attr.cap.max_send_sge;
113
114 attr.cap.max_recv_sge = 1;
115 attr.sq_sig_type = IB_SIGNAL_REQ_WR;
116 attr.qp_type = IB_QPT_RC;
117
118 pr_debug("isert_conn_setup_qp cma_id->device: %p\n",
119 cma_id->device);
120 pr_debug("isert_conn_setup_qp conn_pd->device: %p\n",
121 isert_conn->conn_pd->device);
122
123 ret = rdma_create_qp(cma_id, isert_conn->conn_pd, &attr);
124 if (ret) {
125 pr_err("rdma_create_qp failed for cma_id %d\n", ret);
126 return ret;
127 }
128 isert_conn->conn_qp = cma_id->qp;
129 pr_debug("rdma_create_qp() returned success >>>>>>>>>>>>>>>>>>>>>>>>>.\n");
130
131 return 0;
132}
133
134static void
135isert_cq_event_callback(struct ib_event *e, void *context)
136{
137 pr_debug("isert_cq_event_callback event: %d\n", e->event);
138}
139
140static int
141isert_alloc_rx_descriptors(struct isert_conn *isert_conn)
142{
143 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
144 struct iser_rx_desc *rx_desc;
145 struct ib_sge *rx_sg;
146 u64 dma_addr;
147 int i, j;
148
149 isert_conn->conn_rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS *
150 sizeof(struct iser_rx_desc), GFP_KERNEL);
151 if (!isert_conn->conn_rx_descs)
152 goto fail;
153
154 rx_desc = isert_conn->conn_rx_descs;
155
156 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) {
157 dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc,
158 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
159 if (ib_dma_mapping_error(ib_dev, dma_addr))
160 goto dma_map_fail;
161
162 rx_desc->dma_addr = dma_addr;
163
164 rx_sg = &rx_desc->rx_sg;
165 rx_sg->addr = rx_desc->dma_addr;
166 rx_sg->length = ISER_RX_PAYLOAD_SIZE;
167 rx_sg->lkey = isert_conn->conn_mr->lkey;
168 }
169
170 isert_conn->conn_rx_desc_head = 0;
171 return 0;
172
173dma_map_fail:
174 rx_desc = isert_conn->conn_rx_descs;
175 for (j = 0; j < i; j++, rx_desc++) {
176 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
177 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
178 }
179 kfree(isert_conn->conn_rx_descs);
180 isert_conn->conn_rx_descs = NULL;
181fail:
182 return -ENOMEM;
183}
184
185static void
186isert_free_rx_descriptors(struct isert_conn *isert_conn)
187{
188 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
189 struct iser_rx_desc *rx_desc;
190 int i;
191
192 if (!isert_conn->conn_rx_descs)
193 return;
194
195 rx_desc = isert_conn->conn_rx_descs;
196 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) {
197 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
198 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
199 }
200
201 kfree(isert_conn->conn_rx_descs);
202 isert_conn->conn_rx_descs = NULL;
203}
204
205static void isert_cq_tx_callback(struct ib_cq *, void *);
206static void isert_cq_rx_callback(struct ib_cq *, void *);
207
208static int
209isert_create_device_ib_res(struct isert_device *device)
210{
211 struct ib_device *ib_dev = device->ib_device;
212 struct isert_cq_desc *cq_desc;
213 int ret = 0, i, j;
214
215 device->cqs_used = min_t(int, num_online_cpus(),
216 device->ib_device->num_comp_vectors);
217 device->cqs_used = min(ISERT_MAX_CQ, device->cqs_used);
218 pr_debug("Using %d CQs, device %s supports %d vectors\n",
219 device->cqs_used, device->ib_device->name,
220 device->ib_device->num_comp_vectors);
221 device->cq_desc = kzalloc(sizeof(struct isert_cq_desc) *
222 device->cqs_used, GFP_KERNEL);
223 if (!device->cq_desc) {
224 pr_err("Unable to allocate device->cq_desc\n");
225 return -ENOMEM;
226 }
227 cq_desc = device->cq_desc;
228
229 device->dev_pd = ib_alloc_pd(ib_dev);
230 if (IS_ERR(device->dev_pd)) {
231 ret = PTR_ERR(device->dev_pd);
232 pr_err("ib_alloc_pd failed for dev_pd: %d\n", ret);
233 goto out_cq_desc;
234 }
235
236 for (i = 0; i < device->cqs_used; i++) {
237 cq_desc[i].device = device;
238 cq_desc[i].cq_index = i;
239
240 device->dev_rx_cq[i] = ib_create_cq(device->ib_device,
241 isert_cq_rx_callback,
242 isert_cq_event_callback,
243 (void *)&cq_desc[i],
244 ISER_MAX_RX_CQ_LEN, i);
Wei Yongjun0629b402013-10-29 09:56:34 +0800245 if (IS_ERR(device->dev_rx_cq[i])) {
246 ret = PTR_ERR(device->dev_rx_cq[i]);
247 device->dev_rx_cq[i] = NULL;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800248 goto out_cq;
Wei Yongjun0629b402013-10-29 09:56:34 +0800249 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800250
251 device->dev_tx_cq[i] = ib_create_cq(device->ib_device,
252 isert_cq_tx_callback,
253 isert_cq_event_callback,
254 (void *)&cq_desc[i],
255 ISER_MAX_TX_CQ_LEN, i);
Wei Yongjun0629b402013-10-29 09:56:34 +0800256 if (IS_ERR(device->dev_tx_cq[i])) {
257 ret = PTR_ERR(device->dev_tx_cq[i]);
258 device->dev_tx_cq[i] = NULL;
259 goto out_cq;
260 }
261
262 ret = ib_req_notify_cq(device->dev_rx_cq[i], IB_CQ_NEXT_COMP);
263 if (ret)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800264 goto out_cq;
265
Wei Yongjun0629b402013-10-29 09:56:34 +0800266 ret = ib_req_notify_cq(device->dev_tx_cq[i], IB_CQ_NEXT_COMP);
267 if (ret)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800268 goto out_cq;
269 }
270
271 device->dev_mr = ib_get_dma_mr(device->dev_pd, IB_ACCESS_LOCAL_WRITE);
272 if (IS_ERR(device->dev_mr)) {
273 ret = PTR_ERR(device->dev_mr);
274 pr_err("ib_get_dma_mr failed for dev_mr: %d\n", ret);
275 goto out_cq;
276 }
277
278 return 0;
279
280out_cq:
281 for (j = 0; j < i; j++) {
282 cq_desc = &device->cq_desc[j];
283
284 if (device->dev_rx_cq[j]) {
285 cancel_work_sync(&cq_desc->cq_rx_work);
286 ib_destroy_cq(device->dev_rx_cq[j]);
287 }
288 if (device->dev_tx_cq[j]) {
289 cancel_work_sync(&cq_desc->cq_tx_work);
290 ib_destroy_cq(device->dev_tx_cq[j]);
291 }
292 }
293 ib_dealloc_pd(device->dev_pd);
294
295out_cq_desc:
296 kfree(device->cq_desc);
297
298 return ret;
299}
300
301static void
302isert_free_device_ib_res(struct isert_device *device)
303{
304 struct isert_cq_desc *cq_desc;
305 int i;
306
307 for (i = 0; i < device->cqs_used; i++) {
308 cq_desc = &device->cq_desc[i];
309
310 cancel_work_sync(&cq_desc->cq_rx_work);
311 cancel_work_sync(&cq_desc->cq_tx_work);
312 ib_destroy_cq(device->dev_rx_cq[i]);
313 ib_destroy_cq(device->dev_tx_cq[i]);
314 device->dev_rx_cq[i] = NULL;
315 device->dev_tx_cq[i] = NULL;
316 }
317
318 ib_dereg_mr(device->dev_mr);
319 ib_dealloc_pd(device->dev_pd);
320 kfree(device->cq_desc);
321}
322
323static void
324isert_device_try_release(struct isert_device *device)
325{
326 mutex_lock(&device_list_mutex);
327 device->refcount--;
328 if (!device->refcount) {
329 isert_free_device_ib_res(device);
330 list_del(&device->dev_node);
331 kfree(device);
332 }
333 mutex_unlock(&device_list_mutex);
334}
335
336static struct isert_device *
337isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id)
338{
339 struct isert_device *device;
340 int ret;
341
342 mutex_lock(&device_list_mutex);
343 list_for_each_entry(device, &device_list, dev_node) {
344 if (device->ib_device->node_guid == cma_id->device->node_guid) {
345 device->refcount++;
346 mutex_unlock(&device_list_mutex);
347 return device;
348 }
349 }
350
351 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL);
352 if (!device) {
353 mutex_unlock(&device_list_mutex);
354 return ERR_PTR(-ENOMEM);
355 }
356
357 INIT_LIST_HEAD(&device->dev_node);
358
359 device->ib_device = cma_id->device;
360 ret = isert_create_device_ib_res(device);
361 if (ret) {
362 kfree(device);
363 mutex_unlock(&device_list_mutex);
364 return ERR_PTR(ret);
365 }
366
367 device->refcount++;
368 list_add_tail(&device->dev_node, &device_list);
369 mutex_unlock(&device_list_mutex);
370
371 return device;
372}
373
374static int
375isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
376{
377 struct iscsi_np *np = cma_id->context;
378 struct isert_np *isert_np = np->np_context;
379 struct isert_conn *isert_conn;
380 struct isert_device *device;
381 struct ib_device *ib_dev = cma_id->device;
382 int ret = 0;
383
384 pr_debug("Entering isert_connect_request cma_id: %p, context: %p\n",
385 cma_id, cma_id->context);
386
387 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL);
388 if (!isert_conn) {
389 pr_err("Unable to allocate isert_conn\n");
390 return -ENOMEM;
391 }
392 isert_conn->state = ISER_CONN_INIT;
393 INIT_LIST_HEAD(&isert_conn->conn_accept_node);
394 init_completion(&isert_conn->conn_login_comp);
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -0800395 init_completion(&isert_conn->conn_wait);
396 init_completion(&isert_conn->conn_wait_comp_err);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800397 kref_init(&isert_conn->conn_kref);
398 kref_get(&isert_conn->conn_kref);
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -0700399 mutex_init(&isert_conn->conn_mutex);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800400
401 cma_id->context = isert_conn;
402 isert_conn->conn_cm_id = cma_id;
403 isert_conn->responder_resources = event->param.conn.responder_resources;
404 isert_conn->initiator_depth = event->param.conn.initiator_depth;
405 pr_debug("Using responder_resources: %u initiator_depth: %u\n",
406 isert_conn->responder_resources, isert_conn->initiator_depth);
407
408 isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN +
409 ISER_RX_LOGIN_SIZE, GFP_KERNEL);
410 if (!isert_conn->login_buf) {
411 pr_err("Unable to allocate isert_conn->login_buf\n");
412 ret = -ENOMEM;
413 goto out;
414 }
415
416 isert_conn->login_req_buf = isert_conn->login_buf;
417 isert_conn->login_rsp_buf = isert_conn->login_buf +
418 ISCSI_DEF_MAX_RECV_SEG_LEN;
419 pr_debug("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n",
420 isert_conn->login_buf, isert_conn->login_req_buf,
421 isert_conn->login_rsp_buf);
422
423 isert_conn->login_req_dma = ib_dma_map_single(ib_dev,
424 (void *)isert_conn->login_req_buf,
425 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE);
426
427 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma);
428 if (ret) {
429 pr_err("ib_dma_mapping_error failed for login_req_dma: %d\n",
430 ret);
431 isert_conn->login_req_dma = 0;
432 goto out_login_buf;
433 }
434
435 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev,
436 (void *)isert_conn->login_rsp_buf,
437 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
438
439 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma);
440 if (ret) {
441 pr_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n",
442 ret);
443 isert_conn->login_rsp_dma = 0;
444 goto out_req_dma_map;
445 }
446
447 device = isert_device_find_by_ib_dev(cma_id);
448 if (IS_ERR(device)) {
449 ret = PTR_ERR(device);
450 goto out_rsp_dma_map;
451 }
452
453 isert_conn->conn_device = device;
454 isert_conn->conn_pd = device->dev_pd;
455 isert_conn->conn_mr = device->dev_mr;
456
457 ret = isert_conn_setup_qp(isert_conn, cma_id);
458 if (ret)
459 goto out_conn_dev;
460
461 mutex_lock(&isert_np->np_accept_mutex);
462 list_add_tail(&isert_np->np_accept_list, &isert_conn->conn_accept_node);
463 mutex_unlock(&isert_np->np_accept_mutex);
464
465 pr_debug("isert_connect_request() waking up np_accept_wq: %p\n", np);
466 wake_up(&isert_np->np_accept_wq);
467 return 0;
468
469out_conn_dev:
470 isert_device_try_release(device);
471out_rsp_dma_map:
472 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma,
473 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
474out_req_dma_map:
475 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
476 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE);
477out_login_buf:
478 kfree(isert_conn->login_buf);
479out:
480 kfree(isert_conn);
481 return ret;
482}
483
484static void
485isert_connect_release(struct isert_conn *isert_conn)
486{
487 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
488 struct isert_device *device = isert_conn->conn_device;
489 int cq_index;
490
491 pr_debug("Entering isert_connect_release(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
492
493 if (isert_conn->conn_qp) {
494 cq_index = ((struct isert_cq_desc *)
495 isert_conn->conn_qp->recv_cq->cq_context)->cq_index;
496 pr_debug("isert_connect_release: cq_index: %d\n", cq_index);
497 isert_conn->conn_device->cq_active_qps[cq_index]--;
498
499 rdma_destroy_qp(isert_conn->conn_cm_id);
500 }
501
502 isert_free_rx_descriptors(isert_conn);
503 rdma_destroy_id(isert_conn->conn_cm_id);
504
505 if (isert_conn->login_buf) {
506 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma,
507 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
508 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
509 ISCSI_DEF_MAX_RECV_SEG_LEN,
510 DMA_FROM_DEVICE);
511 kfree(isert_conn->login_buf);
512 }
513 kfree(isert_conn);
514
515 if (device)
516 isert_device_try_release(device);
517
518 pr_debug("Leaving isert_connect_release >>>>>>>>>>>>\n");
519}
520
521static void
522isert_connected_handler(struct rdma_cm_id *cma_id)
523{
524 return;
525}
526
527static void
528isert_release_conn_kref(struct kref *kref)
529{
530 struct isert_conn *isert_conn = container_of(kref,
531 struct isert_conn, conn_kref);
532
533 pr_debug("Calling isert_connect_release for final kref %s/%d\n",
534 current->comm, current->pid);
535
536 isert_connect_release(isert_conn);
537}
538
539static void
540isert_put_conn(struct isert_conn *isert_conn)
541{
542 kref_put(&isert_conn->conn_kref, isert_release_conn_kref);
543}
544
545static void
546isert_disconnect_work(struct work_struct *work)
547{
548 struct isert_conn *isert_conn = container_of(work,
549 struct isert_conn, conn_logout_work);
550
551 pr_debug("isert_disconnect_work(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -0700552 mutex_lock(&isert_conn->conn_mutex);
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -0800553 if (isert_conn->state == ISER_CONN_UP)
554 isert_conn->state = ISER_CONN_TERMINATING;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800555
556 if (isert_conn->post_recv_buf_count == 0 &&
557 atomic_read(&isert_conn->post_send_buf_count) == 0) {
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -0700558 mutex_unlock(&isert_conn->conn_mutex);
559 goto wake_up;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800560 }
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -0700561 if (!isert_conn->conn_cm_id) {
562 mutex_unlock(&isert_conn->conn_mutex);
563 isert_put_conn(isert_conn);
564 return;
565 }
566 if (!isert_conn->logout_posted) {
567 pr_debug("Calling rdma_disconnect for !logout_posted from"
568 " isert_disconnect_work\n");
569 rdma_disconnect(isert_conn->conn_cm_id);
570 mutex_unlock(&isert_conn->conn_mutex);
571 iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
572 goto wake_up;
573 }
574 mutex_unlock(&isert_conn->conn_mutex);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800575
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -0700576wake_up:
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -0800577 complete(&isert_conn->conn_wait);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800578 isert_put_conn(isert_conn);
579}
580
581static void
582isert_disconnected_handler(struct rdma_cm_id *cma_id)
583{
584 struct isert_conn *isert_conn = (struct isert_conn *)cma_id->context;
585
586 INIT_WORK(&isert_conn->conn_logout_work, isert_disconnect_work);
587 schedule_work(&isert_conn->conn_logout_work);
588}
589
590static int
591isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
592{
593 int ret = 0;
594
595 pr_debug("isert_cma_handler: event %d status %d conn %p id %p\n",
596 event->event, event->status, cma_id->context, cma_id);
597
598 switch (event->event) {
599 case RDMA_CM_EVENT_CONNECT_REQUEST:
600 pr_debug("RDMA_CM_EVENT_CONNECT_REQUEST: >>>>>>>>>>>>>>>\n");
601 ret = isert_connect_request(cma_id, event);
602 break;
603 case RDMA_CM_EVENT_ESTABLISHED:
604 pr_debug("RDMA_CM_EVENT_ESTABLISHED >>>>>>>>>>>>>>\n");
605 isert_connected_handler(cma_id);
606 break;
607 case RDMA_CM_EVENT_DISCONNECTED:
608 pr_debug("RDMA_CM_EVENT_DISCONNECTED: >>>>>>>>>>>>>>\n");
609 isert_disconnected_handler(cma_id);
610 break;
611 case RDMA_CM_EVENT_DEVICE_REMOVAL:
612 case RDMA_CM_EVENT_ADDR_CHANGE:
613 break;
614 case RDMA_CM_EVENT_CONNECT_ERROR:
615 default:
616 pr_err("Unknown RDMA CMA event: %d\n", event->event);
617 break;
618 }
619
620 if (ret != 0) {
621 pr_err("isert_cma_handler failed RDMA_CM_EVENT: 0x%08x %d\n",
622 event->event, ret);
623 dump_stack();
624 }
625
626 return ret;
627}
628
629static int
630isert_post_recv(struct isert_conn *isert_conn, u32 count)
631{
632 struct ib_recv_wr *rx_wr, *rx_wr_failed;
633 int i, ret;
634 unsigned int rx_head = isert_conn->conn_rx_desc_head;
635 struct iser_rx_desc *rx_desc;
636
637 for (rx_wr = isert_conn->conn_rx_wr, i = 0; i < count; i++, rx_wr++) {
638 rx_desc = &isert_conn->conn_rx_descs[rx_head];
639 rx_wr->wr_id = (unsigned long)rx_desc;
640 rx_wr->sg_list = &rx_desc->rx_sg;
641 rx_wr->num_sge = 1;
642 rx_wr->next = rx_wr + 1;
643 rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1);
644 }
645
646 rx_wr--;
647 rx_wr->next = NULL; /* mark end of work requests list */
648
649 isert_conn->post_recv_buf_count += count;
650 ret = ib_post_recv(isert_conn->conn_qp, isert_conn->conn_rx_wr,
651 &rx_wr_failed);
652 if (ret) {
653 pr_err("ib_post_recv() failed with ret: %d\n", ret);
654 isert_conn->post_recv_buf_count -= count;
655 } else {
656 pr_debug("isert_post_recv(): Posted %d RX buffers\n", count);
657 isert_conn->conn_rx_desc_head = rx_head;
658 }
659 return ret;
660}
661
662static int
663isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc)
664{
665 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
666 struct ib_send_wr send_wr, *send_wr_failed;
667 int ret;
668
669 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr,
670 ISER_HEADERS_LEN, DMA_TO_DEVICE);
671
672 send_wr.next = NULL;
673 send_wr.wr_id = (unsigned long)tx_desc;
674 send_wr.sg_list = tx_desc->tx_sg;
675 send_wr.num_sge = tx_desc->num_sge;
676 send_wr.opcode = IB_WR_SEND;
677 send_wr.send_flags = IB_SEND_SIGNALED;
678
679 atomic_inc(&isert_conn->post_send_buf_count);
680
681 ret = ib_post_send(isert_conn->conn_qp, &send_wr, &send_wr_failed);
682 if (ret) {
683 pr_err("ib_post_send() failed, ret: %d\n", ret);
684 atomic_dec(&isert_conn->post_send_buf_count);
685 }
686
687 return ret;
688}
689
690static void
691isert_create_send_desc(struct isert_conn *isert_conn,
692 struct isert_cmd *isert_cmd,
693 struct iser_tx_desc *tx_desc)
694{
695 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
696
697 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr,
698 ISER_HEADERS_LEN, DMA_TO_DEVICE);
699
700 memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr));
701 tx_desc->iser_header.flags = ISER_VER;
702
703 tx_desc->num_sge = 1;
704 tx_desc->isert_cmd = isert_cmd;
705
706 if (tx_desc->tx_sg[0].lkey != isert_conn->conn_mr->lkey) {
707 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey;
708 pr_debug("tx_desc %p lkey mismatch, fixing\n", tx_desc);
709 }
710}
711
712static int
713isert_init_tx_hdrs(struct isert_conn *isert_conn,
714 struct iser_tx_desc *tx_desc)
715{
716 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
717 u64 dma_addr;
718
719 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc,
720 ISER_HEADERS_LEN, DMA_TO_DEVICE);
721 if (ib_dma_mapping_error(ib_dev, dma_addr)) {
722 pr_err("ib_dma_mapping_error() failed\n");
723 return -ENOMEM;
724 }
725
726 tx_desc->dma_addr = dma_addr;
727 tx_desc->tx_sg[0].addr = tx_desc->dma_addr;
728 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN;
729 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey;
730
731 pr_debug("isert_init_tx_hdrs: Setup tx_sg[0].addr: 0x%llx length: %u"
732 " lkey: 0x%08x\n", tx_desc->tx_sg[0].addr,
733 tx_desc->tx_sg[0].length, tx_desc->tx_sg[0].lkey);
734
735 return 0;
736}
737
738static void
739isert_init_send_wr(struct isert_cmd *isert_cmd, struct ib_send_wr *send_wr)
740{
741 isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND;
742 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc;
743 send_wr->opcode = IB_WR_SEND;
744 send_wr->send_flags = IB_SEND_SIGNALED;
745 send_wr->sg_list = &isert_cmd->tx_desc.tx_sg[0];
746 send_wr->num_sge = isert_cmd->tx_desc.num_sge;
747}
748
749static int
750isert_rdma_post_recvl(struct isert_conn *isert_conn)
751{
752 struct ib_recv_wr rx_wr, *rx_wr_fail;
753 struct ib_sge sge;
754 int ret;
755
756 memset(&sge, 0, sizeof(struct ib_sge));
757 sge.addr = isert_conn->login_req_dma;
758 sge.length = ISER_RX_LOGIN_SIZE;
759 sge.lkey = isert_conn->conn_mr->lkey;
760
761 pr_debug("Setup sge: addr: %llx length: %d 0x%08x\n",
762 sge.addr, sge.length, sge.lkey);
763
764 memset(&rx_wr, 0, sizeof(struct ib_recv_wr));
765 rx_wr.wr_id = (unsigned long)isert_conn->login_req_buf;
766 rx_wr.sg_list = &sge;
767 rx_wr.num_sge = 1;
768
769 isert_conn->post_recv_buf_count++;
770 ret = ib_post_recv(isert_conn->conn_qp, &rx_wr, &rx_wr_fail);
771 if (ret) {
772 pr_err("ib_post_recv() failed: %d\n", ret);
773 isert_conn->post_recv_buf_count--;
774 }
775
776 pr_debug("ib_post_recv(): returned success >>>>>>>>>>>>>>>>>>>>>>>>\n");
777 return ret;
778}
779
780static int
781isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
782 u32 length)
783{
784 struct isert_conn *isert_conn = conn->context;
785 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
786 struct iser_tx_desc *tx_desc = &isert_conn->conn_login_tx_desc;
787 int ret;
788
789 isert_create_send_desc(isert_conn, NULL, tx_desc);
790
791 memcpy(&tx_desc->iscsi_header, &login->rsp[0],
792 sizeof(struct iscsi_hdr));
793
794 isert_init_tx_hdrs(isert_conn, tx_desc);
795
796 if (length > 0) {
797 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1];
798
799 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma,
800 length, DMA_TO_DEVICE);
801
802 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length);
803
804 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma,
805 length, DMA_TO_DEVICE);
806
807 tx_dsg->addr = isert_conn->login_rsp_dma;
808 tx_dsg->length = length;
809 tx_dsg->lkey = isert_conn->conn_mr->lkey;
810 tx_desc->num_sge = 2;
811 }
812 if (!login->login_failed) {
813 if (login->login_complete) {
814 ret = isert_alloc_rx_descriptors(isert_conn);
815 if (ret)
816 return ret;
817
818 ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX);
819 if (ret)
820 return ret;
821
822 isert_conn->state = ISER_CONN_UP;
823 goto post_send;
824 }
825
826 ret = isert_rdma_post_recvl(isert_conn);
827 if (ret)
828 return ret;
829 }
830post_send:
831 ret = isert_post_send(isert_conn, tx_desc);
832 if (ret)
833 return ret;
834
835 return 0;
836}
837
838static void
839isert_rx_login_req(struct iser_rx_desc *rx_desc, int rx_buflen,
840 struct isert_conn *isert_conn)
841{
842 struct iscsi_conn *conn = isert_conn->conn;
843 struct iscsi_login *login = conn->conn_login;
844 int size;
845
846 if (!login) {
847 pr_err("conn->conn_login is NULL\n");
848 dump_stack();
849 return;
850 }
851
852 if (login->first_request) {
853 struct iscsi_login_req *login_req =
854 (struct iscsi_login_req *)&rx_desc->iscsi_header;
855 /*
856 * Setup the initial iscsi_login values from the leading
857 * login request PDU.
858 */
859 login->leading_connection = (!login_req->tsih) ? 1 : 0;
860 login->current_stage =
861 (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK)
862 >> 2;
863 login->version_min = login_req->min_version;
864 login->version_max = login_req->max_version;
865 memcpy(login->isid, login_req->isid, 6);
866 login->cmd_sn = be32_to_cpu(login_req->cmdsn);
867 login->init_task_tag = login_req->itt;
868 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
869 login->cid = be16_to_cpu(login_req->cid);
870 login->tsih = be16_to_cpu(login_req->tsih);
871 }
872
873 memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN);
874
875 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS);
876 pr_debug("Using login payload size: %d, rx_buflen: %d MAX_KEY_VALUE_PAIRS: %d\n",
877 size, rx_buflen, MAX_KEY_VALUE_PAIRS);
878 memcpy(login->req_buf, &rx_desc->data[0], size);
879
880 complete(&isert_conn->conn_login_comp);
881}
882
883static void
884isert_release_cmd(struct iscsi_cmd *cmd)
885{
886 struct isert_cmd *isert_cmd = container_of(cmd, struct isert_cmd,
887 iscsi_cmd);
888
889 pr_debug("Entering isert_release_cmd %p >>>>>>>>>>>>>>>.\n", isert_cmd);
890
891 kfree(cmd->buf_ptr);
892 kfree(cmd->tmr_req);
893
894 kmem_cache_free(isert_cmd_cache, isert_cmd);
895}
896
897static struct iscsi_cmd
898*isert_alloc_cmd(struct iscsi_conn *conn, gfp_t gfp)
899{
900 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
901 struct isert_cmd *isert_cmd;
902
903 isert_cmd = kmem_cache_zalloc(isert_cmd_cache, gfp);
904 if (!isert_cmd) {
905 pr_err("Unable to allocate isert_cmd\n");
906 return NULL;
907 }
908 isert_cmd->conn = isert_conn;
909 isert_cmd->iscsi_cmd.release_cmd = &isert_release_cmd;
910
911 return &isert_cmd->iscsi_cmd;
912}
913
914static int
915isert_handle_scsi_cmd(struct isert_conn *isert_conn,
916 struct isert_cmd *isert_cmd, struct iser_rx_desc *rx_desc,
917 unsigned char *buf)
918{
919 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
920 struct iscsi_conn *conn = isert_conn->conn;
921 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf;
922 struct scatterlist *sg;
923 int imm_data, imm_data_len, unsol_data, sg_nents, rc;
924 bool dump_payload = false;
925
926 rc = iscsit_setup_scsi_cmd(conn, cmd, buf);
927 if (rc < 0)
928 return rc;
929
930 imm_data = cmd->immediate_data;
931 imm_data_len = cmd->first_burst_len;
932 unsol_data = cmd->unsolicited_data;
933
934 rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
935 if (rc < 0) {
936 return 0;
937 } else if (rc > 0) {
938 dump_payload = true;
939 goto sequence_cmd;
940 }
941
942 if (!imm_data)
943 return 0;
944
945 sg = &cmd->se_cmd.t_data_sg[0];
946 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE));
947
948 pr_debug("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n",
949 sg, sg_nents, &rx_desc->data[0], imm_data_len);
950
951 sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len);
952
953 cmd->write_data_done += imm_data_len;
954
955 if (cmd->write_data_done == cmd->se_cmd.data_length) {
956 spin_lock_bh(&cmd->istate_lock);
957 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
958 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
959 spin_unlock_bh(&cmd->istate_lock);
960 }
961
962sequence_cmd:
Nicholas Bellingeradb97c22013-07-30 04:04:02 +0000963 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800964
965 if (!rc && dump_payload == false && unsol_data)
966 iscsit_set_unsoliticed_dataout(cmd);
967
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800968 return 0;
969}
970
971static int
972isert_handle_iscsi_dataout(struct isert_conn *isert_conn,
973 struct iser_rx_desc *rx_desc, unsigned char *buf)
974{
975 struct scatterlist *sg_start;
976 struct iscsi_conn *conn = isert_conn->conn;
977 struct iscsi_cmd *cmd = NULL;
978 struct iscsi_data *hdr = (struct iscsi_data *)buf;
979 u32 unsol_data_len = ntoh24(hdr->dlength);
980 int rc, sg_nents, sg_off, page_off;
981
982 rc = iscsit_check_dataout_hdr(conn, buf, &cmd);
983 if (rc < 0)
984 return rc;
985 else if (!cmd)
986 return 0;
987 /*
988 * FIXME: Unexpected unsolicited_data out
989 */
990 if (!cmd->unsolicited_data) {
991 pr_err("Received unexpected solicited data payload\n");
992 dump_stack();
993 return -1;
994 }
995
996 pr_debug("Unsolicited DataOut unsol_data_len: %u, write_data_done: %u, data_length: %u\n",
997 unsol_data_len, cmd->write_data_done, cmd->se_cmd.data_length);
998
999 sg_off = cmd->write_data_done / PAGE_SIZE;
1000 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1001 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE));
1002 page_off = cmd->write_data_done % PAGE_SIZE;
1003 /*
1004 * FIXME: Non page-aligned unsolicited_data out
1005 */
1006 if (page_off) {
1007 pr_err("Received unexpected non-page aligned data payload\n");
1008 dump_stack();
1009 return -1;
1010 }
1011 pr_debug("Copying DataOut: sg_start: %p, sg_off: %u sg_nents: %u from %p %u\n",
1012 sg_start, sg_off, sg_nents, &rx_desc->data[0], unsol_data_len);
1013
1014 sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0],
1015 unsol_data_len);
1016
1017 rc = iscsit_check_dataout_payload(cmd, hdr, false);
1018 if (rc < 0)
1019 return rc;
1020
1021 return 0;
1022}
1023
1024static int
1025isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc,
1026 uint32_t read_stag, uint64_t read_va,
1027 uint32_t write_stag, uint64_t write_va)
1028{
1029 struct iscsi_hdr *hdr = &rx_desc->iscsi_header;
1030 struct iscsi_conn *conn = isert_conn->conn;
1031 struct iscsi_cmd *cmd;
1032 struct isert_cmd *isert_cmd;
1033 int ret = -EINVAL;
1034 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1035
1036 switch (opcode) {
1037 case ISCSI_OP_SCSI_CMD:
1038 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1039 if (!cmd)
1040 break;
1041
1042 isert_cmd = container_of(cmd, struct isert_cmd, iscsi_cmd);
1043 isert_cmd->read_stag = read_stag;
1044 isert_cmd->read_va = read_va;
1045 isert_cmd->write_stag = write_stag;
1046 isert_cmd->write_va = write_va;
1047
1048 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd,
1049 rx_desc, (unsigned char *)hdr);
1050 break;
1051 case ISCSI_OP_NOOP_OUT:
1052 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1053 if (!cmd)
1054 break;
1055
1056 ret = iscsit_handle_nop_out(conn, cmd, (unsigned char *)hdr);
1057 break;
1058 case ISCSI_OP_SCSI_DATA_OUT:
1059 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc,
1060 (unsigned char *)hdr);
1061 break;
1062 case ISCSI_OP_SCSI_TMFUNC:
1063 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1064 if (!cmd)
1065 break;
1066
1067 ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1068 (unsigned char *)hdr);
1069 break;
1070 case ISCSI_OP_LOGOUT:
1071 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1072 if (!cmd)
1073 break;
1074
1075 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
1076 if (ret > 0)
1077 wait_for_completion_timeout(&conn->conn_logout_comp,
1078 SECONDS_FOR_LOGOUT_COMP *
1079 HZ);
1080 break;
1081 default:
1082 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1083 dump_stack();
1084 break;
1085 }
1086
1087 return ret;
1088}
1089
1090static void
1091isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn)
1092{
1093 struct iser_hdr *iser_hdr = &rx_desc->iser_header;
1094 uint64_t read_va = 0, write_va = 0;
1095 uint32_t read_stag = 0, write_stag = 0;
1096 int rc;
1097
1098 switch (iser_hdr->flags & 0xF0) {
1099 case ISCSI_CTRL:
1100 if (iser_hdr->flags & ISER_RSV) {
1101 read_stag = be32_to_cpu(iser_hdr->read_stag);
1102 read_va = be64_to_cpu(iser_hdr->read_va);
1103 pr_debug("ISER_RSV: read_stag: 0x%08x read_va: 0x%16llx\n",
1104 read_stag, (unsigned long long)read_va);
1105 }
1106 if (iser_hdr->flags & ISER_WSV) {
1107 write_stag = be32_to_cpu(iser_hdr->write_stag);
1108 write_va = be64_to_cpu(iser_hdr->write_va);
1109 pr_debug("ISER_WSV: write__stag: 0x%08x write_va: 0x%16llx\n",
1110 write_stag, (unsigned long long)write_va);
1111 }
1112
1113 pr_debug("ISER ISCSI_CTRL PDU\n");
1114 break;
1115 case ISER_HELLO:
1116 pr_err("iSER Hello message\n");
1117 break;
1118 default:
1119 pr_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags);
1120 break;
1121 }
1122
1123 rc = isert_rx_opcode(isert_conn, rx_desc,
1124 read_stag, read_va, write_stag, write_va);
1125}
1126
1127static void
1128isert_rx_completion(struct iser_rx_desc *desc, struct isert_conn *isert_conn,
1129 unsigned long xfer_len)
1130{
1131 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1132 struct iscsi_hdr *hdr;
1133 u64 rx_dma;
1134 int rx_buflen, outstanding;
1135
1136 if ((char *)desc == isert_conn->login_req_buf) {
1137 rx_dma = isert_conn->login_req_dma;
1138 rx_buflen = ISER_RX_LOGIN_SIZE;
1139 pr_debug("ISER login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n",
1140 rx_dma, rx_buflen);
1141 } else {
1142 rx_dma = desc->dma_addr;
1143 rx_buflen = ISER_RX_PAYLOAD_SIZE;
1144 pr_debug("ISER req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n",
1145 rx_dma, rx_buflen);
1146 }
1147
1148 ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE);
1149
1150 hdr = &desc->iscsi_header;
1151 pr_debug("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n",
1152 hdr->opcode, hdr->itt, hdr->flags,
1153 (int)(xfer_len - ISER_HEADERS_LEN));
1154
1155 if ((char *)desc == isert_conn->login_req_buf)
1156 isert_rx_login_req(desc, xfer_len - ISER_HEADERS_LEN,
1157 isert_conn);
1158 else
1159 isert_rx_do_work(desc, isert_conn);
1160
1161 ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen,
1162 DMA_FROM_DEVICE);
1163
1164 isert_conn->post_recv_buf_count--;
1165 pr_debug("iSERT: Decremented post_recv_buf_count: %d\n",
1166 isert_conn->post_recv_buf_count);
1167
1168 if ((char *)desc == isert_conn->login_req_buf)
1169 return;
1170
1171 outstanding = isert_conn->post_recv_buf_count;
1172 if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) {
1173 int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding,
1174 ISERT_MIN_POSTED_RX);
1175 err = isert_post_recv(isert_conn, count);
1176 if (err) {
1177 pr_err("isert_post_recv() count: %d failed, %d\n",
1178 count, err);
1179 }
1180 }
1181}
1182
1183static void
1184isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn)
1185{
1186 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
1187 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1188
1189 pr_debug("isert_unmap_cmd >>>>>>>>>>>>>>>>>>>>>>>\n");
1190
1191 if (wr->sge) {
1192 ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, DMA_TO_DEVICE);
1193 wr->sge = NULL;
1194 }
1195
1196 kfree(wr->send_wr);
1197 wr->send_wr = NULL;
1198
1199 kfree(isert_cmd->ib_sge);
1200 isert_cmd->ib_sge = NULL;
1201}
1202
1203static void
1204isert_put_cmd(struct isert_cmd *isert_cmd)
1205{
1206 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
1207 struct isert_conn *isert_conn = isert_cmd->conn;
Nicholas Bellingerc6ccbb92013-07-03 03:11:48 -07001208 struct iscsi_conn *conn = isert_conn->conn;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001209
1210 pr_debug("Entering isert_put_cmd: %p\n", isert_cmd);
1211
1212 switch (cmd->iscsi_opcode) {
1213 case ISCSI_OP_SCSI_CMD:
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001214 spin_lock_bh(&conn->cmd_lock);
1215 if (!list_empty(&cmd->i_conn_node))
Nicholas Bellingeraf737f62014-02-03 12:53:51 -08001216 list_del_init(&cmd->i_conn_node);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001217 spin_unlock_bh(&conn->cmd_lock);
1218
1219 if (cmd->data_direction == DMA_TO_DEVICE)
1220 iscsit_stop_dataout_timer(cmd);
1221
1222 isert_unmap_cmd(isert_cmd, isert_conn);
Nicholas Bellingerc6ccbb92013-07-03 03:11:48 -07001223 transport_generic_free_cmd(&cmd->se_cmd, 0);
1224 break;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001225 case ISCSI_OP_SCSI_TMFUNC:
Nicholas Bellingerc6ccbb92013-07-03 03:11:48 -07001226 spin_lock_bh(&conn->cmd_lock);
1227 if (!list_empty(&cmd->i_conn_node))
Nicholas Bellingeraf737f62014-02-03 12:53:51 -08001228 list_del_init(&cmd->i_conn_node);
Nicholas Bellingerc6ccbb92013-07-03 03:11:48 -07001229 spin_unlock_bh(&conn->cmd_lock);
1230
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001231 transport_generic_free_cmd(&cmd->se_cmd, 0);
1232 break;
1233 case ISCSI_OP_REJECT:
1234 case ISCSI_OP_NOOP_OUT:
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001235 spin_lock_bh(&conn->cmd_lock);
1236 if (!list_empty(&cmd->i_conn_node))
Nicholas Bellingeraf737f62014-02-03 12:53:51 -08001237 list_del_init(&cmd->i_conn_node);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001238 spin_unlock_bh(&conn->cmd_lock);
1239
1240 /*
1241 * Handle special case for REJECT when iscsi_add_reject*() has
1242 * overwritten the original iscsi_opcode assignment, and the
1243 * associated cmd->se_cmd needs to be released.
1244 */
1245 if (cmd->se_cmd.se_tfo != NULL) {
Nicholas Bellingerfff98872013-06-26 02:31:42 -07001246 pr_debug("Calling transport_generic_free_cmd from"
1247 " isert_put_cmd for 0x%02x\n",
1248 cmd->iscsi_opcode);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001249 transport_generic_free_cmd(&cmd->se_cmd, 0);
1250 break;
1251 }
1252 /*
1253 * Fall-through
1254 */
1255 default:
1256 isert_release_cmd(cmd);
1257 break;
1258 }
1259}
1260
1261static void
1262isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev)
1263{
1264 if (tx_desc->dma_addr != 0) {
1265 pr_debug("Calling ib_dma_unmap_single for tx_desc->dma_addr\n");
1266 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr,
1267 ISER_HEADERS_LEN, DMA_TO_DEVICE);
1268 tx_desc->dma_addr = 0;
1269 }
1270}
1271
1272static void
1273isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd,
1274 struct ib_device *ib_dev)
1275{
1276 if (isert_cmd->sense_buf_dma != 0) {
1277 pr_debug("Calling ib_dma_unmap_single for isert_cmd->sense_buf_dma\n");
1278 ib_dma_unmap_single(ib_dev, isert_cmd->sense_buf_dma,
1279 isert_cmd->sense_buf_len, DMA_TO_DEVICE);
1280 isert_cmd->sense_buf_dma = 0;
1281 }
1282
1283 isert_unmap_tx_desc(tx_desc, ib_dev);
1284 isert_put_cmd(isert_cmd);
1285}
1286
1287static void
1288isert_completion_rdma_read(struct iser_tx_desc *tx_desc,
1289 struct isert_cmd *isert_cmd)
1290{
1291 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
1292 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
1293 struct se_cmd *se_cmd = &cmd->se_cmd;
1294 struct ib_device *ib_dev = isert_cmd->conn->conn_cm_id->device;
1295
1296 iscsit_stop_dataout_timer(cmd);
1297
1298 if (wr->sge) {
1299 pr_debug("isert_do_rdma_read_comp: Unmapping wr->sge from t_data_sg\n");
1300 ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, DMA_TO_DEVICE);
1301 wr->sge = NULL;
1302 }
1303
1304 if (isert_cmd->ib_sge) {
1305 pr_debug("isert_do_rdma_read_comp: Freeing isert_cmd->ib_sge\n");
1306 kfree(isert_cmd->ib_sge);
1307 isert_cmd->ib_sge = NULL;
1308 }
1309
1310 cmd->write_data_done = se_cmd->data_length;
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001311 wr->send_wr_num = 0;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001312
1313 pr_debug("isert_do_rdma_read_comp, calling target_execute_cmd\n");
1314 spin_lock_bh(&cmd->istate_lock);
1315 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1316 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1317 spin_unlock_bh(&cmd->istate_lock);
1318
1319 target_execute_cmd(se_cmd);
1320}
1321
1322static void
1323isert_do_control_comp(struct work_struct *work)
1324{
1325 struct isert_cmd *isert_cmd = container_of(work,
1326 struct isert_cmd, comp_work);
1327 struct isert_conn *isert_conn = isert_cmd->conn;
1328 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1329 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
1330
1331 switch (cmd->i_state) {
1332 case ISTATE_SEND_TASKMGTRSP:
1333 pr_debug("Calling iscsit_tmr_post_handler >>>>>>>>>>>>>>>>>\n");
1334
1335 atomic_dec(&isert_conn->post_send_buf_count);
1336 iscsit_tmr_post_handler(cmd, cmd->conn);
1337
1338 cmd->i_state = ISTATE_SENT_STATUS;
1339 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev);
1340 break;
1341 case ISTATE_SEND_REJECT:
1342 pr_debug("Got isert_do_control_comp ISTATE_SEND_REJECT: >>>\n");
1343 atomic_dec(&isert_conn->post_send_buf_count);
1344
1345 cmd->i_state = ISTATE_SENT_STATUS;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001346 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev);
Nicholas Bellingerfff98872013-06-26 02:31:42 -07001347 break;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001348 case ISTATE_SEND_LOGOUTRSP:
1349 pr_debug("Calling iscsit_logout_post_handler >>>>>>>>>>>>>>\n");
1350 /*
1351 * Call atomic_dec(&isert_conn->post_send_buf_count)
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001352 * from isert_wait_conn()
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001353 */
1354 isert_conn->logout_posted = true;
1355 iscsit_logout_post_handler(cmd, cmd->conn);
1356 break;
1357 default:
1358 pr_err("Unknown do_control_comp i_state %d\n", cmd->i_state);
1359 dump_stack();
1360 break;
1361 }
1362}
1363
1364static void
1365isert_response_completion(struct iser_tx_desc *tx_desc,
1366 struct isert_cmd *isert_cmd,
1367 struct isert_conn *isert_conn,
1368 struct ib_device *ib_dev)
1369{
1370 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001371 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001372
1373 if (cmd->i_state == ISTATE_SEND_TASKMGTRSP ||
Nicholas Bellingerfff98872013-06-26 02:31:42 -07001374 cmd->i_state == ISTATE_SEND_LOGOUTRSP ||
1375 cmd->i_state == ISTATE_SEND_REJECT) {
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001376 isert_unmap_tx_desc(tx_desc, ib_dev);
1377
1378 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp);
1379 queue_work(isert_comp_wq, &isert_cmd->comp_work);
1380 return;
1381 }
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001382 atomic_sub(wr->send_wr_num + 1, &isert_conn->post_send_buf_count);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001383
1384 cmd->i_state = ISTATE_SENT_STATUS;
1385 isert_completion_put(tx_desc, isert_cmd, ib_dev);
1386}
1387
1388static void
1389isert_send_completion(struct iser_tx_desc *tx_desc,
1390 struct isert_conn *isert_conn)
1391{
1392 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1393 struct isert_cmd *isert_cmd = tx_desc->isert_cmd;
1394 struct isert_rdma_wr *wr;
1395
1396 if (!isert_cmd) {
1397 atomic_dec(&isert_conn->post_send_buf_count);
1398 isert_unmap_tx_desc(tx_desc, ib_dev);
1399 return;
1400 }
1401 wr = &isert_cmd->rdma_wr;
1402
1403 switch (wr->iser_ib_op) {
1404 case ISER_IB_RECV:
1405 pr_err("isert_send_completion: Got ISER_IB_RECV\n");
1406 dump_stack();
1407 break;
1408 case ISER_IB_SEND:
1409 pr_debug("isert_send_completion: Got ISER_IB_SEND\n");
1410 isert_response_completion(tx_desc, isert_cmd,
1411 isert_conn, ib_dev);
1412 break;
1413 case ISER_IB_RDMA_WRITE:
1414 pr_err("isert_send_completion: Got ISER_IB_RDMA_WRITE\n");
1415 dump_stack();
1416 break;
1417 case ISER_IB_RDMA_READ:
1418 pr_debug("isert_send_completion: Got ISER_IB_RDMA_READ:\n");
1419
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001420 atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001421 isert_completion_rdma_read(tx_desc, isert_cmd);
1422 break;
1423 default:
1424 pr_err("Unknown wr->iser_ib_op: 0x%02x\n", wr->iser_ib_op);
1425 dump_stack();
1426 break;
1427 }
1428}
1429
1430static void
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001431isert_cq_tx_comp_err(struct iser_tx_desc *tx_desc, struct isert_conn *isert_conn)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001432{
1433 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001434 struct isert_cmd *isert_cmd = tx_desc->isert_cmd;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001435
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001436 if (!isert_cmd)
1437 isert_unmap_tx_desc(tx_desc, ib_dev);
1438 else
1439 isert_completion_put(tx_desc, isert_cmd, ib_dev);
1440}
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001441
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001442static void
1443isert_cq_rx_comp_err(struct isert_conn *isert_conn)
1444{
1445 struct iscsi_conn *conn = isert_conn->conn;
1446
1447 if (isert_conn->post_recv_buf_count)
1448 return;
1449
1450 if (conn->sess) {
1451 target_sess_cmd_list_set_waiting(conn->sess->se_sess);
1452 target_wait_for_sess_cmds(conn->sess->se_sess);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001453 }
1454
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001455 while (atomic_read(&isert_conn->post_send_buf_count))
1456 msleep(3000);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001457
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001458 mutex_lock(&isert_conn->conn_mutex);
1459 isert_conn->state = ISER_CONN_DOWN;
1460 mutex_unlock(&isert_conn->conn_mutex);
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -07001461
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001462 complete(&isert_conn->conn_wait_comp_err);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001463}
1464
1465static void
1466isert_cq_tx_work(struct work_struct *work)
1467{
1468 struct isert_cq_desc *cq_desc = container_of(work,
1469 struct isert_cq_desc, cq_tx_work);
1470 struct isert_device *device = cq_desc->device;
1471 int cq_index = cq_desc->cq_index;
1472 struct ib_cq *tx_cq = device->dev_tx_cq[cq_index];
1473 struct isert_conn *isert_conn;
1474 struct iser_tx_desc *tx_desc;
1475 struct ib_wc wc;
1476
1477 while (ib_poll_cq(tx_cq, 1, &wc) == 1) {
1478 tx_desc = (struct iser_tx_desc *)(unsigned long)wc.wr_id;
1479 isert_conn = wc.qp->qp_context;
1480
1481 if (wc.status == IB_WC_SUCCESS) {
1482 isert_send_completion(tx_desc, isert_conn);
1483 } else {
1484 pr_debug("TX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n");
1485 pr_debug("TX wc.status: 0x%08x\n", wc.status);
1486 atomic_dec(&isert_conn->post_send_buf_count);
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001487 isert_cq_tx_comp_err(tx_desc, isert_conn);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001488 }
1489 }
1490
1491 ib_req_notify_cq(tx_cq, IB_CQ_NEXT_COMP);
1492}
1493
1494static void
1495isert_cq_tx_callback(struct ib_cq *cq, void *context)
1496{
1497 struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context;
1498
1499 INIT_WORK(&cq_desc->cq_tx_work, isert_cq_tx_work);
1500 queue_work(isert_comp_wq, &cq_desc->cq_tx_work);
1501}
1502
1503static void
1504isert_cq_rx_work(struct work_struct *work)
1505{
1506 struct isert_cq_desc *cq_desc = container_of(work,
1507 struct isert_cq_desc, cq_rx_work);
1508 struct isert_device *device = cq_desc->device;
1509 int cq_index = cq_desc->cq_index;
1510 struct ib_cq *rx_cq = device->dev_rx_cq[cq_index];
1511 struct isert_conn *isert_conn;
1512 struct iser_rx_desc *rx_desc;
1513 struct ib_wc wc;
1514 unsigned long xfer_len;
1515
1516 while (ib_poll_cq(rx_cq, 1, &wc) == 1) {
1517 rx_desc = (struct iser_rx_desc *)(unsigned long)wc.wr_id;
1518 isert_conn = wc.qp->qp_context;
1519
1520 if (wc.status == IB_WC_SUCCESS) {
1521 xfer_len = (unsigned long)wc.byte_len;
1522 isert_rx_completion(rx_desc, isert_conn, xfer_len);
1523 } else {
1524 pr_debug("RX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n");
1525 if (wc.status != IB_WC_WR_FLUSH_ERR)
1526 pr_debug("RX wc.status: 0x%08x\n", wc.status);
1527
1528 isert_conn->post_recv_buf_count--;
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001529 isert_cq_rx_comp_err(isert_conn);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001530 }
1531 }
1532
1533 ib_req_notify_cq(rx_cq, IB_CQ_NEXT_COMP);
1534}
1535
1536static void
1537isert_cq_rx_callback(struct ib_cq *cq, void *context)
1538{
1539 struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context;
1540
1541 INIT_WORK(&cq_desc->cq_rx_work, isert_cq_rx_work);
1542 queue_work(isert_rx_wq, &cq_desc->cq_rx_work);
1543}
1544
1545static int
1546isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd)
1547{
1548 struct ib_send_wr *wr_failed;
1549 int ret;
1550
1551 atomic_inc(&isert_conn->post_send_buf_count);
1552
1553 ret = ib_post_send(isert_conn->conn_qp, &isert_cmd->tx_desc.send_wr,
1554 &wr_failed);
1555 if (ret) {
1556 pr_err("ib_post_send failed with %d\n", ret);
1557 atomic_dec(&isert_conn->post_send_buf_count);
1558 return ret;
1559 }
1560 return ret;
1561}
1562
1563static int
1564isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
1565{
1566 struct isert_cmd *isert_cmd = container_of(cmd,
1567 struct isert_cmd, iscsi_cmd);
1568 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1569 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1570 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)
1571 &isert_cmd->tx_desc.iscsi_header;
1572
1573 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1574 iscsit_build_rsp_pdu(cmd, conn, true, hdr);
1575 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1576 /*
1577 * Attach SENSE DATA payload to iSCSI Response PDU
1578 */
1579 if (cmd->se_cmd.sense_buffer &&
1580 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
1581 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
1582 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1583 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
1584 u32 padding, sense_len;
1585
1586 put_unaligned_be16(cmd->se_cmd.scsi_sense_length,
1587 cmd->sense_buffer);
1588 cmd->se_cmd.scsi_sense_length += sizeof(__be16);
1589
1590 padding = -(cmd->se_cmd.scsi_sense_length) & 3;
1591 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length);
1592 sense_len = cmd->se_cmd.scsi_sense_length + padding;
1593
1594 isert_cmd->sense_buf_dma = ib_dma_map_single(ib_dev,
1595 (void *)cmd->sense_buffer, sense_len,
1596 DMA_TO_DEVICE);
1597
1598 isert_cmd->sense_buf_len = sense_len;
1599 tx_dsg->addr = isert_cmd->sense_buf_dma;
1600 tx_dsg->length = sense_len;
1601 tx_dsg->lkey = isert_conn->conn_mr->lkey;
1602 isert_cmd->tx_desc.num_sge = 2;
1603 }
1604
1605 isert_init_send_wr(isert_cmd, send_wr);
1606
1607 pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1608
1609 return isert_post_response(isert_conn, isert_cmd);
1610}
1611
1612static int
1613isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
1614 bool nopout_response)
1615{
1616 struct isert_cmd *isert_cmd = container_of(cmd,
1617 struct isert_cmd, iscsi_cmd);
1618 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1619 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1620
1621 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1622 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *)
1623 &isert_cmd->tx_desc.iscsi_header,
1624 nopout_response);
1625 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1626 isert_init_send_wr(isert_cmd, send_wr);
1627
1628 pr_debug("Posting NOPIN Reponse IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1629
1630 return isert_post_response(isert_conn, isert_cmd);
1631}
1632
1633static int
1634isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1635{
1636 struct isert_cmd *isert_cmd = container_of(cmd,
1637 struct isert_cmd, iscsi_cmd);
1638 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1639 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1640
1641 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1642 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *)
1643 &isert_cmd->tx_desc.iscsi_header);
1644 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1645 isert_init_send_wr(isert_cmd, send_wr);
1646
1647 pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1648
1649 return isert_post_response(isert_conn, isert_cmd);
1650}
1651
1652static int
1653isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1654{
1655 struct isert_cmd *isert_cmd = container_of(cmd,
1656 struct isert_cmd, iscsi_cmd);
1657 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1658 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1659
1660 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1661 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *)
1662 &isert_cmd->tx_desc.iscsi_header);
1663 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1664 isert_init_send_wr(isert_cmd, send_wr);
1665
1666 pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1667
1668 return isert_post_response(isert_conn, isert_cmd);
1669}
1670
1671static int
1672isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1673{
1674 struct isert_cmd *isert_cmd = container_of(cmd,
1675 struct isert_cmd, iscsi_cmd);
1676 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1677 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
Nicholas Bellingerfff98872013-06-26 02:31:42 -07001678 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1679 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
1680 struct iscsi_reject *hdr =
1681 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001682
1683 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
Nicholas Bellingerfff98872013-06-26 02:31:42 -07001684 iscsit_build_reject(cmd, conn, hdr);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001685 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
Nicholas Bellingerfff98872013-06-26 02:31:42 -07001686
1687 hton24(hdr->dlength, ISCSI_HDR_LEN);
1688 isert_cmd->sense_buf_dma = ib_dma_map_single(ib_dev,
1689 (void *)cmd->buf_ptr, ISCSI_HDR_LEN,
1690 DMA_TO_DEVICE);
1691 isert_cmd->sense_buf_len = ISCSI_HDR_LEN;
1692 tx_dsg->addr = isert_cmd->sense_buf_dma;
1693 tx_dsg->length = ISCSI_HDR_LEN;
1694 tx_dsg->lkey = isert_conn->conn_mr->lkey;
1695 isert_cmd->tx_desc.num_sge = 2;
1696
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001697 isert_init_send_wr(isert_cmd, send_wr);
1698
1699 pr_debug("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1700
1701 return isert_post_response(isert_conn, isert_cmd);
1702}
1703
1704static int
1705isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
1706 struct ib_sge *ib_sge, struct ib_send_wr *send_wr,
1707 u32 data_left, u32 offset)
1708{
1709 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
1710 struct scatterlist *sg_start, *tmp_sg;
1711 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1712 u32 sg_off, page_off;
1713 int i = 0, sg_nents;
1714
1715 sg_off = offset / PAGE_SIZE;
1716 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1717 sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge);
1718 page_off = offset % PAGE_SIZE;
1719
1720 send_wr->sg_list = ib_sge;
1721 send_wr->num_sge = sg_nents;
1722 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc;
1723 /*
1724 * Perform mapping of TCM scatterlist memory ib_sge dma_addr.
1725 */
1726 for_each_sg(sg_start, tmp_sg, sg_nents, i) {
1727 pr_debug("ISER RDMA from SGL dma_addr: 0x%16llx dma_len: %u, page_off: %u\n",
1728 (unsigned long long)tmp_sg->dma_address,
1729 tmp_sg->length, page_off);
1730
1731 ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off;
1732 ib_sge->length = min_t(u32, data_left,
1733 ib_sg_dma_len(ib_dev, tmp_sg) - page_off);
1734 ib_sge->lkey = isert_conn->conn_mr->lkey;
1735
1736 pr_debug("RDMA ib_sge: addr: 0x%16llx length: %u\n",
1737 ib_sge->addr, ib_sge->length);
1738 page_off = 0;
1739 data_left -= ib_sge->length;
1740 ib_sge++;
1741 pr_debug("Incrementing ib_sge pointer to %p\n", ib_sge);
1742 }
1743
1744 pr_debug("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n",
1745 send_wr->sg_list, send_wr->num_sge);
1746
1747 return sg_nents;
1748}
1749
1750static int
1751isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
1752{
1753 struct se_cmd *se_cmd = &cmd->se_cmd;
1754 struct isert_cmd *isert_cmd = container_of(cmd,
1755 struct isert_cmd, iscsi_cmd);
1756 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
1757 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1758 struct ib_send_wr *wr_failed, *send_wr;
1759 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1760 struct ib_sge *ib_sge;
1761 struct scatterlist *sg;
1762 u32 offset = 0, data_len, data_left, rdma_write_max;
1763 int rc, ret = 0, count, sg_nents, i, ib_sge_cnt;
1764
1765 pr_debug("RDMA_WRITE: data_length: %u\n", se_cmd->data_length);
1766
1767 sg = &se_cmd->t_data_sg[0];
1768 sg_nents = se_cmd->t_data_nents;
1769
1770 count = ib_dma_map_sg(ib_dev, sg, sg_nents, DMA_TO_DEVICE);
1771 if (unlikely(!count)) {
1772 pr_err("Unable to map put_datain SGs\n");
1773 return -EINVAL;
1774 }
1775 wr->sge = sg;
1776 wr->num_sge = sg_nents;
1777 pr_debug("Mapped IB count: %u sg: %p sg_nents: %u for RDMA_WRITE\n",
1778 count, sg, sg_nents);
1779
1780 ib_sge = kzalloc(sizeof(struct ib_sge) * sg_nents, GFP_KERNEL);
1781 if (!ib_sge) {
1782 pr_warn("Unable to allocate datain ib_sge\n");
1783 ret = -ENOMEM;
1784 goto unmap_sg;
1785 }
1786 isert_cmd->ib_sge = ib_sge;
1787
1788 pr_debug("Allocated ib_sge: %p from t_data_ents: %d for RDMA_WRITE\n",
1789 ib_sge, se_cmd->t_data_nents);
1790
1791 wr->send_wr_num = DIV_ROUND_UP(sg_nents, isert_conn->max_sge);
1792 wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num,
1793 GFP_KERNEL);
1794 if (!wr->send_wr) {
1795 pr_err("Unable to allocate wr->send_wr\n");
1796 ret = -ENOMEM;
1797 goto unmap_sg;
1798 }
1799 pr_debug("Allocated wr->send_wr: %p wr->send_wr_num: %u\n",
1800 wr->send_wr, wr->send_wr_num);
1801
1802 iscsit_increment_maxcmdsn(cmd, conn->sess);
1803 cmd->stat_sn = conn->stat_sn++;
1804
1805 wr->isert_cmd = isert_cmd;
1806 rdma_write_max = isert_conn->max_sge * PAGE_SIZE;
1807 data_left = se_cmd->data_length;
1808
1809 for (i = 0; i < wr->send_wr_num; i++) {
1810 send_wr = &isert_cmd->rdma_wr.send_wr[i];
1811 data_len = min(data_left, rdma_write_max);
1812
1813 send_wr->opcode = IB_WR_RDMA_WRITE;
1814 send_wr->send_flags = 0;
1815 send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset;
1816 send_wr->wr.rdma.rkey = isert_cmd->read_stag;
1817
1818 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge,
1819 send_wr, data_len, offset);
1820 ib_sge += ib_sge_cnt;
1821
1822 if (i + 1 == wr->send_wr_num)
1823 send_wr->next = &isert_cmd->tx_desc.send_wr;
1824 else
1825 send_wr->next = &wr->send_wr[i + 1];
1826
1827 offset += data_len;
1828 data_left -= data_len;
1829 }
1830 /*
1831 * Build isert_conn->tx_desc for iSCSI response PDU and attach
1832 */
1833 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1834 iscsit_build_rsp_pdu(cmd, conn, false, (struct iscsi_scsi_rsp *)
1835 &isert_cmd->tx_desc.iscsi_header);
1836 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1837 isert_init_send_wr(isert_cmd, &isert_cmd->tx_desc.send_wr);
1838
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001839 atomic_add(wr->send_wr_num + 1, &isert_conn->post_send_buf_count);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001840
1841 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed);
1842 if (rc) {
1843 pr_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n");
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001844 atomic_sub(wr->send_wr_num + 1, &isert_conn->post_send_buf_count);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001845 }
1846 pr_debug("Posted RDMA_WRITE + Response for iSER Data READ\n");
1847 return 1;
1848
1849unmap_sg:
1850 ib_dma_unmap_sg(ib_dev, sg, sg_nents, DMA_TO_DEVICE);
1851 return ret;
1852}
1853
1854static int
1855isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery)
1856{
1857 struct se_cmd *se_cmd = &cmd->se_cmd;
1858 struct isert_cmd *isert_cmd = container_of(cmd,
1859 struct isert_cmd, iscsi_cmd);
1860 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
1861 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1862 struct ib_send_wr *wr_failed, *send_wr;
1863 struct ib_sge *ib_sge;
1864 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1865 struct scatterlist *sg_start;
1866 u32 sg_off, sg_nents, page_off, va_offset = 0;
1867 u32 offset = 0, data_len, data_left, rdma_write_max;
1868 int rc, ret = 0, count, i, ib_sge_cnt;
1869
1870 pr_debug("RDMA_READ: data_length: %u write_data_done: %u\n",
1871 se_cmd->data_length, cmd->write_data_done);
1872
1873 sg_off = cmd->write_data_done / PAGE_SIZE;
1874 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1875 page_off = cmd->write_data_done % PAGE_SIZE;
1876
1877 pr_debug("RDMA_READ: sg_off: %d, sg_start: %p page_off: %d\n",
1878 sg_off, sg_start, page_off);
1879
1880 data_left = se_cmd->data_length - cmd->write_data_done;
1881 sg_nents = se_cmd->t_data_nents - sg_off;
1882
1883 pr_debug("RDMA_READ: data_left: %d, sg_nents: %d\n",
1884 data_left, sg_nents);
1885
1886 count = ib_dma_map_sg(ib_dev, sg_start, sg_nents, DMA_FROM_DEVICE);
1887 if (unlikely(!count)) {
1888 pr_err("Unable to map get_dataout SGs\n");
1889 return -EINVAL;
1890 }
1891 wr->sge = sg_start;
1892 wr->num_sge = sg_nents;
1893 pr_debug("Mapped IB count: %u sg_start: %p sg_nents: %u for RDMA_READ\n",
1894 count, sg_start, sg_nents);
1895
1896 ib_sge = kzalloc(sizeof(struct ib_sge) * sg_nents, GFP_KERNEL);
1897 if (!ib_sge) {
1898 pr_warn("Unable to allocate dataout ib_sge\n");
1899 ret = -ENOMEM;
1900 goto unmap_sg;
1901 }
1902 isert_cmd->ib_sge = ib_sge;
1903
1904 pr_debug("Using ib_sge: %p from sg_ents: %d for RDMA_READ\n",
1905 ib_sge, sg_nents);
1906
1907 wr->send_wr_num = DIV_ROUND_UP(sg_nents, isert_conn->max_sge);
1908 wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num,
1909 GFP_KERNEL);
1910 if (!wr->send_wr) {
1911 pr_debug("Unable to allocate wr->send_wr\n");
1912 ret = -ENOMEM;
1913 goto unmap_sg;
1914 }
1915 pr_debug("Allocated wr->send_wr: %p wr->send_wr_num: %u\n",
1916 wr->send_wr, wr->send_wr_num);
1917
1918 isert_cmd->tx_desc.isert_cmd = isert_cmd;
1919
1920 wr->iser_ib_op = ISER_IB_RDMA_READ;
1921 wr->isert_cmd = isert_cmd;
1922 rdma_write_max = isert_conn->max_sge * PAGE_SIZE;
1923 offset = cmd->write_data_done;
1924
1925 for (i = 0; i < wr->send_wr_num; i++) {
1926 send_wr = &isert_cmd->rdma_wr.send_wr[i];
1927 data_len = min(data_left, rdma_write_max);
1928
1929 send_wr->opcode = IB_WR_RDMA_READ;
1930 send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset;
1931 send_wr->wr.rdma.rkey = isert_cmd->write_stag;
1932
1933 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge,
1934 send_wr, data_len, offset);
1935 ib_sge += ib_sge_cnt;
1936
1937 if (i + 1 == wr->send_wr_num)
1938 send_wr->send_flags = IB_SEND_SIGNALED;
1939 else
1940 send_wr->next = &wr->send_wr[i + 1];
1941
1942 offset += data_len;
1943 va_offset += data_len;
1944 data_left -= data_len;
1945 }
1946
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001947 atomic_add(wr->send_wr_num, &isert_conn->post_send_buf_count);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001948
1949 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed);
1950 if (rc) {
1951 pr_warn("ib_post_send() failed for IB_WR_RDMA_READ\n");
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001952 atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001953 }
1954 pr_debug("Posted RDMA_READ memory for ISER Data WRITE\n");
1955 return 0;
1956
1957unmap_sg:
1958 ib_dma_unmap_sg(ib_dev, sg_start, sg_nents, DMA_FROM_DEVICE);
1959 return ret;
1960}
1961
1962static int
1963isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
1964{
1965 int ret;
1966
1967 switch (state) {
1968 case ISTATE_SEND_NOPIN_WANT_RESPONSE:
1969 ret = isert_put_nopin(cmd, conn, false);
1970 break;
1971 default:
1972 pr_err("Unknown immediate state: 0x%02x\n", state);
1973 ret = -EINVAL;
1974 break;
1975 }
1976
1977 return ret;
1978}
1979
1980static int
1981isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
1982{
1983 int ret;
1984
1985 switch (state) {
1986 case ISTATE_SEND_LOGOUTRSP:
1987 ret = isert_put_logout_rsp(cmd, conn);
1988 if (!ret) {
1989 pr_debug("Returning iSER Logout -EAGAIN\n");
1990 ret = -EAGAIN;
1991 }
1992 break;
1993 case ISTATE_SEND_NOPIN:
1994 ret = isert_put_nopin(cmd, conn, true);
1995 break;
1996 case ISTATE_SEND_TASKMGTRSP:
1997 ret = isert_put_tm_rsp(cmd, conn);
1998 break;
1999 case ISTATE_SEND_REJECT:
2000 ret = isert_put_reject(cmd, conn);
2001 break;
2002 case ISTATE_SEND_STATUS:
2003 /*
2004 * Special case for sending non GOOD SCSI status from TX thread
2005 * context during pre se_cmd excecution failure.
2006 */
2007 ret = isert_put_response(conn, cmd);
2008 break;
2009 default:
2010 pr_err("Unknown response state: 0x%02x\n", state);
2011 ret = -EINVAL;
2012 break;
2013 }
2014
2015 return ret;
2016}
2017
2018static int
2019isert_setup_np(struct iscsi_np *np,
2020 struct __kernel_sockaddr_storage *ksockaddr)
2021{
2022 struct isert_np *isert_np;
2023 struct rdma_cm_id *isert_lid;
2024 struct sockaddr *sa;
2025 int ret;
2026
2027 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL);
2028 if (!isert_np) {
2029 pr_err("Unable to allocate struct isert_np\n");
2030 return -ENOMEM;
2031 }
2032 init_waitqueue_head(&isert_np->np_accept_wq);
2033 mutex_init(&isert_np->np_accept_mutex);
2034 INIT_LIST_HEAD(&isert_np->np_accept_list);
2035 init_completion(&isert_np->np_login_comp);
2036
2037 sa = (struct sockaddr *)ksockaddr;
2038 pr_debug("ksockaddr: %p, sa: %p\n", ksockaddr, sa);
2039 /*
2040 * Setup the np->np_sockaddr from the passed sockaddr setup
2041 * in iscsi_target_configfs.c code..
2042 */
2043 memcpy(&np->np_sockaddr, ksockaddr,
2044 sizeof(struct __kernel_sockaddr_storage));
2045
2046 isert_lid = rdma_create_id(isert_cma_handler, np, RDMA_PS_TCP,
2047 IB_QPT_RC);
2048 if (IS_ERR(isert_lid)) {
2049 pr_err("rdma_create_id() for isert_listen_handler failed: %ld\n",
2050 PTR_ERR(isert_lid));
2051 ret = PTR_ERR(isert_lid);
2052 goto out;
2053 }
2054
2055 ret = rdma_bind_addr(isert_lid, sa);
2056 if (ret) {
2057 pr_err("rdma_bind_addr() for isert_lid failed: %d\n", ret);
2058 goto out_lid;
2059 }
2060
2061 ret = rdma_listen(isert_lid, ISERT_RDMA_LISTEN_BACKLOG);
2062 if (ret) {
2063 pr_err("rdma_listen() for isert_lid failed: %d\n", ret);
2064 goto out_lid;
2065 }
2066
2067 isert_np->np_cm_id = isert_lid;
2068 np->np_context = isert_np;
2069 pr_debug("Setup isert_lid->context: %p\n", isert_lid->context);
2070
2071 return 0;
2072
2073out_lid:
2074 rdma_destroy_id(isert_lid);
2075out:
2076 kfree(isert_np);
2077 return ret;
2078}
2079
2080static int
2081isert_check_accept_queue(struct isert_np *isert_np)
2082{
2083 int empty;
2084
2085 mutex_lock(&isert_np->np_accept_mutex);
2086 empty = list_empty(&isert_np->np_accept_list);
2087 mutex_unlock(&isert_np->np_accept_mutex);
2088
2089 return empty;
2090}
2091
2092static int
2093isert_rdma_accept(struct isert_conn *isert_conn)
2094{
2095 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id;
2096 struct rdma_conn_param cp;
2097 int ret;
2098
2099 memset(&cp, 0, sizeof(struct rdma_conn_param));
2100 cp.responder_resources = isert_conn->responder_resources;
2101 cp.initiator_depth = isert_conn->initiator_depth;
2102 cp.retry_count = 7;
2103 cp.rnr_retry_count = 7;
2104
2105 pr_debug("Before rdma_accept >>>>>>>>>>>>>>>>>>>>.\n");
2106
2107 ret = rdma_accept(cm_id, &cp);
2108 if (ret) {
2109 pr_err("rdma_accept() failed with: %d\n", ret);
2110 return ret;
2111 }
2112
2113 pr_debug("After rdma_accept >>>>>>>>>>>>>>>>>>>>>.\n");
2114
2115 return 0;
2116}
2117
2118static int
2119isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
2120{
2121 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
2122 int ret;
2123
2124 pr_debug("isert_get_login_rx before conn_login_comp conn: %p\n", conn);
2125
2126 ret = wait_for_completion_interruptible(&isert_conn->conn_login_comp);
2127 if (ret)
2128 return ret;
2129
2130 pr_debug("isert_get_login_rx processing login->req: %p\n", login->req);
2131 return 0;
2132}
2133
2134static void
2135isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn,
2136 struct isert_conn *isert_conn)
2137{
2138 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id;
2139 struct rdma_route *cm_route = &cm_id->route;
2140 struct sockaddr_in *sock_in;
2141 struct sockaddr_in6 *sock_in6;
2142
2143 conn->login_family = np->np_sockaddr.ss_family;
2144
2145 if (np->np_sockaddr.ss_family == AF_INET6) {
2146 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr;
2147 snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c",
2148 &sock_in6->sin6_addr.in6_u);
2149 conn->login_port = ntohs(sock_in6->sin6_port);
2150
2151 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr;
2152 snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c",
2153 &sock_in6->sin6_addr.in6_u);
2154 conn->local_port = ntohs(sock_in6->sin6_port);
2155 } else {
2156 sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr;
2157 sprintf(conn->login_ip, "%pI4",
2158 &sock_in->sin_addr.s_addr);
2159 conn->login_port = ntohs(sock_in->sin_port);
2160
2161 sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr;
2162 sprintf(conn->local_ip, "%pI4",
2163 &sock_in->sin_addr.s_addr);
2164 conn->local_port = ntohs(sock_in->sin_port);
2165 }
2166}
2167
2168static int
2169isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn)
2170{
2171 struct isert_np *isert_np = (struct isert_np *)np->np_context;
2172 struct isert_conn *isert_conn;
2173 int max_accept = 0, ret;
2174
2175accept_wait:
2176 ret = wait_event_interruptible(isert_np->np_accept_wq,
2177 !isert_check_accept_queue(isert_np) ||
2178 np->np_thread_state == ISCSI_NP_THREAD_RESET);
2179 if (max_accept > 5)
2180 return -ENODEV;
2181
2182 spin_lock_bh(&np->np_thread_lock);
2183 if (np->np_thread_state == ISCSI_NP_THREAD_RESET) {
2184 spin_unlock_bh(&np->np_thread_lock);
2185 pr_err("ISCSI_NP_THREAD_RESET for isert_accept_np\n");
2186 return -ENODEV;
2187 }
2188 spin_unlock_bh(&np->np_thread_lock);
2189
2190 mutex_lock(&isert_np->np_accept_mutex);
2191 if (list_empty(&isert_np->np_accept_list)) {
2192 mutex_unlock(&isert_np->np_accept_mutex);
2193 max_accept++;
2194 goto accept_wait;
2195 }
2196 isert_conn = list_first_entry(&isert_np->np_accept_list,
2197 struct isert_conn, conn_accept_node);
2198 list_del_init(&isert_conn->conn_accept_node);
2199 mutex_unlock(&isert_np->np_accept_mutex);
2200
2201 conn->context = isert_conn;
2202 isert_conn->conn = conn;
2203 max_accept = 0;
2204
2205 ret = isert_rdma_post_recvl(isert_conn);
2206 if (ret)
2207 return ret;
2208
2209 ret = isert_rdma_accept(isert_conn);
2210 if (ret)
2211 return ret;
2212
2213 isert_set_conn_info(np, conn, isert_conn);
2214
2215 pr_debug("Processing isert_accept_np: isert_conn: %p\n", isert_conn);
2216 return 0;
2217}
2218
2219static void
2220isert_free_np(struct iscsi_np *np)
2221{
2222 struct isert_np *isert_np = (struct isert_np *)np->np_context;
2223
2224 rdma_destroy_id(isert_np->np_cm_id);
2225
2226 np->np_context = NULL;
2227 kfree(isert_np);
2228}
2229
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08002230static void isert_wait_conn(struct iscsi_conn *conn)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002231{
2232 struct isert_conn *isert_conn = conn->context;
2233
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08002234 pr_debug("isert_wait_conn: Starting \n");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002235 /*
2236 * Decrement post_send_buf_count for special case when called
2237 * from isert_do_control_comp() -> iscsit_logout_post_handler()
2238 */
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -07002239 mutex_lock(&isert_conn->conn_mutex);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002240 if (isert_conn->logout_posted)
2241 atomic_dec(&isert_conn->post_send_buf_count);
2242
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -07002243 if (isert_conn->conn_cm_id && isert_conn->state != ISER_CONN_DOWN) {
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08002244 pr_debug("Calling rdma_disconnect from isert_wait_conn\n");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002245 rdma_disconnect(isert_conn->conn_cm_id);
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -07002246 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002247 /*
2248 * Only wait for conn_wait_comp_err if the isert_conn made it
2249 * into full feature phase..
2250 */
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -07002251 if (isert_conn->state == ISER_CONN_INIT) {
2252 mutex_unlock(&isert_conn->conn_mutex);
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -07002253 return;
2254 }
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08002255 if (isert_conn->state == ISER_CONN_UP)
2256 isert_conn->state = ISER_CONN_TERMINATING;
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -07002257 mutex_unlock(&isert_conn->conn_mutex);
2258
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08002259 wait_for_completion(&isert_conn->conn_wait_comp_err);
2260
2261 wait_for_completion(&isert_conn->conn_wait);
2262}
2263
2264static void isert_free_conn(struct iscsi_conn *conn)
2265{
2266 struct isert_conn *isert_conn = conn->context;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002267
2268 isert_put_conn(isert_conn);
2269}
2270
2271static struct iscsit_transport iser_target_transport = {
2272 .name = "IB/iSER",
2273 .transport_type = ISCSI_INFINIBAND,
2274 .owner = THIS_MODULE,
2275 .iscsit_setup_np = isert_setup_np,
2276 .iscsit_accept_np = isert_accept_np,
2277 .iscsit_free_np = isert_free_np,
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08002278 .iscsit_wait_conn = isert_wait_conn,
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002279 .iscsit_free_conn = isert_free_conn,
2280 .iscsit_alloc_cmd = isert_alloc_cmd,
2281 .iscsit_get_login_rx = isert_get_login_rx,
2282 .iscsit_put_login_tx = isert_put_login_tx,
2283 .iscsit_immediate_queue = isert_immediate_queue,
2284 .iscsit_response_queue = isert_response_queue,
2285 .iscsit_get_dataout = isert_get_dataout,
2286 .iscsit_queue_data_in = isert_put_datain,
2287 .iscsit_queue_status = isert_put_response,
2288};
2289
2290static int __init isert_init(void)
2291{
2292 int ret;
2293
2294 isert_rx_wq = alloc_workqueue("isert_rx_wq", 0, 0);
2295 if (!isert_rx_wq) {
2296 pr_err("Unable to allocate isert_rx_wq\n");
2297 return -ENOMEM;
2298 }
2299
2300 isert_comp_wq = alloc_workqueue("isert_comp_wq", 0, 0);
2301 if (!isert_comp_wq) {
2302 pr_err("Unable to allocate isert_comp_wq\n");
2303 ret = -ENOMEM;
2304 goto destroy_rx_wq;
2305 }
2306
2307 isert_cmd_cache = kmem_cache_create("isert_cmd_cache",
2308 sizeof(struct isert_cmd), __alignof__(struct isert_cmd),
2309 0, NULL);
2310 if (!isert_cmd_cache) {
2311 pr_err("Unable to create isert_cmd_cache\n");
2312 ret = -ENOMEM;
2313 goto destroy_tx_cq;
2314 }
2315
2316 iscsit_register_transport(&iser_target_transport);
2317 pr_debug("iSER_TARGET[0] - Loaded iser_target_transport\n");
2318 return 0;
2319
2320destroy_tx_cq:
2321 destroy_workqueue(isert_comp_wq);
2322destroy_rx_wq:
2323 destroy_workqueue(isert_rx_wq);
2324 return ret;
2325}
2326
2327static void __exit isert_exit(void)
2328{
2329 kmem_cache_destroy(isert_cmd_cache);
2330 destroy_workqueue(isert_comp_wq);
2331 destroy_workqueue(isert_rx_wq);
2332 iscsit_unregister_transport(&iser_target_transport);
2333 pr_debug("iSER_TARGET[0] - Released iser_target_transport\n");
2334}
2335
2336MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure");
2337MODULE_VERSION("0.1");
2338MODULE_AUTHOR("nab@Linux-iSCSI.org");
2339MODULE_LICENSE("GPL");
2340
2341module_init(isert_init);
2342module_exit(isert_exit);