blob: bae20f8bb0344f052ae7df19a6229226b7bbaee6 [file] [log] [blame]
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001/*******************************************************************************
2 * This file contains iSCSI extentions for RDMA (iSER) Verbs
3 *
4 * (c) Copyright 2013 RisingTide Systems LLC.
5 *
6 * Nicholas A. Bellinger <nab@linux-iscsi.org>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 ****************************************************************************/
18
19#include <linux/string.h>
20#include <linux/module.h>
21#include <linux/scatterlist.h>
22#include <linux/socket.h>
23#include <linux/in.h>
24#include <linux/in6.h>
25#include <rdma/ib_verbs.h>
26#include <rdma/rdma_cm.h>
27#include <target/target_core_base.h>
28#include <target/target_core_fabric.h>
29#include <target/iscsi/iscsi_transport.h>
Sagi Grimberg8a2629a2014-04-29 13:13:45 +030030#include <linux/semaphore.h>
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -080031
32#include "isert_proto.h"
33#include "ib_isert.h"
34
35#define ISERT_MAX_CONN 8
36#define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN)
37#define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN)
38
39static DEFINE_MUTEX(device_list_mutex);
40static LIST_HEAD(device_list);
41static struct workqueue_struct *isert_rx_wq;
42static struct workqueue_struct *isert_comp_wq;
43static struct kmem_cache *isert_cmd_cache;
44
45static void
46isert_qp_event_callback(struct ib_event *e, void *context)
47{
48 struct isert_conn *isert_conn = (struct isert_conn *)context;
49
50 pr_err("isert_qp_event_callback event: %d\n", e->event);
51 switch (e->event) {
52 case IB_EVENT_COMM_EST:
53 rdma_notify(isert_conn->conn_cm_id, IB_EVENT_COMM_EST);
54 break;
55 case IB_EVENT_QP_LAST_WQE_REACHED:
56 pr_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED:\n");
57 break;
58 default:
59 break;
60 }
61}
62
63static int
64isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr)
65{
66 int ret;
67
68 ret = ib_query_device(ib_dev, devattr);
69 if (ret) {
70 pr_err("ib_query_device() failed: %d\n", ret);
71 return ret;
72 }
73 pr_debug("devattr->max_sge: %d\n", devattr->max_sge);
74 pr_debug("devattr->max_sge_rd: %d\n", devattr->max_sge_rd);
75
76 return 0;
77}
78
79static int
80isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id)
81{
82 struct isert_device *device = isert_conn->conn_device;
83 struct ib_qp_init_attr attr;
84 struct ib_device_attr devattr;
85 int ret, index, min_index = 0;
86
87 memset(&devattr, 0, sizeof(struct ib_device_attr));
88 ret = isert_query_device(cma_id->device, &devattr);
89 if (ret)
90 return ret;
91
92 mutex_lock(&device_list_mutex);
93 for (index = 0; index < device->cqs_used; index++)
94 if (device->cq_active_qps[index] <
95 device->cq_active_qps[min_index])
96 min_index = index;
97 device->cq_active_qps[min_index]++;
98 pr_debug("isert_conn_setup_qp: Using min_index: %d\n", min_index);
99 mutex_unlock(&device_list_mutex);
100
101 memset(&attr, 0, sizeof(struct ib_qp_init_attr));
102 attr.event_handler = isert_qp_event_callback;
103 attr.qp_context = isert_conn;
104 attr.send_cq = device->dev_tx_cq[min_index];
105 attr.recv_cq = device->dev_rx_cq[min_index];
106 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS;
107 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS;
108 /*
109 * FIXME: Use devattr.max_sge - 2 for max_send_sge as
110 * work-around for RDMA_READ..
111 */
112 attr.cap.max_send_sge = devattr.max_sge - 2;
113 isert_conn->max_sge = attr.cap.max_send_sge;
114
115 attr.cap.max_recv_sge = 1;
116 attr.sq_sig_type = IB_SIGNAL_REQ_WR;
117 attr.qp_type = IB_QPT_RC;
118
119 pr_debug("isert_conn_setup_qp cma_id->device: %p\n",
120 cma_id->device);
121 pr_debug("isert_conn_setup_qp conn_pd->device: %p\n",
122 isert_conn->conn_pd->device);
123
124 ret = rdma_create_qp(cma_id, isert_conn->conn_pd, &attr);
125 if (ret) {
126 pr_err("rdma_create_qp failed for cma_id %d\n", ret);
127 return ret;
128 }
129 isert_conn->conn_qp = cma_id->qp;
130 pr_debug("rdma_create_qp() returned success >>>>>>>>>>>>>>>>>>>>>>>>>.\n");
131
132 return 0;
133}
134
135static void
136isert_cq_event_callback(struct ib_event *e, void *context)
137{
138 pr_debug("isert_cq_event_callback event: %d\n", e->event);
139}
140
141static int
142isert_alloc_rx_descriptors(struct isert_conn *isert_conn)
143{
144 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
145 struct iser_rx_desc *rx_desc;
146 struct ib_sge *rx_sg;
147 u64 dma_addr;
148 int i, j;
149
150 isert_conn->conn_rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS *
151 sizeof(struct iser_rx_desc), GFP_KERNEL);
152 if (!isert_conn->conn_rx_descs)
153 goto fail;
154
155 rx_desc = isert_conn->conn_rx_descs;
156
157 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) {
158 dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc,
159 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
160 if (ib_dma_mapping_error(ib_dev, dma_addr))
161 goto dma_map_fail;
162
163 rx_desc->dma_addr = dma_addr;
164
165 rx_sg = &rx_desc->rx_sg;
166 rx_sg->addr = rx_desc->dma_addr;
167 rx_sg->length = ISER_RX_PAYLOAD_SIZE;
168 rx_sg->lkey = isert_conn->conn_mr->lkey;
169 }
170
171 isert_conn->conn_rx_desc_head = 0;
172 return 0;
173
174dma_map_fail:
175 rx_desc = isert_conn->conn_rx_descs;
176 for (j = 0; j < i; j++, rx_desc++) {
177 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
178 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
179 }
180 kfree(isert_conn->conn_rx_descs);
181 isert_conn->conn_rx_descs = NULL;
182fail:
183 return -ENOMEM;
184}
185
186static void
187isert_free_rx_descriptors(struct isert_conn *isert_conn)
188{
189 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
190 struct iser_rx_desc *rx_desc;
191 int i;
192
193 if (!isert_conn->conn_rx_descs)
194 return;
195
196 rx_desc = isert_conn->conn_rx_descs;
197 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) {
198 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
199 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
200 }
201
202 kfree(isert_conn->conn_rx_descs);
203 isert_conn->conn_rx_descs = NULL;
204}
205
206static void isert_cq_tx_callback(struct ib_cq *, void *);
207static void isert_cq_rx_callback(struct ib_cq *, void *);
208
209static int
210isert_create_device_ib_res(struct isert_device *device)
211{
212 struct ib_device *ib_dev = device->ib_device;
213 struct isert_cq_desc *cq_desc;
214 int ret = 0, i, j;
215
216 device->cqs_used = min_t(int, num_online_cpus(),
217 device->ib_device->num_comp_vectors);
218 device->cqs_used = min(ISERT_MAX_CQ, device->cqs_used);
219 pr_debug("Using %d CQs, device %s supports %d vectors\n",
220 device->cqs_used, device->ib_device->name,
221 device->ib_device->num_comp_vectors);
222 device->cq_desc = kzalloc(sizeof(struct isert_cq_desc) *
223 device->cqs_used, GFP_KERNEL);
224 if (!device->cq_desc) {
225 pr_err("Unable to allocate device->cq_desc\n");
226 return -ENOMEM;
227 }
228 cq_desc = device->cq_desc;
229
230 device->dev_pd = ib_alloc_pd(ib_dev);
231 if (IS_ERR(device->dev_pd)) {
232 ret = PTR_ERR(device->dev_pd);
233 pr_err("ib_alloc_pd failed for dev_pd: %d\n", ret);
234 goto out_cq_desc;
235 }
236
237 for (i = 0; i < device->cqs_used; i++) {
238 cq_desc[i].device = device;
239 cq_desc[i].cq_index = i;
240
241 device->dev_rx_cq[i] = ib_create_cq(device->ib_device,
242 isert_cq_rx_callback,
243 isert_cq_event_callback,
244 (void *)&cq_desc[i],
245 ISER_MAX_RX_CQ_LEN, i);
Wei Yongjun0629b402013-10-29 09:56:34 +0800246 if (IS_ERR(device->dev_rx_cq[i])) {
247 ret = PTR_ERR(device->dev_rx_cq[i]);
248 device->dev_rx_cq[i] = NULL;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800249 goto out_cq;
Wei Yongjun0629b402013-10-29 09:56:34 +0800250 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800251
252 device->dev_tx_cq[i] = ib_create_cq(device->ib_device,
253 isert_cq_tx_callback,
254 isert_cq_event_callback,
255 (void *)&cq_desc[i],
256 ISER_MAX_TX_CQ_LEN, i);
Wei Yongjun0629b402013-10-29 09:56:34 +0800257 if (IS_ERR(device->dev_tx_cq[i])) {
258 ret = PTR_ERR(device->dev_tx_cq[i]);
259 device->dev_tx_cq[i] = NULL;
260 goto out_cq;
261 }
262
263 ret = ib_req_notify_cq(device->dev_rx_cq[i], IB_CQ_NEXT_COMP);
264 if (ret)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800265 goto out_cq;
266
Wei Yongjun0629b402013-10-29 09:56:34 +0800267 ret = ib_req_notify_cq(device->dev_tx_cq[i], IB_CQ_NEXT_COMP);
268 if (ret)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800269 goto out_cq;
270 }
271
272 device->dev_mr = ib_get_dma_mr(device->dev_pd, IB_ACCESS_LOCAL_WRITE);
273 if (IS_ERR(device->dev_mr)) {
274 ret = PTR_ERR(device->dev_mr);
275 pr_err("ib_get_dma_mr failed for dev_mr: %d\n", ret);
276 goto out_cq;
277 }
278
279 return 0;
280
281out_cq:
282 for (j = 0; j < i; j++) {
283 cq_desc = &device->cq_desc[j];
284
285 if (device->dev_rx_cq[j]) {
286 cancel_work_sync(&cq_desc->cq_rx_work);
287 ib_destroy_cq(device->dev_rx_cq[j]);
288 }
289 if (device->dev_tx_cq[j]) {
290 cancel_work_sync(&cq_desc->cq_tx_work);
291 ib_destroy_cq(device->dev_tx_cq[j]);
292 }
293 }
294 ib_dealloc_pd(device->dev_pd);
295
296out_cq_desc:
297 kfree(device->cq_desc);
298
299 return ret;
300}
301
302static void
303isert_free_device_ib_res(struct isert_device *device)
304{
305 struct isert_cq_desc *cq_desc;
306 int i;
307
308 for (i = 0; i < device->cqs_used; i++) {
309 cq_desc = &device->cq_desc[i];
310
311 cancel_work_sync(&cq_desc->cq_rx_work);
312 cancel_work_sync(&cq_desc->cq_tx_work);
313 ib_destroy_cq(device->dev_rx_cq[i]);
314 ib_destroy_cq(device->dev_tx_cq[i]);
315 device->dev_rx_cq[i] = NULL;
316 device->dev_tx_cq[i] = NULL;
317 }
318
319 ib_dereg_mr(device->dev_mr);
320 ib_dealloc_pd(device->dev_pd);
321 kfree(device->cq_desc);
322}
323
324static void
325isert_device_try_release(struct isert_device *device)
326{
327 mutex_lock(&device_list_mutex);
328 device->refcount--;
329 if (!device->refcount) {
330 isert_free_device_ib_res(device);
331 list_del(&device->dev_node);
332 kfree(device);
333 }
334 mutex_unlock(&device_list_mutex);
335}
336
337static struct isert_device *
338isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id)
339{
340 struct isert_device *device;
341 int ret;
342
343 mutex_lock(&device_list_mutex);
344 list_for_each_entry(device, &device_list, dev_node) {
345 if (device->ib_device->node_guid == cma_id->device->node_guid) {
346 device->refcount++;
347 mutex_unlock(&device_list_mutex);
348 return device;
349 }
350 }
351
352 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL);
353 if (!device) {
354 mutex_unlock(&device_list_mutex);
355 return ERR_PTR(-ENOMEM);
356 }
357
358 INIT_LIST_HEAD(&device->dev_node);
359
360 device->ib_device = cma_id->device;
361 ret = isert_create_device_ib_res(device);
362 if (ret) {
363 kfree(device);
364 mutex_unlock(&device_list_mutex);
365 return ERR_PTR(ret);
366 }
367
368 device->refcount++;
369 list_add_tail(&device->dev_node, &device_list);
370 mutex_unlock(&device_list_mutex);
371
372 return device;
373}
374
375static int
376isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
377{
378 struct iscsi_np *np = cma_id->context;
379 struct isert_np *isert_np = np->np_context;
380 struct isert_conn *isert_conn;
381 struct isert_device *device;
382 struct ib_device *ib_dev = cma_id->device;
383 int ret = 0;
384
385 pr_debug("Entering isert_connect_request cma_id: %p, context: %p\n",
386 cma_id, cma_id->context);
387
388 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL);
389 if (!isert_conn) {
390 pr_err("Unable to allocate isert_conn\n");
391 return -ENOMEM;
392 }
393 isert_conn->state = ISER_CONN_INIT;
394 INIT_LIST_HEAD(&isert_conn->conn_accept_node);
395 init_completion(&isert_conn->conn_login_comp);
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -0800396 init_completion(&isert_conn->conn_wait);
397 init_completion(&isert_conn->conn_wait_comp_err);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800398 kref_init(&isert_conn->conn_kref);
399 kref_get(&isert_conn->conn_kref);
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -0700400 mutex_init(&isert_conn->conn_mutex);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800401
402 cma_id->context = isert_conn;
403 isert_conn->conn_cm_id = cma_id;
404 isert_conn->responder_resources = event->param.conn.responder_resources;
405 isert_conn->initiator_depth = event->param.conn.initiator_depth;
406 pr_debug("Using responder_resources: %u initiator_depth: %u\n",
407 isert_conn->responder_resources, isert_conn->initiator_depth);
408
409 isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN +
410 ISER_RX_LOGIN_SIZE, GFP_KERNEL);
411 if (!isert_conn->login_buf) {
412 pr_err("Unable to allocate isert_conn->login_buf\n");
413 ret = -ENOMEM;
414 goto out;
415 }
416
417 isert_conn->login_req_buf = isert_conn->login_buf;
418 isert_conn->login_rsp_buf = isert_conn->login_buf +
419 ISCSI_DEF_MAX_RECV_SEG_LEN;
420 pr_debug("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n",
421 isert_conn->login_buf, isert_conn->login_req_buf,
422 isert_conn->login_rsp_buf);
423
424 isert_conn->login_req_dma = ib_dma_map_single(ib_dev,
425 (void *)isert_conn->login_req_buf,
426 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE);
427
428 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma);
429 if (ret) {
430 pr_err("ib_dma_mapping_error failed for login_req_dma: %d\n",
431 ret);
432 isert_conn->login_req_dma = 0;
433 goto out_login_buf;
434 }
435
436 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev,
437 (void *)isert_conn->login_rsp_buf,
438 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
439
440 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma);
441 if (ret) {
442 pr_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n",
443 ret);
444 isert_conn->login_rsp_dma = 0;
445 goto out_req_dma_map;
446 }
447
448 device = isert_device_find_by_ib_dev(cma_id);
449 if (IS_ERR(device)) {
450 ret = PTR_ERR(device);
451 goto out_rsp_dma_map;
452 }
453
454 isert_conn->conn_device = device;
455 isert_conn->conn_pd = device->dev_pd;
456 isert_conn->conn_mr = device->dev_mr;
457
458 ret = isert_conn_setup_qp(isert_conn, cma_id);
459 if (ret)
460 goto out_conn_dev;
461
462 mutex_lock(&isert_np->np_accept_mutex);
Sagi Grimberg5de94f82014-04-29 13:13:44 +0300463 list_add_tail(&isert_conn->conn_accept_node, &isert_np->np_accept_list);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800464 mutex_unlock(&isert_np->np_accept_mutex);
465
Sagi Grimberg8a2629a2014-04-29 13:13:45 +0300466 pr_debug("isert_connect_request() up np_sem np: %p\n", np);
467 up(&isert_np->np_sem);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800468 return 0;
469
470out_conn_dev:
471 isert_device_try_release(device);
472out_rsp_dma_map:
473 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma,
474 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
475out_req_dma_map:
476 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
477 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE);
478out_login_buf:
479 kfree(isert_conn->login_buf);
480out:
481 kfree(isert_conn);
482 return ret;
483}
484
485static void
486isert_connect_release(struct isert_conn *isert_conn)
487{
488 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
489 struct isert_device *device = isert_conn->conn_device;
490 int cq_index;
491
492 pr_debug("Entering isert_connect_release(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
493
494 if (isert_conn->conn_qp) {
495 cq_index = ((struct isert_cq_desc *)
496 isert_conn->conn_qp->recv_cq->cq_context)->cq_index;
497 pr_debug("isert_connect_release: cq_index: %d\n", cq_index);
498 isert_conn->conn_device->cq_active_qps[cq_index]--;
499
500 rdma_destroy_qp(isert_conn->conn_cm_id);
501 }
502
503 isert_free_rx_descriptors(isert_conn);
504 rdma_destroy_id(isert_conn->conn_cm_id);
505
506 if (isert_conn->login_buf) {
507 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma,
508 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
509 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
510 ISCSI_DEF_MAX_RECV_SEG_LEN,
511 DMA_FROM_DEVICE);
512 kfree(isert_conn->login_buf);
513 }
514 kfree(isert_conn);
515
516 if (device)
517 isert_device_try_release(device);
518
519 pr_debug("Leaving isert_connect_release >>>>>>>>>>>>\n");
520}
521
522static void
523isert_connected_handler(struct rdma_cm_id *cma_id)
524{
525 return;
526}
527
528static void
529isert_release_conn_kref(struct kref *kref)
530{
531 struct isert_conn *isert_conn = container_of(kref,
532 struct isert_conn, conn_kref);
533
534 pr_debug("Calling isert_connect_release for final kref %s/%d\n",
535 current->comm, current->pid);
536
537 isert_connect_release(isert_conn);
538}
539
540static void
541isert_put_conn(struct isert_conn *isert_conn)
542{
543 kref_put(&isert_conn->conn_kref, isert_release_conn_kref);
544}
545
546static void
547isert_disconnect_work(struct work_struct *work)
548{
549 struct isert_conn *isert_conn = container_of(work,
550 struct isert_conn, conn_logout_work);
551
552 pr_debug("isert_disconnect_work(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n");
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -0700553 mutex_lock(&isert_conn->conn_mutex);
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -0800554 if (isert_conn->state == ISER_CONN_UP)
555 isert_conn->state = ISER_CONN_TERMINATING;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800556
557 if (isert_conn->post_recv_buf_count == 0 &&
558 atomic_read(&isert_conn->post_send_buf_count) == 0) {
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -0700559 mutex_unlock(&isert_conn->conn_mutex);
560 goto wake_up;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800561 }
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -0700562 if (!isert_conn->conn_cm_id) {
563 mutex_unlock(&isert_conn->conn_mutex);
564 isert_put_conn(isert_conn);
565 return;
566 }
567 if (!isert_conn->logout_posted) {
568 pr_debug("Calling rdma_disconnect for !logout_posted from"
569 " isert_disconnect_work\n");
570 rdma_disconnect(isert_conn->conn_cm_id);
571 mutex_unlock(&isert_conn->conn_mutex);
572 iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
573 goto wake_up;
574 }
575 mutex_unlock(&isert_conn->conn_mutex);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800576
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -0700577wake_up:
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -0800578 complete(&isert_conn->conn_wait);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800579 isert_put_conn(isert_conn);
580}
581
582static void
583isert_disconnected_handler(struct rdma_cm_id *cma_id)
584{
585 struct isert_conn *isert_conn = (struct isert_conn *)cma_id->context;
586
587 INIT_WORK(&isert_conn->conn_logout_work, isert_disconnect_work);
588 schedule_work(&isert_conn->conn_logout_work);
589}
590
591static int
592isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
593{
594 int ret = 0;
595
596 pr_debug("isert_cma_handler: event %d status %d conn %p id %p\n",
597 event->event, event->status, cma_id->context, cma_id);
598
599 switch (event->event) {
600 case RDMA_CM_EVENT_CONNECT_REQUEST:
601 pr_debug("RDMA_CM_EVENT_CONNECT_REQUEST: >>>>>>>>>>>>>>>\n");
602 ret = isert_connect_request(cma_id, event);
603 break;
604 case RDMA_CM_EVENT_ESTABLISHED:
605 pr_debug("RDMA_CM_EVENT_ESTABLISHED >>>>>>>>>>>>>>\n");
606 isert_connected_handler(cma_id);
607 break;
608 case RDMA_CM_EVENT_DISCONNECTED:
609 pr_debug("RDMA_CM_EVENT_DISCONNECTED: >>>>>>>>>>>>>>\n");
610 isert_disconnected_handler(cma_id);
611 break;
612 case RDMA_CM_EVENT_DEVICE_REMOVAL:
613 case RDMA_CM_EVENT_ADDR_CHANGE:
614 break;
615 case RDMA_CM_EVENT_CONNECT_ERROR:
616 default:
617 pr_err("Unknown RDMA CMA event: %d\n", event->event);
618 break;
619 }
620
621 if (ret != 0) {
622 pr_err("isert_cma_handler failed RDMA_CM_EVENT: 0x%08x %d\n",
623 event->event, ret);
624 dump_stack();
625 }
626
627 return ret;
628}
629
630static int
631isert_post_recv(struct isert_conn *isert_conn, u32 count)
632{
633 struct ib_recv_wr *rx_wr, *rx_wr_failed;
634 int i, ret;
635 unsigned int rx_head = isert_conn->conn_rx_desc_head;
636 struct iser_rx_desc *rx_desc;
637
638 for (rx_wr = isert_conn->conn_rx_wr, i = 0; i < count; i++, rx_wr++) {
639 rx_desc = &isert_conn->conn_rx_descs[rx_head];
640 rx_wr->wr_id = (unsigned long)rx_desc;
641 rx_wr->sg_list = &rx_desc->rx_sg;
642 rx_wr->num_sge = 1;
643 rx_wr->next = rx_wr + 1;
644 rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1);
645 }
646
647 rx_wr--;
648 rx_wr->next = NULL; /* mark end of work requests list */
649
650 isert_conn->post_recv_buf_count += count;
651 ret = ib_post_recv(isert_conn->conn_qp, isert_conn->conn_rx_wr,
652 &rx_wr_failed);
653 if (ret) {
654 pr_err("ib_post_recv() failed with ret: %d\n", ret);
655 isert_conn->post_recv_buf_count -= count;
656 } else {
657 pr_debug("isert_post_recv(): Posted %d RX buffers\n", count);
658 isert_conn->conn_rx_desc_head = rx_head;
659 }
660 return ret;
661}
662
663static int
664isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc)
665{
666 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
667 struct ib_send_wr send_wr, *send_wr_failed;
668 int ret;
669
670 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr,
671 ISER_HEADERS_LEN, DMA_TO_DEVICE);
672
673 send_wr.next = NULL;
674 send_wr.wr_id = (unsigned long)tx_desc;
675 send_wr.sg_list = tx_desc->tx_sg;
676 send_wr.num_sge = tx_desc->num_sge;
677 send_wr.opcode = IB_WR_SEND;
678 send_wr.send_flags = IB_SEND_SIGNALED;
679
680 atomic_inc(&isert_conn->post_send_buf_count);
681
682 ret = ib_post_send(isert_conn->conn_qp, &send_wr, &send_wr_failed);
683 if (ret) {
684 pr_err("ib_post_send() failed, ret: %d\n", ret);
685 atomic_dec(&isert_conn->post_send_buf_count);
686 }
687
688 return ret;
689}
690
691static void
692isert_create_send_desc(struct isert_conn *isert_conn,
693 struct isert_cmd *isert_cmd,
694 struct iser_tx_desc *tx_desc)
695{
696 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
697
698 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr,
699 ISER_HEADERS_LEN, DMA_TO_DEVICE);
700
701 memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr));
702 tx_desc->iser_header.flags = ISER_VER;
703
704 tx_desc->num_sge = 1;
705 tx_desc->isert_cmd = isert_cmd;
706
707 if (tx_desc->tx_sg[0].lkey != isert_conn->conn_mr->lkey) {
708 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey;
709 pr_debug("tx_desc %p lkey mismatch, fixing\n", tx_desc);
710 }
711}
712
713static int
714isert_init_tx_hdrs(struct isert_conn *isert_conn,
715 struct iser_tx_desc *tx_desc)
716{
717 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
718 u64 dma_addr;
719
720 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc,
721 ISER_HEADERS_LEN, DMA_TO_DEVICE);
722 if (ib_dma_mapping_error(ib_dev, dma_addr)) {
723 pr_err("ib_dma_mapping_error() failed\n");
724 return -ENOMEM;
725 }
726
727 tx_desc->dma_addr = dma_addr;
728 tx_desc->tx_sg[0].addr = tx_desc->dma_addr;
729 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN;
730 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey;
731
732 pr_debug("isert_init_tx_hdrs: Setup tx_sg[0].addr: 0x%llx length: %u"
733 " lkey: 0x%08x\n", tx_desc->tx_sg[0].addr,
734 tx_desc->tx_sg[0].length, tx_desc->tx_sg[0].lkey);
735
736 return 0;
737}
738
739static void
740isert_init_send_wr(struct isert_cmd *isert_cmd, struct ib_send_wr *send_wr)
741{
742 isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND;
743 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc;
744 send_wr->opcode = IB_WR_SEND;
745 send_wr->send_flags = IB_SEND_SIGNALED;
746 send_wr->sg_list = &isert_cmd->tx_desc.tx_sg[0];
747 send_wr->num_sge = isert_cmd->tx_desc.num_sge;
748}
749
750static int
751isert_rdma_post_recvl(struct isert_conn *isert_conn)
752{
753 struct ib_recv_wr rx_wr, *rx_wr_fail;
754 struct ib_sge sge;
755 int ret;
756
757 memset(&sge, 0, sizeof(struct ib_sge));
758 sge.addr = isert_conn->login_req_dma;
759 sge.length = ISER_RX_LOGIN_SIZE;
760 sge.lkey = isert_conn->conn_mr->lkey;
761
762 pr_debug("Setup sge: addr: %llx length: %d 0x%08x\n",
763 sge.addr, sge.length, sge.lkey);
764
765 memset(&rx_wr, 0, sizeof(struct ib_recv_wr));
766 rx_wr.wr_id = (unsigned long)isert_conn->login_req_buf;
767 rx_wr.sg_list = &sge;
768 rx_wr.num_sge = 1;
769
770 isert_conn->post_recv_buf_count++;
771 ret = ib_post_recv(isert_conn->conn_qp, &rx_wr, &rx_wr_fail);
772 if (ret) {
773 pr_err("ib_post_recv() failed: %d\n", ret);
774 isert_conn->post_recv_buf_count--;
775 }
776
777 pr_debug("ib_post_recv(): returned success >>>>>>>>>>>>>>>>>>>>>>>>\n");
778 return ret;
779}
780
781static int
782isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
783 u32 length)
784{
785 struct isert_conn *isert_conn = conn->context;
786 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
787 struct iser_tx_desc *tx_desc = &isert_conn->conn_login_tx_desc;
788 int ret;
789
790 isert_create_send_desc(isert_conn, NULL, tx_desc);
791
792 memcpy(&tx_desc->iscsi_header, &login->rsp[0],
793 sizeof(struct iscsi_hdr));
794
795 isert_init_tx_hdrs(isert_conn, tx_desc);
796
797 if (length > 0) {
798 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1];
799
800 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma,
801 length, DMA_TO_DEVICE);
802
803 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length);
804
805 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma,
806 length, DMA_TO_DEVICE);
807
808 tx_dsg->addr = isert_conn->login_rsp_dma;
809 tx_dsg->length = length;
810 tx_dsg->lkey = isert_conn->conn_mr->lkey;
811 tx_desc->num_sge = 2;
812 }
813 if (!login->login_failed) {
814 if (login->login_complete) {
815 ret = isert_alloc_rx_descriptors(isert_conn);
816 if (ret)
817 return ret;
818
819 ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX);
820 if (ret)
821 return ret;
822
823 isert_conn->state = ISER_CONN_UP;
824 goto post_send;
825 }
826
827 ret = isert_rdma_post_recvl(isert_conn);
828 if (ret)
829 return ret;
830 }
831post_send:
832 ret = isert_post_send(isert_conn, tx_desc);
833 if (ret)
834 return ret;
835
836 return 0;
837}
838
839static void
840isert_rx_login_req(struct iser_rx_desc *rx_desc, int rx_buflen,
841 struct isert_conn *isert_conn)
842{
843 struct iscsi_conn *conn = isert_conn->conn;
844 struct iscsi_login *login = conn->conn_login;
845 int size;
846
847 if (!login) {
848 pr_err("conn->conn_login is NULL\n");
849 dump_stack();
850 return;
851 }
852
853 if (login->first_request) {
854 struct iscsi_login_req *login_req =
855 (struct iscsi_login_req *)&rx_desc->iscsi_header;
856 /*
857 * Setup the initial iscsi_login values from the leading
858 * login request PDU.
859 */
860 login->leading_connection = (!login_req->tsih) ? 1 : 0;
861 login->current_stage =
862 (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK)
863 >> 2;
864 login->version_min = login_req->min_version;
865 login->version_max = login_req->max_version;
866 memcpy(login->isid, login_req->isid, 6);
867 login->cmd_sn = be32_to_cpu(login_req->cmdsn);
868 login->init_task_tag = login_req->itt;
869 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
870 login->cid = be16_to_cpu(login_req->cid);
871 login->tsih = be16_to_cpu(login_req->tsih);
872 }
873
874 memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN);
875
876 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS);
877 pr_debug("Using login payload size: %d, rx_buflen: %d MAX_KEY_VALUE_PAIRS: %d\n",
878 size, rx_buflen, MAX_KEY_VALUE_PAIRS);
879 memcpy(login->req_buf, &rx_desc->data[0], size);
880
881 complete(&isert_conn->conn_login_comp);
882}
883
884static void
885isert_release_cmd(struct iscsi_cmd *cmd)
886{
887 struct isert_cmd *isert_cmd = container_of(cmd, struct isert_cmd,
888 iscsi_cmd);
889
890 pr_debug("Entering isert_release_cmd %p >>>>>>>>>>>>>>>.\n", isert_cmd);
891
892 kfree(cmd->buf_ptr);
893 kfree(cmd->tmr_req);
894
895 kmem_cache_free(isert_cmd_cache, isert_cmd);
896}
897
898static struct iscsi_cmd
899*isert_alloc_cmd(struct iscsi_conn *conn, gfp_t gfp)
900{
901 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
902 struct isert_cmd *isert_cmd;
903
904 isert_cmd = kmem_cache_zalloc(isert_cmd_cache, gfp);
905 if (!isert_cmd) {
906 pr_err("Unable to allocate isert_cmd\n");
907 return NULL;
908 }
909 isert_cmd->conn = isert_conn;
910 isert_cmd->iscsi_cmd.release_cmd = &isert_release_cmd;
911
912 return &isert_cmd->iscsi_cmd;
913}
914
915static int
916isert_handle_scsi_cmd(struct isert_conn *isert_conn,
917 struct isert_cmd *isert_cmd, struct iser_rx_desc *rx_desc,
918 unsigned char *buf)
919{
920 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
921 struct iscsi_conn *conn = isert_conn->conn;
922 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf;
923 struct scatterlist *sg;
924 int imm_data, imm_data_len, unsol_data, sg_nents, rc;
925 bool dump_payload = false;
926
927 rc = iscsit_setup_scsi_cmd(conn, cmd, buf);
928 if (rc < 0)
929 return rc;
930
931 imm_data = cmd->immediate_data;
932 imm_data_len = cmd->first_burst_len;
933 unsol_data = cmd->unsolicited_data;
934
935 rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
936 if (rc < 0) {
937 return 0;
938 } else if (rc > 0) {
939 dump_payload = true;
940 goto sequence_cmd;
941 }
942
943 if (!imm_data)
944 return 0;
945
946 sg = &cmd->se_cmd.t_data_sg[0];
947 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE));
948
949 pr_debug("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n",
950 sg, sg_nents, &rx_desc->data[0], imm_data_len);
951
952 sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len);
953
954 cmd->write_data_done += imm_data_len;
955
956 if (cmd->write_data_done == cmd->se_cmd.data_length) {
957 spin_lock_bh(&cmd->istate_lock);
958 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
959 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
960 spin_unlock_bh(&cmd->istate_lock);
961 }
962
963sequence_cmd:
Nicholas Bellingeradb97c22013-07-30 04:04:02 +0000964 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800965
966 if (!rc && dump_payload == false && unsol_data)
967 iscsit_set_unsoliticed_dataout(cmd);
Nicholas Bellinger553e4c52014-05-23 00:48:35 -0700968 else if (dump_payload && imm_data)
969 target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800970
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800971 return 0;
972}
973
974static int
975isert_handle_iscsi_dataout(struct isert_conn *isert_conn,
976 struct iser_rx_desc *rx_desc, unsigned char *buf)
977{
978 struct scatterlist *sg_start;
979 struct iscsi_conn *conn = isert_conn->conn;
980 struct iscsi_cmd *cmd = NULL;
981 struct iscsi_data *hdr = (struct iscsi_data *)buf;
982 u32 unsol_data_len = ntoh24(hdr->dlength);
983 int rc, sg_nents, sg_off, page_off;
984
985 rc = iscsit_check_dataout_hdr(conn, buf, &cmd);
986 if (rc < 0)
987 return rc;
988 else if (!cmd)
989 return 0;
990 /*
991 * FIXME: Unexpected unsolicited_data out
992 */
993 if (!cmd->unsolicited_data) {
994 pr_err("Received unexpected solicited data payload\n");
995 dump_stack();
996 return -1;
997 }
998
999 pr_debug("Unsolicited DataOut unsol_data_len: %u, write_data_done: %u, data_length: %u\n",
1000 unsol_data_len, cmd->write_data_done, cmd->se_cmd.data_length);
1001
1002 sg_off = cmd->write_data_done / PAGE_SIZE;
1003 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1004 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE));
1005 page_off = cmd->write_data_done % PAGE_SIZE;
1006 /*
1007 * FIXME: Non page-aligned unsolicited_data out
1008 */
1009 if (page_off) {
1010 pr_err("Received unexpected non-page aligned data payload\n");
1011 dump_stack();
1012 return -1;
1013 }
1014 pr_debug("Copying DataOut: sg_start: %p, sg_off: %u sg_nents: %u from %p %u\n",
1015 sg_start, sg_off, sg_nents, &rx_desc->data[0], unsol_data_len);
1016
1017 sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0],
1018 unsol_data_len);
1019
1020 rc = iscsit_check_dataout_payload(cmd, hdr, false);
1021 if (rc < 0)
1022 return rc;
1023
1024 return 0;
1025}
1026
1027static int
1028isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc,
1029 uint32_t read_stag, uint64_t read_va,
1030 uint32_t write_stag, uint64_t write_va)
1031{
1032 struct iscsi_hdr *hdr = &rx_desc->iscsi_header;
1033 struct iscsi_conn *conn = isert_conn->conn;
1034 struct iscsi_cmd *cmd;
1035 struct isert_cmd *isert_cmd;
1036 int ret = -EINVAL;
1037 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1038
1039 switch (opcode) {
1040 case ISCSI_OP_SCSI_CMD:
1041 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1042 if (!cmd)
1043 break;
1044
1045 isert_cmd = container_of(cmd, struct isert_cmd, iscsi_cmd);
1046 isert_cmd->read_stag = read_stag;
1047 isert_cmd->read_va = read_va;
1048 isert_cmd->write_stag = write_stag;
1049 isert_cmd->write_va = write_va;
1050
1051 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd,
1052 rx_desc, (unsigned char *)hdr);
1053 break;
1054 case ISCSI_OP_NOOP_OUT:
1055 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1056 if (!cmd)
1057 break;
1058
1059 ret = iscsit_handle_nop_out(conn, cmd, (unsigned char *)hdr);
1060 break;
1061 case ISCSI_OP_SCSI_DATA_OUT:
1062 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc,
1063 (unsigned char *)hdr);
1064 break;
1065 case ISCSI_OP_SCSI_TMFUNC:
1066 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1067 if (!cmd)
1068 break;
1069
1070 ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1071 (unsigned char *)hdr);
1072 break;
1073 case ISCSI_OP_LOGOUT:
1074 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1075 if (!cmd)
1076 break;
1077
1078 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
1079 if (ret > 0)
1080 wait_for_completion_timeout(&conn->conn_logout_comp,
1081 SECONDS_FOR_LOGOUT_COMP *
1082 HZ);
1083 break;
1084 default:
1085 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1086 dump_stack();
1087 break;
1088 }
1089
1090 return ret;
1091}
1092
1093static void
1094isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn)
1095{
1096 struct iser_hdr *iser_hdr = &rx_desc->iser_header;
1097 uint64_t read_va = 0, write_va = 0;
1098 uint32_t read_stag = 0, write_stag = 0;
1099 int rc;
1100
1101 switch (iser_hdr->flags & 0xF0) {
1102 case ISCSI_CTRL:
1103 if (iser_hdr->flags & ISER_RSV) {
1104 read_stag = be32_to_cpu(iser_hdr->read_stag);
1105 read_va = be64_to_cpu(iser_hdr->read_va);
1106 pr_debug("ISER_RSV: read_stag: 0x%08x read_va: 0x%16llx\n",
1107 read_stag, (unsigned long long)read_va);
1108 }
1109 if (iser_hdr->flags & ISER_WSV) {
1110 write_stag = be32_to_cpu(iser_hdr->write_stag);
1111 write_va = be64_to_cpu(iser_hdr->write_va);
1112 pr_debug("ISER_WSV: write__stag: 0x%08x write_va: 0x%16llx\n",
1113 write_stag, (unsigned long long)write_va);
1114 }
1115
1116 pr_debug("ISER ISCSI_CTRL PDU\n");
1117 break;
1118 case ISER_HELLO:
1119 pr_err("iSER Hello message\n");
1120 break;
1121 default:
1122 pr_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags);
1123 break;
1124 }
1125
1126 rc = isert_rx_opcode(isert_conn, rx_desc,
1127 read_stag, read_va, write_stag, write_va);
1128}
1129
1130static void
1131isert_rx_completion(struct iser_rx_desc *desc, struct isert_conn *isert_conn,
1132 unsigned long xfer_len)
1133{
1134 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1135 struct iscsi_hdr *hdr;
1136 u64 rx_dma;
1137 int rx_buflen, outstanding;
1138
1139 if ((char *)desc == isert_conn->login_req_buf) {
1140 rx_dma = isert_conn->login_req_dma;
1141 rx_buflen = ISER_RX_LOGIN_SIZE;
1142 pr_debug("ISER login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n",
1143 rx_dma, rx_buflen);
1144 } else {
1145 rx_dma = desc->dma_addr;
1146 rx_buflen = ISER_RX_PAYLOAD_SIZE;
1147 pr_debug("ISER req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n",
1148 rx_dma, rx_buflen);
1149 }
1150
1151 ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE);
1152
1153 hdr = &desc->iscsi_header;
1154 pr_debug("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n",
1155 hdr->opcode, hdr->itt, hdr->flags,
1156 (int)(xfer_len - ISER_HEADERS_LEN));
1157
1158 if ((char *)desc == isert_conn->login_req_buf)
1159 isert_rx_login_req(desc, xfer_len - ISER_HEADERS_LEN,
1160 isert_conn);
1161 else
1162 isert_rx_do_work(desc, isert_conn);
1163
1164 ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen,
1165 DMA_FROM_DEVICE);
1166
1167 isert_conn->post_recv_buf_count--;
1168 pr_debug("iSERT: Decremented post_recv_buf_count: %d\n",
1169 isert_conn->post_recv_buf_count);
1170
1171 if ((char *)desc == isert_conn->login_req_buf)
1172 return;
1173
1174 outstanding = isert_conn->post_recv_buf_count;
1175 if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) {
1176 int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding,
1177 ISERT_MIN_POSTED_RX);
1178 err = isert_post_recv(isert_conn, count);
1179 if (err) {
1180 pr_err("isert_post_recv() count: %d failed, %d\n",
1181 count, err);
1182 }
1183 }
1184}
1185
1186static void
1187isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn)
1188{
1189 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
1190 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1191
1192 pr_debug("isert_unmap_cmd >>>>>>>>>>>>>>>>>>>>>>>\n");
1193
1194 if (wr->sge) {
1195 ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, DMA_TO_DEVICE);
1196 wr->sge = NULL;
1197 }
1198
1199 kfree(wr->send_wr);
1200 wr->send_wr = NULL;
1201
1202 kfree(isert_cmd->ib_sge);
1203 isert_cmd->ib_sge = NULL;
1204}
1205
1206static void
Nicholas Bellingerd3066062014-03-30 15:50:03 -07001207isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001208{
1209 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
1210 struct isert_conn *isert_conn = isert_cmd->conn;
Nicholas Bellingerc6ccbb92013-07-03 03:11:48 -07001211 struct iscsi_conn *conn = isert_conn->conn;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001212
1213 pr_debug("Entering isert_put_cmd: %p\n", isert_cmd);
1214
1215 switch (cmd->iscsi_opcode) {
1216 case ISCSI_OP_SCSI_CMD:
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001217 spin_lock_bh(&conn->cmd_lock);
1218 if (!list_empty(&cmd->i_conn_node))
Nicholas Bellingeraf737f62014-02-03 12:53:51 -08001219 list_del_init(&cmd->i_conn_node);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001220 spin_unlock_bh(&conn->cmd_lock);
1221
Nicholas Bellingerd3066062014-03-30 15:50:03 -07001222 if (cmd->data_direction == DMA_TO_DEVICE) {
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001223 iscsit_stop_dataout_timer(cmd);
Nicholas Bellingerd3066062014-03-30 15:50:03 -07001224 /*
1225 * Check for special case during comp_err where
1226 * WRITE_PENDING has been handed off from core,
1227 * but requires an extra target_put_sess_cmd()
1228 * before transport_generic_free_cmd() below.
1229 */
1230 if (comp_err &&
1231 cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) {
1232 struct se_cmd *se_cmd = &cmd->se_cmd;
1233
1234 target_put_sess_cmd(se_cmd->se_sess, se_cmd);
1235 }
1236 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001237
1238 isert_unmap_cmd(isert_cmd, isert_conn);
Nicholas Bellingerc6ccbb92013-07-03 03:11:48 -07001239 transport_generic_free_cmd(&cmd->se_cmd, 0);
1240 break;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001241 case ISCSI_OP_SCSI_TMFUNC:
Nicholas Bellingerc6ccbb92013-07-03 03:11:48 -07001242 spin_lock_bh(&conn->cmd_lock);
1243 if (!list_empty(&cmd->i_conn_node))
Nicholas Bellingeraf737f62014-02-03 12:53:51 -08001244 list_del_init(&cmd->i_conn_node);
Nicholas Bellingerc6ccbb92013-07-03 03:11:48 -07001245 spin_unlock_bh(&conn->cmd_lock);
1246
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001247 transport_generic_free_cmd(&cmd->se_cmd, 0);
1248 break;
1249 case ISCSI_OP_REJECT:
1250 case ISCSI_OP_NOOP_OUT:
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001251 spin_lock_bh(&conn->cmd_lock);
1252 if (!list_empty(&cmd->i_conn_node))
Nicholas Bellingeraf737f62014-02-03 12:53:51 -08001253 list_del_init(&cmd->i_conn_node);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001254 spin_unlock_bh(&conn->cmd_lock);
1255
1256 /*
1257 * Handle special case for REJECT when iscsi_add_reject*() has
1258 * overwritten the original iscsi_opcode assignment, and the
1259 * associated cmd->se_cmd needs to be released.
1260 */
1261 if (cmd->se_cmd.se_tfo != NULL) {
Nicholas Bellingerfff98872013-06-26 02:31:42 -07001262 pr_debug("Calling transport_generic_free_cmd from"
1263 " isert_put_cmd for 0x%02x\n",
1264 cmd->iscsi_opcode);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001265 transport_generic_free_cmd(&cmd->se_cmd, 0);
1266 break;
1267 }
1268 /*
1269 * Fall-through
1270 */
1271 default:
1272 isert_release_cmd(cmd);
1273 break;
1274 }
1275}
1276
1277static void
1278isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev)
1279{
1280 if (tx_desc->dma_addr != 0) {
1281 pr_debug("Calling ib_dma_unmap_single for tx_desc->dma_addr\n");
1282 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr,
1283 ISER_HEADERS_LEN, DMA_TO_DEVICE);
1284 tx_desc->dma_addr = 0;
1285 }
1286}
1287
1288static void
1289isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd,
Nicholas Bellingerd3066062014-03-30 15:50:03 -07001290 struct ib_device *ib_dev, bool comp_err)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001291{
1292 if (isert_cmd->sense_buf_dma != 0) {
1293 pr_debug("Calling ib_dma_unmap_single for isert_cmd->sense_buf_dma\n");
1294 ib_dma_unmap_single(ib_dev, isert_cmd->sense_buf_dma,
1295 isert_cmd->sense_buf_len, DMA_TO_DEVICE);
1296 isert_cmd->sense_buf_dma = 0;
1297 }
1298
1299 isert_unmap_tx_desc(tx_desc, ib_dev);
Nicholas Bellingerd3066062014-03-30 15:50:03 -07001300 isert_put_cmd(isert_cmd, comp_err);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001301}
1302
1303static void
1304isert_completion_rdma_read(struct iser_tx_desc *tx_desc,
1305 struct isert_cmd *isert_cmd)
1306{
1307 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
1308 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
1309 struct se_cmd *se_cmd = &cmd->se_cmd;
1310 struct ib_device *ib_dev = isert_cmd->conn->conn_cm_id->device;
1311
1312 iscsit_stop_dataout_timer(cmd);
1313
1314 if (wr->sge) {
1315 pr_debug("isert_do_rdma_read_comp: Unmapping wr->sge from t_data_sg\n");
1316 ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, DMA_TO_DEVICE);
1317 wr->sge = NULL;
1318 }
1319
1320 if (isert_cmd->ib_sge) {
1321 pr_debug("isert_do_rdma_read_comp: Freeing isert_cmd->ib_sge\n");
1322 kfree(isert_cmd->ib_sge);
1323 isert_cmd->ib_sge = NULL;
1324 }
1325
1326 cmd->write_data_done = se_cmd->data_length;
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001327 wr->send_wr_num = 0;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001328
1329 pr_debug("isert_do_rdma_read_comp, calling target_execute_cmd\n");
1330 spin_lock_bh(&cmd->istate_lock);
1331 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1332 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1333 spin_unlock_bh(&cmd->istate_lock);
1334
1335 target_execute_cmd(se_cmd);
1336}
1337
1338static void
1339isert_do_control_comp(struct work_struct *work)
1340{
1341 struct isert_cmd *isert_cmd = container_of(work,
1342 struct isert_cmd, comp_work);
1343 struct isert_conn *isert_conn = isert_cmd->conn;
1344 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1345 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
1346
1347 switch (cmd->i_state) {
1348 case ISTATE_SEND_TASKMGTRSP:
1349 pr_debug("Calling iscsit_tmr_post_handler >>>>>>>>>>>>>>>>>\n");
1350
1351 atomic_dec(&isert_conn->post_send_buf_count);
1352 iscsit_tmr_post_handler(cmd, cmd->conn);
1353
1354 cmd->i_state = ISTATE_SENT_STATUS;
Nicholas Bellingerd3066062014-03-30 15:50:03 -07001355 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001356 break;
1357 case ISTATE_SEND_REJECT:
1358 pr_debug("Got isert_do_control_comp ISTATE_SEND_REJECT: >>>\n");
1359 atomic_dec(&isert_conn->post_send_buf_count);
1360
1361 cmd->i_state = ISTATE_SENT_STATUS;
Nicholas Bellingerd3066062014-03-30 15:50:03 -07001362 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false);
Nicholas Bellingerfff98872013-06-26 02:31:42 -07001363 break;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001364 case ISTATE_SEND_LOGOUTRSP:
1365 pr_debug("Calling iscsit_logout_post_handler >>>>>>>>>>>>>>\n");
1366 /*
1367 * Call atomic_dec(&isert_conn->post_send_buf_count)
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001368 * from isert_wait_conn()
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001369 */
1370 isert_conn->logout_posted = true;
1371 iscsit_logout_post_handler(cmd, cmd->conn);
1372 break;
1373 default:
1374 pr_err("Unknown do_control_comp i_state %d\n", cmd->i_state);
1375 dump_stack();
1376 break;
1377 }
1378}
1379
1380static void
1381isert_response_completion(struct iser_tx_desc *tx_desc,
1382 struct isert_cmd *isert_cmd,
1383 struct isert_conn *isert_conn,
1384 struct ib_device *ib_dev)
1385{
1386 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001387 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001388
1389 if (cmd->i_state == ISTATE_SEND_TASKMGTRSP ||
Nicholas Bellingerfff98872013-06-26 02:31:42 -07001390 cmd->i_state == ISTATE_SEND_LOGOUTRSP ||
1391 cmd->i_state == ISTATE_SEND_REJECT) {
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001392 isert_unmap_tx_desc(tx_desc, ib_dev);
1393
1394 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp);
1395 queue_work(isert_comp_wq, &isert_cmd->comp_work);
1396 return;
1397 }
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001398 atomic_sub(wr->send_wr_num + 1, &isert_conn->post_send_buf_count);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001399
1400 cmd->i_state = ISTATE_SENT_STATUS;
Nicholas Bellingerd3066062014-03-30 15:50:03 -07001401 isert_completion_put(tx_desc, isert_cmd, ib_dev, false);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001402}
1403
1404static void
1405isert_send_completion(struct iser_tx_desc *tx_desc,
1406 struct isert_conn *isert_conn)
1407{
1408 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1409 struct isert_cmd *isert_cmd = tx_desc->isert_cmd;
1410 struct isert_rdma_wr *wr;
1411
1412 if (!isert_cmd) {
1413 atomic_dec(&isert_conn->post_send_buf_count);
1414 isert_unmap_tx_desc(tx_desc, ib_dev);
1415 return;
1416 }
1417 wr = &isert_cmd->rdma_wr;
1418
1419 switch (wr->iser_ib_op) {
1420 case ISER_IB_RECV:
1421 pr_err("isert_send_completion: Got ISER_IB_RECV\n");
1422 dump_stack();
1423 break;
1424 case ISER_IB_SEND:
1425 pr_debug("isert_send_completion: Got ISER_IB_SEND\n");
1426 isert_response_completion(tx_desc, isert_cmd,
1427 isert_conn, ib_dev);
1428 break;
1429 case ISER_IB_RDMA_WRITE:
1430 pr_err("isert_send_completion: Got ISER_IB_RDMA_WRITE\n");
1431 dump_stack();
1432 break;
1433 case ISER_IB_RDMA_READ:
1434 pr_debug("isert_send_completion: Got ISER_IB_RDMA_READ:\n");
1435
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001436 atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001437 isert_completion_rdma_read(tx_desc, isert_cmd);
1438 break;
1439 default:
1440 pr_err("Unknown wr->iser_ib_op: 0x%02x\n", wr->iser_ib_op);
1441 dump_stack();
1442 break;
1443 }
1444}
1445
1446static void
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001447isert_cq_tx_comp_err(struct iser_tx_desc *tx_desc, struct isert_conn *isert_conn)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001448{
1449 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001450 struct isert_cmd *isert_cmd = tx_desc->isert_cmd;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001451
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001452 if (!isert_cmd)
1453 isert_unmap_tx_desc(tx_desc, ib_dev);
1454 else
Nicholas Bellingerd3066062014-03-30 15:50:03 -07001455 isert_completion_put(tx_desc, isert_cmd, ib_dev, true);
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001456}
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001457
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001458static void
1459isert_cq_rx_comp_err(struct isert_conn *isert_conn)
1460{
1461 struct iscsi_conn *conn = isert_conn->conn;
1462
1463 if (isert_conn->post_recv_buf_count)
1464 return;
1465
1466 if (conn->sess) {
1467 target_sess_cmd_list_set_waiting(conn->sess->se_sess);
1468 target_wait_for_sess_cmds(conn->sess->se_sess);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001469 }
1470
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001471 while (atomic_read(&isert_conn->post_send_buf_count))
1472 msleep(3000);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001473
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001474 mutex_lock(&isert_conn->conn_mutex);
1475 isert_conn->state = ISER_CONN_DOWN;
1476 mutex_unlock(&isert_conn->conn_mutex);
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -07001477
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001478 complete(&isert_conn->conn_wait_comp_err);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001479}
1480
1481static void
1482isert_cq_tx_work(struct work_struct *work)
1483{
1484 struct isert_cq_desc *cq_desc = container_of(work,
1485 struct isert_cq_desc, cq_tx_work);
1486 struct isert_device *device = cq_desc->device;
1487 int cq_index = cq_desc->cq_index;
1488 struct ib_cq *tx_cq = device->dev_tx_cq[cq_index];
1489 struct isert_conn *isert_conn;
1490 struct iser_tx_desc *tx_desc;
1491 struct ib_wc wc;
1492
1493 while (ib_poll_cq(tx_cq, 1, &wc) == 1) {
1494 tx_desc = (struct iser_tx_desc *)(unsigned long)wc.wr_id;
1495 isert_conn = wc.qp->qp_context;
1496
1497 if (wc.status == IB_WC_SUCCESS) {
1498 isert_send_completion(tx_desc, isert_conn);
1499 } else {
1500 pr_debug("TX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n");
1501 pr_debug("TX wc.status: 0x%08x\n", wc.status);
1502 atomic_dec(&isert_conn->post_send_buf_count);
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001503 isert_cq_tx_comp_err(tx_desc, isert_conn);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001504 }
1505 }
1506
1507 ib_req_notify_cq(tx_cq, IB_CQ_NEXT_COMP);
1508}
1509
1510static void
1511isert_cq_tx_callback(struct ib_cq *cq, void *context)
1512{
1513 struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context;
1514
1515 INIT_WORK(&cq_desc->cq_tx_work, isert_cq_tx_work);
1516 queue_work(isert_comp_wq, &cq_desc->cq_tx_work);
1517}
1518
1519static void
1520isert_cq_rx_work(struct work_struct *work)
1521{
1522 struct isert_cq_desc *cq_desc = container_of(work,
1523 struct isert_cq_desc, cq_rx_work);
1524 struct isert_device *device = cq_desc->device;
1525 int cq_index = cq_desc->cq_index;
1526 struct ib_cq *rx_cq = device->dev_rx_cq[cq_index];
1527 struct isert_conn *isert_conn;
1528 struct iser_rx_desc *rx_desc;
1529 struct ib_wc wc;
1530 unsigned long xfer_len;
1531
1532 while (ib_poll_cq(rx_cq, 1, &wc) == 1) {
1533 rx_desc = (struct iser_rx_desc *)(unsigned long)wc.wr_id;
1534 isert_conn = wc.qp->qp_context;
1535
1536 if (wc.status == IB_WC_SUCCESS) {
1537 xfer_len = (unsigned long)wc.byte_len;
1538 isert_rx_completion(rx_desc, isert_conn, xfer_len);
1539 } else {
1540 pr_debug("RX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n");
1541 if (wc.status != IB_WC_WR_FLUSH_ERR)
1542 pr_debug("RX wc.status: 0x%08x\n", wc.status);
1543
1544 isert_conn->post_recv_buf_count--;
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08001545 isert_cq_rx_comp_err(isert_conn);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001546 }
1547 }
1548
1549 ib_req_notify_cq(rx_cq, IB_CQ_NEXT_COMP);
1550}
1551
1552static void
1553isert_cq_rx_callback(struct ib_cq *cq, void *context)
1554{
1555 struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context;
1556
1557 INIT_WORK(&cq_desc->cq_rx_work, isert_cq_rx_work);
1558 queue_work(isert_rx_wq, &cq_desc->cq_rx_work);
1559}
1560
1561static int
1562isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd)
1563{
1564 struct ib_send_wr *wr_failed;
1565 int ret;
1566
1567 atomic_inc(&isert_conn->post_send_buf_count);
1568
1569 ret = ib_post_send(isert_conn->conn_qp, &isert_cmd->tx_desc.send_wr,
1570 &wr_failed);
1571 if (ret) {
1572 pr_err("ib_post_send failed with %d\n", ret);
1573 atomic_dec(&isert_conn->post_send_buf_count);
1574 return ret;
1575 }
1576 return ret;
1577}
1578
1579static int
1580isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
1581{
1582 struct isert_cmd *isert_cmd = container_of(cmd,
1583 struct isert_cmd, iscsi_cmd);
1584 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1585 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1586 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)
1587 &isert_cmd->tx_desc.iscsi_header;
1588
1589 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1590 iscsit_build_rsp_pdu(cmd, conn, true, hdr);
1591 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1592 /*
1593 * Attach SENSE DATA payload to iSCSI Response PDU
1594 */
1595 if (cmd->se_cmd.sense_buffer &&
1596 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
1597 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
1598 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1599 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
1600 u32 padding, sense_len;
1601
1602 put_unaligned_be16(cmd->se_cmd.scsi_sense_length,
1603 cmd->sense_buffer);
1604 cmd->se_cmd.scsi_sense_length += sizeof(__be16);
1605
1606 padding = -(cmd->se_cmd.scsi_sense_length) & 3;
1607 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length);
1608 sense_len = cmd->se_cmd.scsi_sense_length + padding;
1609
1610 isert_cmd->sense_buf_dma = ib_dma_map_single(ib_dev,
1611 (void *)cmd->sense_buffer, sense_len,
1612 DMA_TO_DEVICE);
1613
1614 isert_cmd->sense_buf_len = sense_len;
1615 tx_dsg->addr = isert_cmd->sense_buf_dma;
1616 tx_dsg->length = sense_len;
1617 tx_dsg->lkey = isert_conn->conn_mr->lkey;
1618 isert_cmd->tx_desc.num_sge = 2;
1619 }
1620
1621 isert_init_send_wr(isert_cmd, send_wr);
1622
1623 pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1624
1625 return isert_post_response(isert_conn, isert_cmd);
1626}
1627
1628static int
1629isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
1630 bool nopout_response)
1631{
1632 struct isert_cmd *isert_cmd = container_of(cmd,
1633 struct isert_cmd, iscsi_cmd);
1634 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1635 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1636
1637 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1638 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *)
1639 &isert_cmd->tx_desc.iscsi_header,
1640 nopout_response);
1641 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1642 isert_init_send_wr(isert_cmd, send_wr);
1643
1644 pr_debug("Posting NOPIN Reponse IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1645
1646 return isert_post_response(isert_conn, isert_cmd);
1647}
1648
1649static int
1650isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1651{
1652 struct isert_cmd *isert_cmd = container_of(cmd,
1653 struct isert_cmd, iscsi_cmd);
1654 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1655 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1656
1657 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1658 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *)
1659 &isert_cmd->tx_desc.iscsi_header);
1660 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1661 isert_init_send_wr(isert_cmd, send_wr);
1662
1663 pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1664
1665 return isert_post_response(isert_conn, isert_cmd);
1666}
1667
1668static int
1669isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1670{
1671 struct isert_cmd *isert_cmd = container_of(cmd,
1672 struct isert_cmd, iscsi_cmd);
1673 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1674 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1675
1676 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1677 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *)
1678 &isert_cmd->tx_desc.iscsi_header);
1679 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1680 isert_init_send_wr(isert_cmd, send_wr);
1681
1682 pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1683
1684 return isert_post_response(isert_conn, isert_cmd);
1685}
1686
1687static int
1688isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1689{
1690 struct isert_cmd *isert_cmd = container_of(cmd,
1691 struct isert_cmd, iscsi_cmd);
1692 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1693 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
Nicholas Bellingerfff98872013-06-26 02:31:42 -07001694 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1695 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
1696 struct iscsi_reject *hdr =
1697 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001698
1699 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
Nicholas Bellingerfff98872013-06-26 02:31:42 -07001700 iscsit_build_reject(cmd, conn, hdr);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001701 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
Nicholas Bellingerfff98872013-06-26 02:31:42 -07001702
1703 hton24(hdr->dlength, ISCSI_HDR_LEN);
1704 isert_cmd->sense_buf_dma = ib_dma_map_single(ib_dev,
1705 (void *)cmd->buf_ptr, ISCSI_HDR_LEN,
1706 DMA_TO_DEVICE);
1707 isert_cmd->sense_buf_len = ISCSI_HDR_LEN;
1708 tx_dsg->addr = isert_cmd->sense_buf_dma;
1709 tx_dsg->length = ISCSI_HDR_LEN;
1710 tx_dsg->lkey = isert_conn->conn_mr->lkey;
1711 isert_cmd->tx_desc.num_sge = 2;
1712
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001713 isert_init_send_wr(isert_cmd, send_wr);
1714
1715 pr_debug("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1716
1717 return isert_post_response(isert_conn, isert_cmd);
1718}
1719
1720static int
1721isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
1722 struct ib_sge *ib_sge, struct ib_send_wr *send_wr,
1723 u32 data_left, u32 offset)
1724{
1725 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd;
1726 struct scatterlist *sg_start, *tmp_sg;
1727 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1728 u32 sg_off, page_off;
1729 int i = 0, sg_nents;
1730
1731 sg_off = offset / PAGE_SIZE;
1732 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1733 sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge);
1734 page_off = offset % PAGE_SIZE;
1735
1736 send_wr->sg_list = ib_sge;
1737 send_wr->num_sge = sg_nents;
1738 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc;
1739 /*
1740 * Perform mapping of TCM scatterlist memory ib_sge dma_addr.
1741 */
1742 for_each_sg(sg_start, tmp_sg, sg_nents, i) {
1743 pr_debug("ISER RDMA from SGL dma_addr: 0x%16llx dma_len: %u, page_off: %u\n",
1744 (unsigned long long)tmp_sg->dma_address,
1745 tmp_sg->length, page_off);
1746
1747 ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off;
1748 ib_sge->length = min_t(u32, data_left,
1749 ib_sg_dma_len(ib_dev, tmp_sg) - page_off);
1750 ib_sge->lkey = isert_conn->conn_mr->lkey;
1751
1752 pr_debug("RDMA ib_sge: addr: 0x%16llx length: %u\n",
1753 ib_sge->addr, ib_sge->length);
1754 page_off = 0;
1755 data_left -= ib_sge->length;
1756 ib_sge++;
1757 pr_debug("Incrementing ib_sge pointer to %p\n", ib_sge);
1758 }
1759
1760 pr_debug("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n",
1761 send_wr->sg_list, send_wr->num_sge);
1762
1763 return sg_nents;
1764}
1765
1766static int
1767isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
1768{
1769 struct se_cmd *se_cmd = &cmd->se_cmd;
1770 struct isert_cmd *isert_cmd = container_of(cmd,
1771 struct isert_cmd, iscsi_cmd);
1772 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
1773 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1774 struct ib_send_wr *wr_failed, *send_wr;
1775 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1776 struct ib_sge *ib_sge;
1777 struct scatterlist *sg;
1778 u32 offset = 0, data_len, data_left, rdma_write_max;
1779 int rc, ret = 0, count, sg_nents, i, ib_sge_cnt;
1780
1781 pr_debug("RDMA_WRITE: data_length: %u\n", se_cmd->data_length);
1782
1783 sg = &se_cmd->t_data_sg[0];
1784 sg_nents = se_cmd->t_data_nents;
1785
1786 count = ib_dma_map_sg(ib_dev, sg, sg_nents, DMA_TO_DEVICE);
1787 if (unlikely(!count)) {
1788 pr_err("Unable to map put_datain SGs\n");
1789 return -EINVAL;
1790 }
1791 wr->sge = sg;
1792 wr->num_sge = sg_nents;
1793 pr_debug("Mapped IB count: %u sg: %p sg_nents: %u for RDMA_WRITE\n",
1794 count, sg, sg_nents);
1795
1796 ib_sge = kzalloc(sizeof(struct ib_sge) * sg_nents, GFP_KERNEL);
1797 if (!ib_sge) {
1798 pr_warn("Unable to allocate datain ib_sge\n");
1799 ret = -ENOMEM;
1800 goto unmap_sg;
1801 }
1802 isert_cmd->ib_sge = ib_sge;
1803
1804 pr_debug("Allocated ib_sge: %p from t_data_ents: %d for RDMA_WRITE\n",
1805 ib_sge, se_cmd->t_data_nents);
1806
1807 wr->send_wr_num = DIV_ROUND_UP(sg_nents, isert_conn->max_sge);
1808 wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num,
1809 GFP_KERNEL);
1810 if (!wr->send_wr) {
1811 pr_err("Unable to allocate wr->send_wr\n");
1812 ret = -ENOMEM;
1813 goto unmap_sg;
1814 }
1815 pr_debug("Allocated wr->send_wr: %p wr->send_wr_num: %u\n",
1816 wr->send_wr, wr->send_wr_num);
1817
1818 iscsit_increment_maxcmdsn(cmd, conn->sess);
1819 cmd->stat_sn = conn->stat_sn++;
1820
1821 wr->isert_cmd = isert_cmd;
1822 rdma_write_max = isert_conn->max_sge * PAGE_SIZE;
1823 data_left = se_cmd->data_length;
1824
1825 for (i = 0; i < wr->send_wr_num; i++) {
1826 send_wr = &isert_cmd->rdma_wr.send_wr[i];
1827 data_len = min(data_left, rdma_write_max);
1828
1829 send_wr->opcode = IB_WR_RDMA_WRITE;
1830 send_wr->send_flags = 0;
1831 send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset;
1832 send_wr->wr.rdma.rkey = isert_cmd->read_stag;
1833
1834 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge,
1835 send_wr, data_len, offset);
1836 ib_sge += ib_sge_cnt;
1837
1838 if (i + 1 == wr->send_wr_num)
1839 send_wr->next = &isert_cmd->tx_desc.send_wr;
1840 else
1841 send_wr->next = &wr->send_wr[i + 1];
1842
1843 offset += data_len;
1844 data_left -= data_len;
1845 }
1846 /*
1847 * Build isert_conn->tx_desc for iSCSI response PDU and attach
1848 */
1849 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1850 iscsit_build_rsp_pdu(cmd, conn, false, (struct iscsi_scsi_rsp *)
1851 &isert_cmd->tx_desc.iscsi_header);
1852 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1853 isert_init_send_wr(isert_cmd, &isert_cmd->tx_desc.send_wr);
1854
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001855 atomic_add(wr->send_wr_num + 1, &isert_conn->post_send_buf_count);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001856
1857 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed);
1858 if (rc) {
1859 pr_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n");
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001860 atomic_sub(wr->send_wr_num + 1, &isert_conn->post_send_buf_count);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001861 }
1862 pr_debug("Posted RDMA_WRITE + Response for iSER Data READ\n");
1863 return 1;
1864
1865unmap_sg:
1866 ib_dma_unmap_sg(ib_dev, sg, sg_nents, DMA_TO_DEVICE);
1867 return ret;
1868}
1869
1870static int
1871isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery)
1872{
1873 struct se_cmd *se_cmd = &cmd->se_cmd;
1874 struct isert_cmd *isert_cmd = container_of(cmd,
1875 struct isert_cmd, iscsi_cmd);
1876 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
1877 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
1878 struct ib_send_wr *wr_failed, *send_wr;
1879 struct ib_sge *ib_sge;
1880 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1881 struct scatterlist *sg_start;
1882 u32 sg_off, sg_nents, page_off, va_offset = 0;
1883 u32 offset = 0, data_len, data_left, rdma_write_max;
1884 int rc, ret = 0, count, i, ib_sge_cnt;
1885
1886 pr_debug("RDMA_READ: data_length: %u write_data_done: %u\n",
1887 se_cmd->data_length, cmd->write_data_done);
1888
1889 sg_off = cmd->write_data_done / PAGE_SIZE;
1890 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1891 page_off = cmd->write_data_done % PAGE_SIZE;
1892
1893 pr_debug("RDMA_READ: sg_off: %d, sg_start: %p page_off: %d\n",
1894 sg_off, sg_start, page_off);
1895
1896 data_left = se_cmd->data_length - cmd->write_data_done;
1897 sg_nents = se_cmd->t_data_nents - sg_off;
1898
1899 pr_debug("RDMA_READ: data_left: %d, sg_nents: %d\n",
1900 data_left, sg_nents);
1901
1902 count = ib_dma_map_sg(ib_dev, sg_start, sg_nents, DMA_FROM_DEVICE);
1903 if (unlikely(!count)) {
1904 pr_err("Unable to map get_dataout SGs\n");
1905 return -EINVAL;
1906 }
1907 wr->sge = sg_start;
1908 wr->num_sge = sg_nents;
1909 pr_debug("Mapped IB count: %u sg_start: %p sg_nents: %u for RDMA_READ\n",
1910 count, sg_start, sg_nents);
1911
1912 ib_sge = kzalloc(sizeof(struct ib_sge) * sg_nents, GFP_KERNEL);
1913 if (!ib_sge) {
1914 pr_warn("Unable to allocate dataout ib_sge\n");
1915 ret = -ENOMEM;
1916 goto unmap_sg;
1917 }
1918 isert_cmd->ib_sge = ib_sge;
1919
1920 pr_debug("Using ib_sge: %p from sg_ents: %d for RDMA_READ\n",
1921 ib_sge, sg_nents);
1922
1923 wr->send_wr_num = DIV_ROUND_UP(sg_nents, isert_conn->max_sge);
1924 wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num,
1925 GFP_KERNEL);
1926 if (!wr->send_wr) {
1927 pr_debug("Unable to allocate wr->send_wr\n");
1928 ret = -ENOMEM;
1929 goto unmap_sg;
1930 }
1931 pr_debug("Allocated wr->send_wr: %p wr->send_wr_num: %u\n",
1932 wr->send_wr, wr->send_wr_num);
1933
1934 isert_cmd->tx_desc.isert_cmd = isert_cmd;
1935
1936 wr->iser_ib_op = ISER_IB_RDMA_READ;
1937 wr->isert_cmd = isert_cmd;
1938 rdma_write_max = isert_conn->max_sge * PAGE_SIZE;
1939 offset = cmd->write_data_done;
1940
1941 for (i = 0; i < wr->send_wr_num; i++) {
1942 send_wr = &isert_cmd->rdma_wr.send_wr[i];
1943 data_len = min(data_left, rdma_write_max);
1944
1945 send_wr->opcode = IB_WR_RDMA_READ;
1946 send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset;
1947 send_wr->wr.rdma.rkey = isert_cmd->write_stag;
1948
1949 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge,
1950 send_wr, data_len, offset);
1951 ib_sge += ib_sge_cnt;
1952
1953 if (i + 1 == wr->send_wr_num)
1954 send_wr->send_flags = IB_SEND_SIGNALED;
1955 else
1956 send_wr->next = &wr->send_wr[i + 1];
1957
1958 offset += data_len;
1959 va_offset += data_len;
1960 data_left -= data_len;
1961 }
1962
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001963 atomic_add(wr->send_wr_num, &isert_conn->post_send_buf_count);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001964
1965 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed);
1966 if (rc) {
1967 pr_warn("ib_post_send() failed for IB_WR_RDMA_READ\n");
Nicholas Bellinger0bf95492014-02-27 09:05:03 -08001968 atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001969 }
1970 pr_debug("Posted RDMA_READ memory for ISER Data WRITE\n");
1971 return 0;
1972
1973unmap_sg:
1974 ib_dma_unmap_sg(ib_dev, sg_start, sg_nents, DMA_FROM_DEVICE);
1975 return ret;
1976}
1977
1978static int
1979isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
1980{
1981 int ret;
1982
1983 switch (state) {
1984 case ISTATE_SEND_NOPIN_WANT_RESPONSE:
1985 ret = isert_put_nopin(cmd, conn, false);
1986 break;
1987 default:
1988 pr_err("Unknown immediate state: 0x%02x\n", state);
1989 ret = -EINVAL;
1990 break;
1991 }
1992
1993 return ret;
1994}
1995
1996static int
1997isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
1998{
1999 int ret;
2000
2001 switch (state) {
2002 case ISTATE_SEND_LOGOUTRSP:
2003 ret = isert_put_logout_rsp(cmd, conn);
2004 if (!ret) {
2005 pr_debug("Returning iSER Logout -EAGAIN\n");
2006 ret = -EAGAIN;
2007 }
2008 break;
2009 case ISTATE_SEND_NOPIN:
2010 ret = isert_put_nopin(cmd, conn, true);
2011 break;
2012 case ISTATE_SEND_TASKMGTRSP:
2013 ret = isert_put_tm_rsp(cmd, conn);
2014 break;
2015 case ISTATE_SEND_REJECT:
2016 ret = isert_put_reject(cmd, conn);
2017 break;
2018 case ISTATE_SEND_STATUS:
2019 /*
2020 * Special case for sending non GOOD SCSI status from TX thread
2021 * context during pre se_cmd excecution failure.
2022 */
2023 ret = isert_put_response(conn, cmd);
2024 break;
2025 default:
2026 pr_err("Unknown response state: 0x%02x\n", state);
2027 ret = -EINVAL;
2028 break;
2029 }
2030
2031 return ret;
2032}
2033
2034static int
2035isert_setup_np(struct iscsi_np *np,
2036 struct __kernel_sockaddr_storage *ksockaddr)
2037{
2038 struct isert_np *isert_np;
2039 struct rdma_cm_id *isert_lid;
2040 struct sockaddr *sa;
2041 int ret;
2042
2043 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL);
2044 if (!isert_np) {
2045 pr_err("Unable to allocate struct isert_np\n");
2046 return -ENOMEM;
2047 }
Sagi Grimberg8a2629a2014-04-29 13:13:45 +03002048 sema_init(&isert_np->np_sem, 0);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002049 mutex_init(&isert_np->np_accept_mutex);
2050 INIT_LIST_HEAD(&isert_np->np_accept_list);
2051 init_completion(&isert_np->np_login_comp);
2052
2053 sa = (struct sockaddr *)ksockaddr;
2054 pr_debug("ksockaddr: %p, sa: %p\n", ksockaddr, sa);
2055 /*
2056 * Setup the np->np_sockaddr from the passed sockaddr setup
2057 * in iscsi_target_configfs.c code..
2058 */
2059 memcpy(&np->np_sockaddr, ksockaddr,
2060 sizeof(struct __kernel_sockaddr_storage));
2061
2062 isert_lid = rdma_create_id(isert_cma_handler, np, RDMA_PS_TCP,
2063 IB_QPT_RC);
2064 if (IS_ERR(isert_lid)) {
2065 pr_err("rdma_create_id() for isert_listen_handler failed: %ld\n",
2066 PTR_ERR(isert_lid));
2067 ret = PTR_ERR(isert_lid);
2068 goto out;
2069 }
2070
2071 ret = rdma_bind_addr(isert_lid, sa);
2072 if (ret) {
2073 pr_err("rdma_bind_addr() for isert_lid failed: %d\n", ret);
2074 goto out_lid;
2075 }
2076
2077 ret = rdma_listen(isert_lid, ISERT_RDMA_LISTEN_BACKLOG);
2078 if (ret) {
2079 pr_err("rdma_listen() for isert_lid failed: %d\n", ret);
2080 goto out_lid;
2081 }
2082
2083 isert_np->np_cm_id = isert_lid;
2084 np->np_context = isert_np;
2085 pr_debug("Setup isert_lid->context: %p\n", isert_lid->context);
2086
2087 return 0;
2088
2089out_lid:
2090 rdma_destroy_id(isert_lid);
2091out:
2092 kfree(isert_np);
2093 return ret;
2094}
2095
2096static int
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002097isert_rdma_accept(struct isert_conn *isert_conn)
2098{
2099 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id;
2100 struct rdma_conn_param cp;
2101 int ret;
2102
2103 memset(&cp, 0, sizeof(struct rdma_conn_param));
2104 cp.responder_resources = isert_conn->responder_resources;
2105 cp.initiator_depth = isert_conn->initiator_depth;
2106 cp.retry_count = 7;
2107 cp.rnr_retry_count = 7;
2108
2109 pr_debug("Before rdma_accept >>>>>>>>>>>>>>>>>>>>.\n");
2110
2111 ret = rdma_accept(cm_id, &cp);
2112 if (ret) {
2113 pr_err("rdma_accept() failed with: %d\n", ret);
2114 return ret;
2115 }
2116
2117 pr_debug("After rdma_accept >>>>>>>>>>>>>>>>>>>>>.\n");
2118
2119 return 0;
2120}
2121
2122static int
2123isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
2124{
2125 struct isert_conn *isert_conn = (struct isert_conn *)conn->context;
2126 int ret;
2127
2128 pr_debug("isert_get_login_rx before conn_login_comp conn: %p\n", conn);
2129
2130 ret = wait_for_completion_interruptible(&isert_conn->conn_login_comp);
2131 if (ret)
2132 return ret;
2133
2134 pr_debug("isert_get_login_rx processing login->req: %p\n", login->req);
2135 return 0;
2136}
2137
2138static void
2139isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn,
2140 struct isert_conn *isert_conn)
2141{
2142 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id;
2143 struct rdma_route *cm_route = &cm_id->route;
2144 struct sockaddr_in *sock_in;
2145 struct sockaddr_in6 *sock_in6;
2146
2147 conn->login_family = np->np_sockaddr.ss_family;
2148
2149 if (np->np_sockaddr.ss_family == AF_INET6) {
2150 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr;
2151 snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c",
2152 &sock_in6->sin6_addr.in6_u);
2153 conn->login_port = ntohs(sock_in6->sin6_port);
2154
2155 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr;
2156 snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c",
2157 &sock_in6->sin6_addr.in6_u);
2158 conn->local_port = ntohs(sock_in6->sin6_port);
2159 } else {
2160 sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr;
2161 sprintf(conn->login_ip, "%pI4",
2162 &sock_in->sin_addr.s_addr);
2163 conn->login_port = ntohs(sock_in->sin_port);
2164
2165 sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr;
2166 sprintf(conn->local_ip, "%pI4",
2167 &sock_in->sin_addr.s_addr);
2168 conn->local_port = ntohs(sock_in->sin_port);
2169 }
2170}
2171
2172static int
2173isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn)
2174{
2175 struct isert_np *isert_np = (struct isert_np *)np->np_context;
2176 struct isert_conn *isert_conn;
2177 int max_accept = 0, ret;
2178
2179accept_wait:
Sagi Grimberg8a2629a2014-04-29 13:13:45 +03002180 ret = down_interruptible(&isert_np->np_sem);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002181 if (max_accept > 5)
2182 return -ENODEV;
2183
2184 spin_lock_bh(&np->np_thread_lock);
2185 if (np->np_thread_state == ISCSI_NP_THREAD_RESET) {
2186 spin_unlock_bh(&np->np_thread_lock);
Sagi Grimberg8a2629a2014-04-29 13:13:45 +03002187 pr_debug("ISCSI_NP_THREAD_RESET for isert_accept_np\n");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002188 return -ENODEV;
2189 }
2190 spin_unlock_bh(&np->np_thread_lock);
2191
2192 mutex_lock(&isert_np->np_accept_mutex);
2193 if (list_empty(&isert_np->np_accept_list)) {
2194 mutex_unlock(&isert_np->np_accept_mutex);
2195 max_accept++;
2196 goto accept_wait;
2197 }
2198 isert_conn = list_first_entry(&isert_np->np_accept_list,
2199 struct isert_conn, conn_accept_node);
2200 list_del_init(&isert_conn->conn_accept_node);
2201 mutex_unlock(&isert_np->np_accept_mutex);
2202
2203 conn->context = isert_conn;
2204 isert_conn->conn = conn;
2205 max_accept = 0;
2206
2207 ret = isert_rdma_post_recvl(isert_conn);
2208 if (ret)
2209 return ret;
2210
2211 ret = isert_rdma_accept(isert_conn);
2212 if (ret)
2213 return ret;
2214
2215 isert_set_conn_info(np, conn, isert_conn);
2216
2217 pr_debug("Processing isert_accept_np: isert_conn: %p\n", isert_conn);
2218 return 0;
2219}
2220
2221static void
2222isert_free_np(struct iscsi_np *np)
2223{
2224 struct isert_np *isert_np = (struct isert_np *)np->np_context;
2225
2226 rdma_destroy_id(isert_np->np_cm_id);
2227
2228 np->np_context = NULL;
2229 kfree(isert_np);
2230}
2231
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08002232static void isert_wait_conn(struct iscsi_conn *conn)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002233{
2234 struct isert_conn *isert_conn = conn->context;
2235
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08002236 pr_debug("isert_wait_conn: Starting \n");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002237 /*
2238 * Decrement post_send_buf_count for special case when called
2239 * from isert_do_control_comp() -> iscsit_logout_post_handler()
2240 */
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -07002241 mutex_lock(&isert_conn->conn_mutex);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002242 if (isert_conn->logout_posted)
2243 atomic_dec(&isert_conn->post_send_buf_count);
2244
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -07002245 if (isert_conn->conn_cm_id && isert_conn->state != ISER_CONN_DOWN) {
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08002246 pr_debug("Calling rdma_disconnect from isert_wait_conn\n");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002247 rdma_disconnect(isert_conn->conn_cm_id);
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -07002248 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002249 /*
2250 * Only wait for conn_wait_comp_err if the isert_conn made it
2251 * into full feature phase..
2252 */
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -07002253 if (isert_conn->state == ISER_CONN_INIT) {
2254 mutex_unlock(&isert_conn->conn_mutex);
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -07002255 return;
2256 }
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08002257 if (isert_conn->state == ISER_CONN_UP)
2258 isert_conn->state = ISER_CONN_TERMINATING;
Nicholas Bellingerd9e507c2013-07-03 03:05:37 -07002259 mutex_unlock(&isert_conn->conn_mutex);
2260
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08002261 wait_for_completion(&isert_conn->conn_wait_comp_err);
2262
2263 wait_for_completion(&isert_conn->conn_wait);
2264}
2265
2266static void isert_free_conn(struct iscsi_conn *conn)
2267{
2268 struct isert_conn *isert_conn = conn->context;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002269
2270 isert_put_conn(isert_conn);
2271}
2272
2273static struct iscsit_transport iser_target_transport = {
2274 .name = "IB/iSER",
2275 .transport_type = ISCSI_INFINIBAND,
2276 .owner = THIS_MODULE,
2277 .iscsit_setup_np = isert_setup_np,
2278 .iscsit_accept_np = isert_accept_np,
2279 .iscsit_free_np = isert_free_np,
Nicholas Bellingerd8bd97a02014-02-03 12:54:39 -08002280 .iscsit_wait_conn = isert_wait_conn,
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002281 .iscsit_free_conn = isert_free_conn,
2282 .iscsit_alloc_cmd = isert_alloc_cmd,
2283 .iscsit_get_login_rx = isert_get_login_rx,
2284 .iscsit_put_login_tx = isert_put_login_tx,
2285 .iscsit_immediate_queue = isert_immediate_queue,
2286 .iscsit_response_queue = isert_response_queue,
2287 .iscsit_get_dataout = isert_get_dataout,
2288 .iscsit_queue_data_in = isert_put_datain,
2289 .iscsit_queue_status = isert_put_response,
2290};
2291
2292static int __init isert_init(void)
2293{
2294 int ret;
2295
2296 isert_rx_wq = alloc_workqueue("isert_rx_wq", 0, 0);
2297 if (!isert_rx_wq) {
2298 pr_err("Unable to allocate isert_rx_wq\n");
2299 return -ENOMEM;
2300 }
2301
2302 isert_comp_wq = alloc_workqueue("isert_comp_wq", 0, 0);
2303 if (!isert_comp_wq) {
2304 pr_err("Unable to allocate isert_comp_wq\n");
2305 ret = -ENOMEM;
2306 goto destroy_rx_wq;
2307 }
2308
2309 isert_cmd_cache = kmem_cache_create("isert_cmd_cache",
2310 sizeof(struct isert_cmd), __alignof__(struct isert_cmd),
2311 0, NULL);
2312 if (!isert_cmd_cache) {
2313 pr_err("Unable to create isert_cmd_cache\n");
2314 ret = -ENOMEM;
2315 goto destroy_tx_cq;
2316 }
2317
2318 iscsit_register_transport(&iser_target_transport);
2319 pr_debug("iSER_TARGET[0] - Loaded iser_target_transport\n");
2320 return 0;
2321
2322destroy_tx_cq:
2323 destroy_workqueue(isert_comp_wq);
2324destroy_rx_wq:
2325 destroy_workqueue(isert_rx_wq);
2326 return ret;
2327}
2328
2329static void __exit isert_exit(void)
2330{
2331 kmem_cache_destroy(isert_cmd_cache);
2332 destroy_workqueue(isert_comp_wq);
2333 destroy_workqueue(isert_rx_wq);
2334 iscsit_unregister_transport(&iser_target_transport);
2335 pr_debug("iSER_TARGET[0] - Released iser_target_transport\n");
2336}
2337
2338MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure");
2339MODULE_VERSION("0.1");
2340MODULE_AUTHOR("nab@Linux-iSCSI.org");
2341MODULE_LICENSE("GPL");
2342
2343module_init(isert_init);
2344module_exit(isert_exit);