aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/infiniband/hw/qib/qib.h3
-rw-r--r--drivers/infiniband/hw/qib/qib_driver.c9
-rw-r--r--drivers/infiniband/hw/qib/qib_qp.c77
-rw-r--r--drivers/infiniband/hw/qib/qib_verbs.c36
-rw-r--r--drivers/infiniband/hw/qib/qib_verbs.h3
5 files changed, 87 insertions, 41 deletions
diff --git a/drivers/infiniband/hw/qib/qib.h b/drivers/infiniband/hw/qib/qib.h
index 97e623383e1a..b881bdc401f5 100644
--- a/drivers/infiniband/hw/qib/qib.h
+++ b/drivers/infiniband/hw/qib/qib.h
@@ -223,6 +223,9 @@ struct qib_ctxtdata {
/* ctxt rcvhdrq head offset */
u32 head;
u32 pkt_count;
+ /* lookaside fields */
+ struct qib_qp *lookaside_qp;
+ u32 lookaside_qpn;
/* QPs waiting for context processing */
struct list_head qp_wait_list;
};
diff --git a/drivers/infiniband/hw/qib/qib_driver.c b/drivers/infiniband/hw/qib/qib_driver.c
index 89264ffc7ee9..d35c9d38ceee 100644
--- a/drivers/infiniband/hw/qib/qib_driver.c
+++ b/drivers/infiniband/hw/qib/qib_driver.c
@@ -547,6 +547,15 @@ move_along:
updegr = 0;
}
}
+ /*
+ * Notify qib_destroy_qp() if it is waiting
+ * for lookaside_qp to finish.
+ */
+ if (rcd->lookaside_qp) {
+ if (atomic_dec_and_test(&rcd->lookaside_qp->refcount))
+ wake_up(&rcd->lookaside_qp->wait);
+ rcd->lookaside_qp = NULL;
+ }
rcd->head = l;
rcd->pkt_count += i;
diff --git a/drivers/infiniband/hw/qib/qib_qp.c b/drivers/infiniband/hw/qib/qib_qp.c
index 9d094f910360..500981bce9c0 100644
--- a/drivers/infiniband/hw/qib/qib_qp.c
+++ b/drivers/infiniband/hw/qib/qib_qp.c
@@ -34,6 +34,7 @@
#include <linux/err.h>
#include <linux/vmalloc.h>
+#include <linux/jhash.h>
#include "qib.h"
@@ -204,6 +205,13 @@ static void free_qpn(struct qib_qpn_table *qpt, u32 qpn)
clear_bit(qpn & BITS_PER_PAGE_MASK, map->page);
}
+static inline unsigned qpn_hash(struct qib_ibdev *dev, u32 qpn)
+{
+ return jhash_1word(qpn, dev->qp_rnd) &
+ (dev->qp_table_size - 1);
+}
+
+
/*
* Put the QP into the hash table.
* The hash table holds a reference to the QP.
@@ -211,22 +219,23 @@ static void free_qpn(struct qib_qpn_table *qpt, u32 qpn)
static void insert_qp(struct qib_ibdev *dev, struct qib_qp *qp)
{
struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
- unsigned n = qp->ibqp.qp_num % dev->qp_table_size;
unsigned long flags;
+ unsigned n = qpn_hash(dev, qp->ibqp.qp_num);
spin_lock_irqsave(&dev->qpt_lock, flags);
+ atomic_inc(&qp->refcount);
if (qp->ibqp.qp_num == 0)
- ibp->qp0 = qp;
+ rcu_assign_pointer(ibp->qp0, qp);
else if (qp->ibqp.qp_num == 1)
- ibp->qp1 = qp;
+ rcu_assign_pointer(ibp->qp1, qp);
else {
qp->next = dev->qp_table[n];
- dev->qp_table[n] = qp;
+ rcu_assign_pointer(dev->qp_table[n], qp);
}
- atomic_inc(&qp->refcount);
spin_unlock_irqrestore(&dev->qpt_lock, flags);
+ synchronize_rcu();
}
/*
@@ -236,29 +245,32 @@ static void insert_qp(struct qib_ibdev *dev, struct qib_qp *qp)
static void remove_qp(struct qib_ibdev *dev, struct qib_qp *qp)
{
struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
- struct qib_qp *q, **qpp;
+ unsigned n = qpn_hash(dev, qp->ibqp.qp_num);
unsigned long flags;
- qpp = &dev->qp_table[qp->ibqp.qp_num % dev->qp_table_size];
-
spin_lock_irqsave(&dev->qpt_lock, flags);
if (ibp->qp0 == qp) {
- ibp->qp0 = NULL;
atomic_dec(&qp->refcount);
+ rcu_assign_pointer(ibp->qp0, NULL);
} else if (ibp->qp1 == qp) {
- ibp->qp1 = NULL;
atomic_dec(&qp->refcount);
- } else
+ rcu_assign_pointer(ibp->qp1, NULL);
+ } else {
+ struct qib_qp *q, **qpp;
+
+ qpp = &dev->qp_table[n];
for (; (q = *qpp) != NULL; qpp = &q->next)
if (q == qp) {
- *qpp = qp->next;
- qp->next = NULL;
atomic_dec(&qp->refcount);
+ rcu_assign_pointer(*qpp, qp->next);
+ qp->next = NULL;
break;
}
+ }
spin_unlock_irqrestore(&dev->qpt_lock, flags);
+ synchronize_rcu();
}
/**
@@ -280,21 +292,24 @@ unsigned qib_free_all_qps(struct qib_devdata *dd)
if (!qib_mcast_tree_empty(ibp))
qp_inuse++;
- if (ibp->qp0)
+ rcu_read_lock();
+ if (rcu_dereference(ibp->qp0))
qp_inuse++;
- if (ibp->qp1)
+ if (rcu_dereference(ibp->qp1))
qp_inuse++;
+ rcu_read_unlock();
}
spin_lock_irqsave(&dev->qpt_lock, flags);
for (n = 0; n < dev->qp_table_size; n++) {
qp = dev->qp_table[n];
- dev->qp_table[n] = NULL;
+ rcu_assign_pointer(dev->qp_table[n], NULL);
for (; qp; qp = qp->next)
qp_inuse++;
}
spin_unlock_irqrestore(&dev->qpt_lock, flags);
+ synchronize_rcu();
return qp_inuse;
}
@@ -309,25 +324,28 @@ unsigned qib_free_all_qps(struct qib_devdata *dd)
*/
struct qib_qp *qib_lookup_qpn(struct qib_ibport *ibp, u32 qpn)
{
- struct qib_ibdev *dev = &ppd_from_ibp(ibp)->dd->verbs_dev;
- unsigned long flags;
- struct qib_qp *qp;
+ struct qib_qp *qp = NULL;
- spin_lock_irqsave(&dev->qpt_lock, flags);
+ if (unlikely(qpn <= 1)) {
+ rcu_read_lock();
+ if (qpn == 0)
+ qp = rcu_dereference(ibp->qp0);
+ else
+ qp = rcu_dereference(ibp->qp1);
+ } else {
+ struct qib_ibdev *dev = &ppd_from_ibp(ibp)->dd->verbs_dev;
+ unsigned n = qpn_hash(dev, qpn);
- if (qpn == 0)
- qp = ibp->qp0;
- else if (qpn == 1)
- qp = ibp->qp1;
- else
- for (qp = dev->qp_table[qpn % dev->qp_table_size]; qp;
- qp = qp->next)
+ rcu_read_lock();
+ for (qp = dev->qp_table[n]; rcu_dereference(qp); qp = qp->next)
if (qp->ibqp.qp_num == qpn)
break;
+ }
if (qp)
- atomic_inc(&qp->refcount);
+ if (unlikely(!atomic_inc_not_zero(&qp->refcount)))
+ qp = NULL;
- spin_unlock_irqrestore(&dev->qpt_lock, flags);
+ rcu_read_unlock();
return qp;
}
@@ -1015,6 +1033,7 @@ struct ib_qp *qib_create_qp(struct ib_pd *ibpd,
ret = ERR_PTR(-ENOMEM);
goto bail_swq;
}
+ RCU_INIT_POINTER(qp->next, NULL);
if (init_attr->srq)
sz = 0;
else {
diff --git a/drivers/infiniband/hw/qib/qib_verbs.c b/drivers/infiniband/hw/qib/qib_verbs.c
index 9fab40488850..9627cb737125 100644
--- a/drivers/infiniband/hw/qib/qib_verbs.c
+++ b/drivers/infiniband/hw/qib/qib_verbs.c
@@ -38,11 +38,12 @@
#include <linux/utsname.h>
#include <linux/rculist.h>
#include <linux/mm.h>
+#include <linux/random.h>
#include "qib.h"
#include "qib_common.h"
-static unsigned int ib_qib_qp_table_size = 251;
+static unsigned int ib_qib_qp_table_size = 256;
module_param_named(qp_table_size, ib_qib_qp_table_size, uint, S_IRUGO);
MODULE_PARM_DESC(qp_table_size, "QP table size");
@@ -659,17 +660,25 @@ void qib_ib_rcv(struct qib_ctxtdata *rcd, void *rhdr, void *data, u32 tlen)
if (atomic_dec_return(&mcast->refcount) <= 1)
wake_up(&mcast->wait);
} else {
- qp = qib_lookup_qpn(ibp, qp_num);
- if (!qp)
- goto drop;
+ if (rcd->lookaside_qp) {
+ if (rcd->lookaside_qpn != qp_num) {
+ if (atomic_dec_and_test(
+ &rcd->lookaside_qp->refcount))
+ wake_up(
+ &rcd->lookaside_qp->wait);
+ rcd->lookaside_qp = NULL;
+ }
+ }
+ if (!rcd->lookaside_qp) {
+ qp = qib_lookup_qpn(ibp, qp_num);
+ if (!qp)
+ goto drop;
+ rcd->lookaside_qp = qp;
+ rcd->lookaside_qpn = qp_num;
+ } else
+ qp = rcd->lookaside_qp;
ibp->n_unicast_rcv++;
qib_qp_rcv(rcd, hdr, lnh == QIB_LRH_GRH, data, tlen, qp);
- /*
- * Notify qib_destroy_qp() if it is waiting
- * for us to finish.
- */
- if (atomic_dec_and_test(&qp->refcount))
- wake_up(&qp->wait);
}
return;
@@ -1974,6 +1983,8 @@ static void init_ibport(struct qib_pportdata *ppd)
ibp->z_excessive_buffer_overrun_errors =
cntrs.excessive_buffer_overrun_errors;
ibp->z_vl15_dropped = cntrs.vl15_dropped;
+ RCU_INIT_POINTER(ibp->qp0, NULL);
+ RCU_INIT_POINTER(ibp->qp1, NULL);
}
/**
@@ -1990,12 +2001,15 @@ int qib_register_ib_device(struct qib_devdata *dd)
int ret;
dev->qp_table_size = ib_qib_qp_table_size;
- dev->qp_table = kzalloc(dev->qp_table_size * sizeof *dev->qp_table,
+ get_random_bytes(&dev->qp_rnd, sizeof(dev->qp_rnd));
+ dev->qp_table = kmalloc(dev->qp_table_size * sizeof *dev->qp_table,
GFP_KERNEL);
if (!dev->qp_table) {
ret = -ENOMEM;
goto err_qpt;
}
+ for (i = 0; i < dev->qp_table_size; i++)
+ RCU_INIT_POINTER(dev->qp_table[i], NULL);
for (i = 0; i < dd->num_pports; i++)
init_ibport(ppd + i);
diff --git a/drivers/infiniband/hw/qib/qib_verbs.h b/drivers/infiniband/hw/qib/qib_verbs.h
index ec3711f743c5..d7b6109528a4 100644
--- a/drivers/infiniband/hw/qib/qib_verbs.h
+++ b/drivers/infiniband/hw/qib/qib_verbs.h
@@ -724,7 +724,8 @@ struct qib_ibdev {
dma_addr_t pio_hdrs_phys;
/* list of QPs waiting for RNR timer */
spinlock_t pending_lock; /* protect wait lists, PMA counters, etc. */
- unsigned qp_table_size; /* size of the hash table */
+ u32 qp_table_size; /* size of the hash table */
+ u32 qp_rnd; /* random bytes for hash */
spinlock_t qpt_lock;
u32 n_piowait;
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-14 02:58:40 +0000