From fe174357eb2deb184c93269846c92adf5743115b Mon Sep 17 00:00:00 2001 From: Roland Dreier Date: Tue, 12 Feb 2008 14:38:22 -0800 Subject: [PATCH 01/10] IB/mthca: Add missing sg_init_table() in mthca_map_user_db() Usually harmless, since the scatterlist is always hard-coded to a length of 1, but it triggers a BUG() if CONFIG_DEBUG_SG=y, so we better fix it. This fixes . Signed-off-by: Roland Dreier --- drivers/infiniband/hw/mthca/mthca_memfree.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/infiniband/hw/mthca/mthca_memfree.c b/drivers/infiniband/hw/mthca/mthca_memfree.c index 1f4d27d7c16d..252db0822f6c 100644 --- a/drivers/infiniband/hw/mthca/mthca_memfree.c +++ b/drivers/infiniband/hw/mthca/mthca_memfree.c @@ -542,6 +542,7 @@ struct mthca_user_db_table *mthca_init_user_db_tab(struct mthca_dev *dev) for (i = 0; i < npages; ++i) { db_tab->page[i].refcount = 0; db_tab->page[i].uvirt = 0; + sg_init_table(&db_tab->page[i].mem, 1); } return db_tab; From ab64b960673250518e748f8b4f1545447136b68b Mon Sep 17 00:00:00 2001 From: Roland Dreier Date: Tue, 12 Feb 2008 14:38:27 -0800 Subject: [PATCH 02/10] IB/cm: Remove debug printk()s that snuck upstream Pesky little devils, sneaking around... Signed-off-by: Roland Dreier --- drivers/infiniband/core/cm.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index 638b727d42e0..435e2767a183 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -3587,8 +3587,6 @@ static void cm_release_port_obj(struct kobject *obj) { struct cm_port *cm_port; - printk(KERN_ERR "free cm port\n"); - cm_port = container_of(obj, struct cm_port, port_obj); kfree(cm_port); } @@ -3601,8 +3599,6 @@ static void cm_release_dev_obj(struct kobject *obj) { struct cm_device *cm_dev; - printk(KERN_ERR "free cm dev\n"); - cm_dev = container_of(obj, struct cm_device, dev_obj); kfree(cm_dev); } From 7c7a9bccd2ba9f17e4b588461f140578a0a7b073 Mon Sep 17 00:00:00 2001 From: Roland Dreier Date: Tue, 12 Feb 2008 14:38:27 -0800 Subject: [PATCH 03/10] IB/cm: Fix infiniband_cm class kobject ref counting Commit 9af57b7a ("IB/cm: Add basic performance counters") introduced a bug in how the reference count for cm_class.subsys.kobj was handled: the path that released a device did a kobject_put() on that kobject, but there was no kobject_get() in the path the handles adding a device. So the reference count ended up too low, which leads to bad things. Fix up and simplify the reference counting to avoid this. (Actually, I introduced the bug when fixing the patch up to match some of Greg's kobject changes, but who's counting) Signed-off-by: Roland Dreier --- drivers/infiniband/core/cm.c | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index 435e2767a183..b10ade92efed 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -3612,18 +3612,12 @@ struct class cm_class = { }; EXPORT_SYMBOL(cm_class); -static void cm_remove_fs_obj(struct kobject *obj) -{ - kobject_put(obj->parent); - kobject_put(obj); -} - static int cm_create_port_fs(struct cm_port *port) { int i, ret; ret = kobject_init_and_add(&port->port_obj, &cm_port_obj_type, - kobject_get(&port->cm_dev->dev_obj), + &port->cm_dev->dev_obj, "%d", port->port_num); if (ret) { kfree(port); @@ -3633,7 +3627,7 @@ static int cm_create_port_fs(struct cm_port *port) for (i = 0; i < CM_COUNTER_GROUPS; i++) { ret = kobject_init_and_add(&port->counter_group[i].obj, &cm_counter_obj_type, - kobject_get(&port->port_obj), + &port->port_obj, "%s", counter_group_names[i]); if (ret) goto error; @@ -3643,8 +3637,8 @@ static int cm_create_port_fs(struct cm_port *port) error: while (i--) - cm_remove_fs_obj(&port->counter_group[i].obj); - cm_remove_fs_obj(&port->port_obj); + kobject_put(&port->counter_group[i].obj); + kobject_put(&port->port_obj); return ret; } @@ -3654,9 +3648,9 @@ static void cm_remove_port_fs(struct cm_port *port) int i; for (i = 0; i < CM_COUNTER_GROUPS; i++) - cm_remove_fs_obj(&port->counter_group[i].obj); + kobject_put(&port->counter_group[i].obj); - cm_remove_fs_obj(&port->port_obj); + kobject_put(&port->port_obj); } static void cm_add_one(struct ib_device *device) @@ -3740,7 +3734,7 @@ error1: ib_unregister_mad_agent(port->mad_agent); cm_remove_port_fs(port); } - cm_remove_fs_obj(&cm_dev->dev_obj); + kobject_put(&cm_dev->dev_obj); } static void cm_remove_one(struct ib_device *device) @@ -3767,7 +3761,7 @@ static void cm_remove_one(struct ib_device *device) ib_unregister_mad_agent(port->mad_agent); cm_remove_port_fs(port); } - cm_remove_fs_obj(&cm_dev->dev_obj); + kobject_put(&cm_dev->dev_obj); } static int __init ib_cm_init(void) From 8704e9a8790cc9e394198663c1c9150c899fb9a2 Mon Sep 17 00:00:00 2001 From: Steve Wise Date: Tue, 12 Feb 2008 16:09:29 -0600 Subject: [PATCH 04/10] RDMA/cxgb3: Fail loopback connections The cxgb3 HW and driver don't support loopback RDMA connections. So fail any connection attempt where the destination address is local. Signed-off-by: Steve Wise Signed-off-by: Roland Dreier --- drivers/infiniband/hw/cxgb3/iwch_cm.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/drivers/infiniband/hw/cxgb3/iwch_cm.c b/drivers/infiniband/hw/cxgb3/iwch_cm.c index e9a08fa3dffe..320f2b6ddee6 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_cm.c +++ b/drivers/infiniband/hw/cxgb3/iwch_cm.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include @@ -1784,6 +1785,17 @@ err: return err; } +static int is_loopback_dst(struct iw_cm_id *cm_id) +{ + struct net_device *dev; + + dev = ip_dev_find(&init_net, cm_id->remote_addr.sin_addr.s_addr); + if (!dev) + return 0; + dev_put(dev); + return 1; +} + int iwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) { int err = 0; @@ -1791,6 +1803,11 @@ int iwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) struct iwch_ep *ep; struct rtable *rt; + if (is_loopback_dst(cm_id)) { + err = -ENOSYS; + goto out; + } + ep = alloc_ep(sizeof(*ep), GFP_KERNEL); if (!ep) { printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __FUNCTION__); From 5163dc1a645bc9ed7984fa484f1a77378c166d23 Mon Sep 17 00:00:00 2001 From: Marcin Slusarz Date: Wed, 13 Feb 2008 00:06:08 +0100 Subject: [PATCH 05/10] IB/mthca: Convert to use be16_add_cpu() replace: big_endian_variable = cpu_to_beX(beX_to_cpu(big_endian_variable) + expression_in_cpu_byteorder); with: beX_add_cpu(&big_endian_variable, expression_in_cpu_byteorder); Generated with a semantic patch. Signed-off-by: Marcin Slusarz Cc: Sean Hefty Cc: Hal Rosenstock Signed-off-by: Roland Dreier --- drivers/infiniband/hw/mthca/mthca_cq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mthca/mthca_cq.c b/drivers/infiniband/hw/mthca/mthca_cq.c index 6bd9f1393349..1e1e336d3ef9 100644 --- a/drivers/infiniband/hw/mthca/mthca_cq.c +++ b/drivers/infiniband/hw/mthca/mthca_cq.c @@ -473,7 +473,7 @@ static void handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq, if (!(new_wqe & cpu_to_be32(0x3f)) || (!cqe->db_cnt && dbd)) return; - cqe->db_cnt = cpu_to_be16(be16_to_cpu(cqe->db_cnt) - dbd); + be16_add_cpu(&cqe->db_cnt, -dbd); cqe->wqe = new_wqe; cqe->syndrome = SYNDROME_WR_FLUSH_ERR; From 167c42655cca188657aa9bb4e06d1194af3c73a5 Mon Sep 17 00:00:00 2001 From: Jack Morgenstein Date: Wed, 13 Feb 2008 16:23:50 +0200 Subject: [PATCH 06/10] IPoIB: On P_Key change event, reset state properly In P_Key event handling, if the old P_Key is no longer available, the driver must call ipoib_ib_dev_stop() -- just as it does when the P_Key is still available (see procedure __ipoib_ib_dev_flush()). When a P_Key becomes available, the driver will perform ipoib_open(), which assumes that the QP is in RESET, the cm_id has been destroyed/deleted, etc. If ipoib_ib_dev_stop() is not called as described above, then these assumptions will be false, and the attempt to bring the interface up will fail. Found by Mellanox QA. Signed-off-by: Jack Morgenstein Signed-off-by: Roland Dreier --- drivers/infiniband/ulp/ipoib/ipoib_ib.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ib.c b/drivers/infiniband/ulp/ipoib/ipoib_ib.c index 9d3e778dc56d..08c4396cf418 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_ib.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_ib.c @@ -780,6 +780,7 @@ static void __ipoib_ib_dev_flush(struct ipoib_dev_priv *priv, int pkey_event) if (ib_find_pkey(priv->ca, priv->port, priv->pkey, &new_index)) { clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags); ipoib_ib_dev_down(dev, 0); + ipoib_ib_dev_stop(dev, 0); ipoib_pkey_dev_delay_open(dev); return; } From a9d1884925c80b96a621939a4fef5d74de58debe Mon Sep 17 00:00:00 2001 From: Eli Cohen Date: Thu, 14 Feb 2008 13:15:28 +0200 Subject: [PATCH 07/10] IPoIB: Remove unused struct ipoib_cm_tx.ibwc member struct ipoib_cm_tx.ibwc is unused since commit 1b524963 ("IPoIB/cm: Use common CQ for CM send completions"), so remove it. Signed-off-by: Eli Cohen --- drivers/infiniband/ulp/ipoib/ipoib.h | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/infiniband/ulp/ipoib/ipoib.h b/drivers/infiniband/ulp/ipoib/ipoib.h index f9b7caa54143..054fab8e27a0 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib.h +++ b/drivers/infiniband/ulp/ipoib/ipoib.h @@ -209,7 +209,6 @@ struct ipoib_cm_tx { unsigned tx_tail; unsigned long flags; u32 mtu; - struct ib_wc ibwc[IPOIB_NUM_WC]; }; struct ipoib_cm_rx_buf { From e6028c0e004d334bb9ed75d4c918f4c763af1b9f Mon Sep 17 00:00:00 2001 From: Jack Morgenstein Date: Thu, 14 Feb 2008 10:39:36 -0800 Subject: [PATCH 08/10] IB/mlx4: mlx4_ib_fmr_alloc() should call mlx4_fmr_enable() Currently mlx4_ib_fmr_alloc() calls mlx4_mr_enable() instead of mlx4_fmr_enable(). The two functions are equivalent at the moment, but this is not really correct (and the change is needed to fix a bug). Signed-off-by: Jack Morgenstein Signed-off-by: Roland Dreier --- drivers/infiniband/hw/mlx4/mr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c index 7dc91a3e712d..fe2c2e94a5f8 100644 --- a/drivers/infiniband/hw/mlx4/mr.c +++ b/drivers/infiniband/hw/mlx4/mr.c @@ -199,7 +199,7 @@ struct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int acc, if (err) goto err_free; - err = mlx4_mr_enable(to_mdev(pd->device)->dev, &fmr->mfmr.mr); + err = mlx4_fmr_enable(to_mdev(pd->device)->dev, &fmr->mfmr); if (err) goto err_mr; From 11e75a7455a7bc73e752c0c985986c2b1f8c930a Mon Sep 17 00:00:00 2001 From: Jack Morgenstein Date: Thu, 14 Feb 2008 13:41:29 +0200 Subject: [PATCH 09/10] mlx4_core: Move table_find from fmr_alloc to fmr_enable mlx4_table_find (for FMR MPTs) requires that ICM memory already be mapped. Before this fix, FMR allocation depended on ICM memory already being mapped for the MPT entry. If all currently mapped entries are taken, the find operation fails (even if the MPT ICM table still had more entries, which were just not mapped yet). This fix moves the mpt find operation to fmr_enable, to guarantee that any required ICM memory mapping has already occurred. Found by Oren Duer of Mellanox. Signed-off-by: Jack Morgenstein Signed-off-by: Roland Dreier --- drivers/net/mlx4/mr.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/drivers/net/mlx4/mr.c b/drivers/net/mlx4/mr.c index 679dfdb6807f..79b317b88c86 100644 --- a/drivers/net/mlx4/mr.c +++ b/drivers/net/mlx4/mr.c @@ -578,13 +578,6 @@ int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages, goto err_free; } - fmr->mpt = mlx4_table_find(&priv->mr_table.dmpt_table, - key_to_hw_index(fmr->mr.key), NULL); - if (!fmr->mpt) { - err = -ENOMEM; - goto err_free; - } - return 0; err_free: @@ -595,7 +588,19 @@ EXPORT_SYMBOL_GPL(mlx4_fmr_alloc); int mlx4_fmr_enable(struct mlx4_dev *dev, struct mlx4_fmr *fmr) { - return mlx4_mr_enable(dev, &fmr->mr); + struct mlx4_priv *priv = mlx4_priv(dev); + int err; + + err = mlx4_mr_enable(dev, &fmr->mr); + if (err) + return err; + + fmr->mpt = mlx4_table_find(&priv->mr_table.dmpt_table, + key_to_hw_index(fmr->mr.key), NULL); + if (!fmr->mpt) + return -ENOMEM; + + return 0; } EXPORT_SYMBOL_GPL(mlx4_fmr_enable); From ead595aeb0974171eddd012df115424752413c26 Mon Sep 17 00:00:00 2001 From: Sean Hefty Date: Wed, 13 Feb 2008 14:33:53 -0800 Subject: [PATCH 10/10] RDMA/cma: Do not issue MRA if user rejects connection request There's an undesirable interaction with issuing MRA requests to increase connection timeouts and the listen backlog. When the rdma_cm receives a connection request, it queues an MRA with the ib_cm. (The ib_cm will send an MRA if it receives a duplicate REQ.) The rdma_cm will then create a new rdma_cm_id and give that to the user, which in this case is the rdma_user_cm. If the listen backlog maintained in the rdma_user_cm is full, it destroys the rdma_cm_id, which in turns destroys the ib_cm_id. The ib_cm_id generates a REJ because the state of the ib_cm_id has changed to MRA sent, versus REQ received. When the backlog is full, we just want to drop the REQ so that it is retried later. Fix this by deferring queuing the MRA until after the user of the rdma_cm has examined the connection request. Signed-off-by: Sean Hefty Signed-off-by: Roland Dreier --- drivers/infiniband/core/cma.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index 1eff1b2c0e08..34507daaf9b6 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -1107,7 +1107,6 @@ static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) event.param.ud.private_data_len = IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset; } else { - ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0); conn_id = cma_new_conn_id(&listen_id->id, ib_event); cma_set_req_event_data(&event, &ib_event->param.req_rcvd, ib_event->private_data, offset); @@ -1130,6 +1129,15 @@ static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event) ret = conn_id->id.event_handler(&conn_id->id, &event); if (!ret) { + /* + * Acquire mutex to prevent user executing rdma_destroy_id() + * while we're accessing the cm_id. + */ + mutex_lock(&lock); + if (cma_comp(conn_id, CMA_CONNECT) && + !cma_is_ud_ps(conn_id->id.ps)) + ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0); + mutex_unlock(&lock); cma_enable_remove(conn_id); goto out; }