2007-09-10 19:50:12 +02:00
|
|
|
/*
|
|
|
|
* Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
|
|
|
|
*
|
|
|
|
* This software is available to you under a choice of one of two
|
|
|
|
* licenses. You may choose to be licensed under the terms of the GNU
|
|
|
|
* General Public License (GPL) Version 2, available from the file
|
|
|
|
* COPYING in the main directory of this source tree, or the BSD-type
|
|
|
|
* license below:
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
*
|
|
|
|
* Redistributions of source code must retain the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
*
|
|
|
|
* Redistributions in binary form must reproduce the above
|
|
|
|
* copyright notice, this list of conditions and the following
|
|
|
|
* disclaimer in the documentation and/or other materials provided
|
|
|
|
* with the distribution.
|
|
|
|
*
|
|
|
|
* Neither the name of the Network Appliance, Inc. nor the names of
|
|
|
|
* its contributors may be used to endorse or promote products
|
|
|
|
* derived from this software without specific prior written
|
|
|
|
* permission.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
|
|
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
|
|
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
|
|
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
|
|
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
|
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
|
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
|
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef _LINUX_SUNRPC_XPRT_RDMA_H
|
|
|
|
#define _LINUX_SUNRPC_XPRT_RDMA_H
|
|
|
|
|
|
|
|
#include <linux/wait.h> /* wait_queue_head_t, etc */
|
|
|
|
#include <linux/spinlock.h> /* spinlock_t, etc */
|
2011-07-27 01:09:06 +02:00
|
|
|
#include <linux/atomic.h> /* atomic_t, etc */
|
2014-05-28 16:32:17 +02:00
|
|
|
#include <linux/workqueue.h> /* struct work_struct */
|
2007-09-10 19:50:12 +02:00
|
|
|
|
|
|
|
#include <rdma/rdma_cm.h> /* RDMA connection api */
|
|
|
|
#include <rdma/ib_verbs.h> /* RDMA verbs api */
|
|
|
|
|
|
|
|
#include <linux/sunrpc/clnt.h> /* rpc_xprt */
|
|
|
|
#include <linux/sunrpc/rpc_rdma.h> /* RPC/RDMA protocol */
|
|
|
|
#include <linux/sunrpc/xprtrdma.h> /* xprt parameters */
|
|
|
|
|
2008-10-09 21:01:41 +02:00
|
|
|
#define RDMA_RESOLVE_TIMEOUT (5000) /* 5 seconds */
|
|
|
|
#define RDMA_CONNECT_RETRY_MAX (2) /* retries if no listener backlog */
|
|
|
|
|
2016-01-07 20:50:10 +01:00
|
|
|
#define RPCRDMA_BIND_TO (60U * HZ)
|
|
|
|
#define RPCRDMA_INIT_REEST_TO (5U * HZ)
|
|
|
|
#define RPCRDMA_MAX_REEST_TO (30U * HZ)
|
|
|
|
#define RPCRDMA_IDLE_DISC_TO (5U * 60 * HZ)
|
|
|
|
|
2007-09-10 19:50:12 +02:00
|
|
|
/*
|
|
|
|
* Interface Adapter -- one per transport instance
|
|
|
|
*/
|
|
|
|
struct rpcrdma_ia {
|
2015-03-30 20:34:21 +02:00
|
|
|
const struct rpcrdma_memreg_ops *ri_ops;
|
2015-05-26 17:51:56 +02:00
|
|
|
struct ib_device *ri_device;
|
2007-09-10 19:50:12 +02:00
|
|
|
struct rdma_cm_id *ri_id;
|
|
|
|
struct ib_pd *ri_pd;
|
|
|
|
struct completion ri_done;
|
|
|
|
int ri_async_rc;
|
2016-09-15 16:57:07 +02:00
|
|
|
unsigned int ri_max_segs;
|
2014-05-28 16:32:00 +02:00
|
|
|
unsigned int ri_max_frmr_depth;
|
2016-05-02 20:41:05 +02:00
|
|
|
unsigned int ri_max_inline_write;
|
|
|
|
unsigned int ri_max_inline_read;
|
2016-09-15 16:57:16 +02:00
|
|
|
bool ri_reminv_expected;
|
2015-01-21 17:03:35 +01:00
|
|
|
struct ib_qp_attr ri_qp_attr;
|
|
|
|
struct ib_qp_init_attr ri_qp_init_attr;
|
2007-09-10 19:50:12 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
/*
|
|
|
|
* RDMA Endpoint -- one per transport instance
|
|
|
|
*/
|
|
|
|
|
|
|
|
struct rpcrdma_ep {
|
|
|
|
atomic_t rep_cqcount;
|
|
|
|
int rep_cqinit;
|
|
|
|
int rep_connected;
|
|
|
|
struct ib_qp_init_attr rep_attr;
|
|
|
|
wait_queue_head_t rep_connect_wait;
|
2016-09-15 16:57:07 +02:00
|
|
|
struct rpcrdma_connect_private rep_cm_private;
|
2007-09-10 19:50:12 +02:00
|
|
|
struct rdma_conn_param rep_remote_cma;
|
|
|
|
struct sockaddr_storage rep_remote_addr;
|
2014-05-28 16:32:17 +02:00
|
|
|
struct delayed_work rep_connect_worker;
|
2007-09-10 19:50:12 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
#define INIT_CQCOUNT(ep) atomic_set(&(ep)->rep_cqcount, (ep)->rep_cqinit)
|
|
|
|
#define DECR_CQCOUNT(ep) atomic_sub_return(1, &(ep)->rep_cqcount)
|
|
|
|
|
2015-10-24 23:27:51 +02:00
|
|
|
/* Pre-allocate extra Work Requests for handling backward receives
|
|
|
|
* and sends. This is a fixed value because the Work Queues are
|
|
|
|
* allocated when the forward channel is set up.
|
|
|
|
*/
|
|
|
|
#if defined(CONFIG_SUNRPC_BACKCHANNEL)
|
|
|
|
#define RPCRDMA_BACKWARD_WRS (8)
|
|
|
|
#else
|
|
|
|
#define RPCRDMA_BACKWARD_WRS (0)
|
|
|
|
#endif
|
|
|
|
|
2015-01-21 17:04:00 +01:00
|
|
|
/* Registered buffer -- registered kmalloc'd memory for RDMA SEND/RECV
|
|
|
|
*
|
|
|
|
* The below structure appears at the front of a large region of kmalloc'd
|
|
|
|
* memory, which always starts on a good alignment boundary.
|
|
|
|
*/
|
|
|
|
|
|
|
|
struct rpcrdma_regbuf {
|
|
|
|
struct ib_sge rg_iov;
|
xprtrdma: Delay DMA mapping Send and Receive buffers
Currently, each regbuf is allocated and DMA mapped at the same time.
This is done during transport creation.
When a device driver is unloaded, every DMA-mapped buffer in use by
a transport has to be unmapped, and then remapped to the new
device if the driver is loaded again. Remapping will have to be done
_after_ the connect worker has set up the new device.
But there's an ordering problem:
call_allocate, which invokes xprt_rdma_allocate which calls
rpcrdma_alloc_regbuf to allocate Send buffers, happens _before_
the connect worker can run to set up the new device.
Instead, at transport creation, allocate each buffer, but leave it
unmapped. Once the RPC carries these buffers into ->send_request, by
which time a transport connection should have been established,
check to see that the RPC's buffers have been DMA mapped. If not,
map them there.
When device driver unplug support is added, it will simply unmap all
the transport's regbufs, but it doesn't have to deallocate the
underlying memory.
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>
2016-09-15 16:56:18 +02:00
|
|
|
struct ib_device *rg_device;
|
2016-09-15 16:56:10 +02:00
|
|
|
enum dma_data_direction rg_direction;
|
2015-01-21 17:04:00 +01:00
|
|
|
__be32 rg_base[0] __attribute__ ((aligned(256)));
|
|
|
|
};
|
|
|
|
|
|
|
|
static inline u64
|
|
|
|
rdmab_addr(struct rpcrdma_regbuf *rb)
|
|
|
|
{
|
|
|
|
return rb->rg_iov.addr;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline u32
|
|
|
|
rdmab_length(struct rpcrdma_regbuf *rb)
|
|
|
|
{
|
|
|
|
return rb->rg_iov.length;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline u32
|
|
|
|
rdmab_lkey(struct rpcrdma_regbuf *rb)
|
|
|
|
{
|
|
|
|
return rb->rg_iov.lkey;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline struct rpcrdma_msg *
|
|
|
|
rdmab_to_msg(struct rpcrdma_regbuf *rb)
|
|
|
|
{
|
|
|
|
return (struct rpcrdma_msg *)rb->rg_base;
|
|
|
|
}
|
|
|
|
|
2016-01-07 20:50:10 +01:00
|
|
|
#define RPCRDMA_DEF_GFP (GFP_NOIO | __GFP_NOWARN)
|
|
|
|
|
2016-05-02 20:40:56 +02:00
|
|
|
/* To ensure a transport can always make forward progress,
|
|
|
|
* the number of RDMA segments allowed in header chunk lists
|
|
|
|
* is capped at 8. This prevents less-capable devices and
|
|
|
|
* memory registrations from overrunning the Send buffer
|
|
|
|
* while building chunk lists.
|
|
|
|
*
|
|
|
|
* Elements of the Read list take up more room than the
|
|
|
|
* Write list or Reply chunk. 8 read segments means the Read
|
|
|
|
* list (or Write list or Reply chunk) cannot consume more
|
|
|
|
* than
|
|
|
|
*
|
|
|
|
* ((8 + 2) * read segment size) + 1 XDR words, or 244 bytes.
|
|
|
|
*
|
|
|
|
* And the fixed part of the header is another 24 bytes.
|
|
|
|
*
|
|
|
|
* The smallest inline threshold is 1024 bytes, ensuring that
|
|
|
|
* at least 750 bytes are available for RPC messages.
|
|
|
|
*/
|
2016-09-15 16:56:02 +02:00
|
|
|
enum {
|
|
|
|
RPCRDMA_MAX_HDR_SEGS = 8,
|
|
|
|
RPCRDMA_HDRBUF_SIZE = 256,
|
|
|
|
};
|
2016-05-02 20:40:56 +02:00
|
|
|
|
2007-09-10 19:50:12 +02:00
|
|
|
/*
|
|
|
|
* struct rpcrdma_rep -- this structure encapsulates state required to recv
|
|
|
|
* and complete a reply, asychronously. It needs several pieces of
|
|
|
|
* state:
|
|
|
|
* o recv buffer (posted to provider)
|
|
|
|
* o ib_sge (also donated to provider)
|
|
|
|
* o status of reply (length, success or not)
|
2016-06-29 19:54:25 +02:00
|
|
|
* o bookkeeping state to get run by reply handler (list, etc)
|
2007-09-10 19:50:12 +02:00
|
|
|
*
|
2016-06-29 19:54:25 +02:00
|
|
|
* These are allocated during initialization, per-transport instance.
|
2007-09-10 19:50:12 +02:00
|
|
|
*
|
|
|
|
* N of these are associated with a transport instance, and stored in
|
|
|
|
* struct rpcrdma_buffer. N is the max number of outstanding requests.
|
|
|
|
*/
|
|
|
|
|
|
|
|
struct rpcrdma_rep {
|
2016-03-04 17:28:36 +01:00
|
|
|
struct ib_cqe rr_cqe;
|
2015-01-21 17:04:25 +01:00
|
|
|
unsigned int rr_len;
|
2016-09-15 16:57:16 +02:00
|
|
|
int rr_wc_flags;
|
|
|
|
u32 rr_inv_rkey;
|
2015-05-26 17:51:56 +02:00
|
|
|
struct ib_device *rr_device;
|
2015-05-26 17:51:37 +02:00
|
|
|
struct rpcrdma_xprt *rr_rxprt;
|
2015-10-24 23:27:10 +02:00
|
|
|
struct work_struct rr_work;
|
2015-01-21 17:04:25 +01:00
|
|
|
struct list_head rr_list;
|
2016-09-15 16:56:51 +02:00
|
|
|
struct ib_recv_wr rr_recv_wr;
|
2015-01-21 17:04:25 +01:00
|
|
|
struct rpcrdma_regbuf *rr_rdmabuf;
|
2007-09-10 19:50:12 +02:00
|
|
|
};
|
|
|
|
|
2015-10-24 23:26:54 +02:00
|
|
|
#define RPCRDMA_BAD_LEN (~0U)
|
|
|
|
|
2014-07-29 23:24:09 +02:00
|
|
|
/*
|
|
|
|
* struct rpcrdma_mw - external memory region metadata
|
|
|
|
*
|
|
|
|
* An external memory region is any buffer or page that is registered
|
|
|
|
* on the fly (ie, not pre-registered).
|
|
|
|
*
|
2014-07-29 23:24:28 +02:00
|
|
|
* Each rpcrdma_buffer has a list of free MWs anchored in rb_mws. During
|
2014-07-29 23:24:09 +02:00
|
|
|
* call_allocate, rpcrdma_buffer_get() assigns one to each segment in
|
|
|
|
* an rpcrdma_req. Then rpcrdma_register_external() grabs these to keep
|
|
|
|
* track of registration metadata while each RPC is pending.
|
|
|
|
* rpcrdma_deregister_external() uses this metadata to unmap and
|
|
|
|
* release these resources when an RPC is complete.
|
|
|
|
*/
|
|
|
|
enum rpcrdma_frmr_state {
|
|
|
|
FRMR_IS_INVALID, /* ready to be used */
|
|
|
|
FRMR_IS_VALID, /* in use */
|
2016-11-07 22:16:24 +01:00
|
|
|
FRMR_FLUSHED_FR, /* flushed FASTREG WR */
|
|
|
|
FRMR_FLUSHED_LI, /* flushed LOCALINV WR */
|
2014-07-29 23:24:09 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
struct rpcrdma_frmr {
|
|
|
|
struct ib_mr *fr_mr;
|
2016-03-04 17:28:53 +01:00
|
|
|
struct ib_cqe fr_cqe;
|
2014-07-29 23:24:09 +02:00
|
|
|
enum rpcrdma_frmr_state fr_state;
|
2016-03-04 17:28:53 +01:00
|
|
|
struct completion fr_linv_done;
|
2015-12-16 23:22:31 +01:00
|
|
|
union {
|
|
|
|
struct ib_reg_wr fr_regwr;
|
|
|
|
struct ib_send_wr fr_invwr;
|
|
|
|
};
|
2014-07-29 23:24:09 +02:00
|
|
|
};
|
|
|
|
|
2015-05-26 17:53:23 +02:00
|
|
|
struct rpcrdma_fmr {
|
2016-06-29 19:52:37 +02:00
|
|
|
struct ib_fmr *fm_mr;
|
|
|
|
u64 *fm_physaddrs;
|
2014-07-29 23:24:09 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
struct rpcrdma_mw {
|
2016-06-29 19:52:21 +02:00
|
|
|
struct list_head mw_list;
|
|
|
|
struct scatterlist *mw_sg;
|
|
|
|
int mw_nents;
|
|
|
|
enum dma_data_direction mw_dir;
|
2014-07-29 23:24:09 +02:00
|
|
|
union {
|
2015-05-26 17:53:23 +02:00
|
|
|
struct rpcrdma_fmr fmr;
|
2014-07-29 23:24:09 +02:00
|
|
|
struct rpcrdma_frmr frmr;
|
2016-03-04 17:28:45 +01:00
|
|
|
};
|
2016-05-02 20:42:29 +02:00
|
|
|
struct rpcrdma_xprt *mw_xprt;
|
2016-06-29 19:54:16 +02:00
|
|
|
u32 mw_handle;
|
|
|
|
u32 mw_length;
|
|
|
|
u64 mw_offset;
|
2014-07-29 23:24:28 +02:00
|
|
|
struct list_head mw_all;
|
2014-07-29 23:24:09 +02:00
|
|
|
};
|
|
|
|
|
2007-09-10 19:50:12 +02:00
|
|
|
/*
|
|
|
|
* struct rpcrdma_req -- structure central to the request/reply sequence.
|
|
|
|
*
|
|
|
|
* N of these are associated with a transport instance, and stored in
|
|
|
|
* struct rpcrdma_buffer. N is the max number of outstanding requests.
|
|
|
|
*
|
|
|
|
* It includes pre-registered buffer memory for send AND recv.
|
|
|
|
* The recv buffer, however, is not owned by this structure, and
|
|
|
|
* is "donated" to the hardware when a recv is posted. When a
|
|
|
|
* reply is handled, the recv buffer used is given back to the
|
|
|
|
* struct rpcrdma_req associated with the request.
|
|
|
|
*
|
|
|
|
* In addition to the basic memory, this structure includes an array
|
|
|
|
* of iovs for send operations. The reason is that the iovs passed to
|
|
|
|
* ib_post_{send,recv} must not be modified until the work request
|
|
|
|
* completes.
|
|
|
|
*/
|
|
|
|
|
2016-06-29 19:54:25 +02:00
|
|
|
/* Maximum number of page-sized "segments" per chunk list to be
|
|
|
|
* registered or invalidated. Must handle a Reply chunk:
|
|
|
|
*/
|
|
|
|
enum {
|
|
|
|
RPCRDMA_MAX_IOV_SEGS = 3,
|
|
|
|
RPCRDMA_MAX_DATA_SEGS = ((1 * 1024 * 1024) / PAGE_SIZE) + 1,
|
|
|
|
RPCRDMA_MAX_SEGS = RPCRDMA_MAX_DATA_SEGS +
|
|
|
|
RPCRDMA_MAX_IOV_SEGS,
|
|
|
|
};
|
|
|
|
|
2007-09-10 19:50:12 +02:00
|
|
|
struct rpcrdma_mr_seg { /* chunk descriptors */
|
|
|
|
u32 mr_len; /* length of chunk or segment */
|
|
|
|
struct page *mr_page; /* owning page, if any */
|
|
|
|
char *mr_offset; /* kva if no page, else offset */
|
|
|
|
};
|
|
|
|
|
xprtrdma: Use gathered Send for large inline messages
An RPC Call message that is sent inline but that has a data payload
(ie, one or more items in rq_snd_buf's page list) must be "pulled
up:"
- call_allocate has to reserve enough RPC Call buffer space to
accommodate the data payload
- call_transmit has to memcopy the rq_snd_buf's page list and tail
into its head iovec before it is sent
As the inline threshold is increased beyond its current 1KB default,
however, this means data payloads of more than a few KB are copied
by the host CPU. For example, if the inline threshold is increased
just to 4KB, then NFS WRITE requests up to 4KB would involve a
memcpy of the NFS WRITE's payload data into the RPC Call buffer.
This is an undesirable amount of participation by the host CPU.
The inline threshold may be much larger than 4KB in the future,
after negotiation with a peer server.
Instead of copying the components of rq_snd_buf into its head iovec,
construct a gather list of these components, and send them all in
place. The same approach is already used in the Linux server's
RPC-over-RDMA reply path.
This mechanism also eliminates the need for rpcrdma_tail_pullup,
which is used to manage the XDR pad and trailing inline content when
a Read list is present.
This requires that the pages in rq_snd_buf's page list be DMA-mapped
during marshaling, and unmapped when a data-bearing RPC is
completed. This is slightly less efficient for very small I/O
payloads, but significantly more efficient as data payload size and
inline threshold increase past a kilobyte.
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>
2016-09-15 16:57:24 +02:00
|
|
|
/* Reserve enough Send SGEs to send a maximum size inline request:
|
|
|
|
* - RPC-over-RDMA header
|
|
|
|
* - xdr_buf head iovec
|
|
|
|
* - RPCRDMA_MAX_INLINE bytes, possibly unaligned, in pages
|
|
|
|
* - xdr_buf tail iovec
|
|
|
|
*/
|
|
|
|
enum {
|
|
|
|
RPCRDMA_MAX_SEND_PAGES = PAGE_SIZE + RPCRDMA_MAX_INLINE - 1,
|
|
|
|
RPCRDMA_MAX_PAGE_SGES = (RPCRDMA_MAX_SEND_PAGES >> PAGE_SHIFT) + 1,
|
|
|
|
RPCRDMA_MAX_SEND_SGES = 1 + 1 + RPCRDMA_MAX_PAGE_SGES + 1,
|
|
|
|
};
|
2015-08-03 19:03:39 +02:00
|
|
|
|
2016-06-29 19:54:25 +02:00
|
|
|
struct rpcrdma_buffer;
|
2007-09-10 19:50:12 +02:00
|
|
|
struct rpcrdma_req {
|
2015-10-24 23:27:02 +02:00
|
|
|
struct list_head rl_free;
|
xprtrdma: Use gathered Send for large inline messages
An RPC Call message that is sent inline but that has a data payload
(ie, one or more items in rq_snd_buf's page list) must be "pulled
up:"
- call_allocate has to reserve enough RPC Call buffer space to
accommodate the data payload
- call_transmit has to memcopy the rq_snd_buf's page list and tail
into its head iovec before it is sent
As the inline threshold is increased beyond its current 1KB default,
however, this means data payloads of more than a few KB are copied
by the host CPU. For example, if the inline threshold is increased
just to 4KB, then NFS WRITE requests up to 4KB would involve a
memcpy of the NFS WRITE's payload data into the RPC Call buffer.
This is an undesirable amount of participation by the host CPU.
The inline threshold may be much larger than 4KB in the future,
after negotiation with a peer server.
Instead of copying the components of rq_snd_buf into its head iovec,
construct a gather list of these components, and send them all in
place. The same approach is already used in the Linux server's
RPC-over-RDMA reply path.
This mechanism also eliminates the need for rpcrdma_tail_pullup,
which is used to manage the XDR pad and trailing inline content when
a Read list is present.
This requires that the pages in rq_snd_buf's page list be DMA-mapped
during marshaling, and unmapped when a data-bearing RPC is
completed. This is slightly less efficient for very small I/O
payloads, but significantly more efficient as data payload size and
inline threshold increase past a kilobyte.
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>
2016-09-15 16:57:24 +02:00
|
|
|
unsigned int rl_mapped_sges;
|
2015-08-03 19:03:39 +02:00
|
|
|
unsigned int rl_connect_cookie;
|
|
|
|
struct rpcrdma_buffer *rl_buffer;
|
2016-09-15 16:56:43 +02:00
|
|
|
struct rpcrdma_rep *rl_reply;
|
|
|
|
struct ib_send_wr rl_send_wr;
|
xprtrdma: Use gathered Send for large inline messages
An RPC Call message that is sent inline but that has a data payload
(ie, one or more items in rq_snd_buf's page list) must be "pulled
up:"
- call_allocate has to reserve enough RPC Call buffer space to
accommodate the data payload
- call_transmit has to memcopy the rq_snd_buf's page list and tail
into its head iovec before it is sent
As the inline threshold is increased beyond its current 1KB default,
however, this means data payloads of more than a few KB are copied
by the host CPU. For example, if the inline threshold is increased
just to 4KB, then NFS WRITE requests up to 4KB would involve a
memcpy of the NFS WRITE's payload data into the RPC Call buffer.
This is an undesirable amount of participation by the host CPU.
The inline threshold may be much larger than 4KB in the future,
after negotiation with a peer server.
Instead of copying the components of rq_snd_buf into its head iovec,
construct a gather list of these components, and send them all in
place. The same approach is already used in the Linux server's
RPC-over-RDMA reply path.
This mechanism also eliminates the need for rpcrdma_tail_pullup,
which is used to manage the XDR pad and trailing inline content when
a Read list is present.
This requires that the pages in rq_snd_buf's page list be DMA-mapped
during marshaling, and unmapped when a data-bearing RPC is
completed. This is slightly less efficient for very small I/O
payloads, but significantly more efficient as data payload size and
inline threshold increase past a kilobyte.
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>
2016-09-15 16:57:24 +02:00
|
|
|
struct ib_sge rl_send_sge[RPCRDMA_MAX_SEND_SGES];
|
xprtrdma: Initialize separate RPC call and reply buffers
RPC-over-RDMA needs to separate its RPC call and reply buffers.
o When an RPC Call is sent, rq_snd_buf is DMA mapped for an RDMA
Send operation using DMA_TO_DEVICE
o If the client expects a large RPC reply, it DMA maps rq_rcv_buf
as part of a Reply chunk using DMA_FROM_DEVICE
The two mappings are for data movement in opposite directions.
DMA-API.txt suggests that if these mappings share a DMA cacheline,
bad things can happen. This could occur in the final bytes of
rq_snd_buf and the first bytes of rq_rcv_buf if the two buffers
happen to share a DMA cacheline.
On x86_64 the cacheline size is typically 8 bytes, and RPC call
messages are usually much smaller than the send buffer, so this
hasn't been a noticeable problem. But the DMA cacheline size can be
larger on other platforms.
Also, often rq_rcv_buf starts most of the way into a page, thus
an additional RDMA segment is needed to map and register the end of
that buffer. Try to avoid that scenario to reduce the cost of
registering and invalidating Reply chunks.
Instead of carrying a single regbuf that covers both rq_snd_buf and
rq_rcv_buf, each struct rpcrdma_req now carries one regbuf for
rq_snd_buf and one regbuf for rq_rcv_buf.
Some incidental changes worth noting:
- To clear out some spaghetti, refactor xprt_rdma_allocate.
- The value stored in rg_size is the same as the value stored in
the iov.length field, so eliminate rg_size
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>
2016-09-15 16:55:53 +02:00
|
|
|
struct rpcrdma_regbuf *rl_rdmabuf; /* xprt header */
|
|
|
|
struct rpcrdma_regbuf *rl_sendbuf; /* rq_snd_buf */
|
|
|
|
struct rpcrdma_regbuf *rl_recvbuf; /* rq_rcv_buf */
|
2015-10-24 23:27:43 +02:00
|
|
|
|
2016-03-04 17:28:53 +01:00
|
|
|
struct ib_cqe rl_cqe;
|
2015-10-24 23:27:43 +02:00
|
|
|
struct list_head rl_all;
|
|
|
|
bool rl_backchannel;
|
2016-06-29 19:54:25 +02:00
|
|
|
|
|
|
|
struct list_head rl_registered; /* registered segments */
|
|
|
|
struct rpcrdma_mr_seg rl_segments[RPCRDMA_MAX_SEGS];
|
2007-09-10 19:50:12 +02:00
|
|
|
};
|
2015-01-21 17:04:08 +01:00
|
|
|
|
2016-09-15 16:55:45 +02:00
|
|
|
static inline void
|
|
|
|
rpcrdma_set_xprtdata(struct rpc_rqst *rqst, struct rpcrdma_req *req)
|
|
|
|
{
|
|
|
|
rqst->rq_xprtdata = req;
|
|
|
|
}
|
|
|
|
|
2015-01-21 17:04:08 +01:00
|
|
|
static inline struct rpcrdma_req *
|
|
|
|
rpcr_to_rdmar(struct rpc_rqst *rqst)
|
|
|
|
{
|
2016-09-15 16:55:45 +02:00
|
|
|
return rqst->rq_xprtdata;
|
2015-01-21 17:04:08 +01:00
|
|
|
}
|
2007-09-10 19:50:12 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* struct rpcrdma_buffer -- holds list/queue of pre-registered memory for
|
|
|
|
* inline requests/replies, and client/server credits.
|
|
|
|
*
|
|
|
|
* One of these is associated with a transport instance
|
|
|
|
*/
|
|
|
|
struct rpcrdma_buffer {
|
2015-05-26 17:53:13 +02:00
|
|
|
spinlock_t rb_mwlock; /* protect rb_mws list */
|
|
|
|
struct list_head rb_mws;
|
|
|
|
struct list_head rb_all;
|
|
|
|
char *rb_pool;
|
|
|
|
|
2015-10-24 23:27:02 +02:00
|
|
|
spinlock_t rb_lock; /* protect buf lists */
|
xprtrdma: Fix receive buffer accounting
An RPC can terminate before its reply arrives, if a credential
problem or a soft timeout occurs. After this happens, xprtrdma
reports it is out of Receive buffers.
A Receive buffer is posted before each RPC is sent, and returned to
the buffer pool when a reply is received. If no reply is received
for an RPC, that Receive buffer remains posted. But xprtrdma tries
to post another when the next RPC is sent.
If this happens a few dozen times, there are no receive buffers left
to be posted at send time. I don't see a way for a transport
connection to recover at that point, and it will spit warnings and
unnecessarily delay RPCs on occasion for its remaining lifetime.
Commit 1e465fd4ff47 ("xprtrdma: Replace send and receive arrays")
removed a little bit of logic to detect this case and not provide
a Receive buffer so no more buffers are posted, and then transport
operation continues correctly. We didn't understand what that logic
did, and it wasn't commented, so it was removed as part of the
overhaul to support backchannel requests.
Restore it, but be wary of the need to keep extra Receives posted
to deal with backchannel requests.
Fixes: 1e465fd4ff47 ("xprtrdma: Replace send and receive arrays")
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Reviewed-by: Anna Schumaker <Anna.Schumaker@Netapp.com>
Signed-off-by: Trond Myklebust <trond.myklebust@primarydata.com>
2016-09-06 17:22:58 +02:00
|
|
|
int rb_send_count, rb_recv_count;
|
2015-10-24 23:27:02 +02:00
|
|
|
struct list_head rb_send_bufs;
|
|
|
|
struct list_head rb_recv_bufs;
|
2015-05-26 17:53:13 +02:00
|
|
|
u32 rb_max_requests;
|
2016-03-04 17:28:27 +01:00
|
|
|
atomic_t rb_credits; /* most recent credit grant */
|
2015-10-24 23:27:43 +02:00
|
|
|
|
|
|
|
u32 rb_bc_srv_max_requests;
|
|
|
|
spinlock_t rb_reqslock; /* protect rb_allreqs */
|
|
|
|
struct list_head rb_allreqs;
|
2016-01-07 20:50:10 +01:00
|
|
|
|
|
|
|
u32 rb_bc_max_requests;
|
2016-06-29 19:52:54 +02:00
|
|
|
|
|
|
|
spinlock_t rb_recovery_lock; /* protect rb_stale_mrs */
|
|
|
|
struct list_head rb_stale_mrs;
|
|
|
|
struct delayed_work rb_recovery_worker;
|
2016-06-29 19:54:00 +02:00
|
|
|
struct delayed_work rb_refresh_worker;
|
2007-09-10 19:50:12 +02:00
|
|
|
};
|
|
|
|
#define rdmab_to_ia(b) (&container_of((b), struct rpcrdma_xprt, rx_buf)->rx_ia)
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Internal structure for transport instance creation. This
|
|
|
|
* exists primarily for modularity.
|
|
|
|
*
|
|
|
|
* This data should be set with mount options
|
|
|
|
*/
|
|
|
|
struct rpcrdma_create_data_internal {
|
|
|
|
struct sockaddr_storage addr; /* RDMA server address */
|
|
|
|
unsigned int max_requests; /* max requests (slots) in flight */
|
|
|
|
unsigned int rsize; /* mount rsize - max read hdr+data */
|
|
|
|
unsigned int wsize; /* mount wsize - max write hdr+data */
|
|
|
|
unsigned int inline_rsize; /* max non-rdma read data payload */
|
|
|
|
unsigned int inline_wsize; /* max non-rdma write data payload */
|
|
|
|
unsigned int padding; /* non-rdma write header padding */
|
|
|
|
};
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Statistics for RPCRDMA
|
|
|
|
*/
|
|
|
|
struct rpcrdma_stats {
|
|
|
|
unsigned long read_chunk_count;
|
|
|
|
unsigned long write_chunk_count;
|
|
|
|
unsigned long reply_chunk_count;
|
|
|
|
|
|
|
|
unsigned long long total_rdma_request;
|
|
|
|
unsigned long long total_rdma_reply;
|
|
|
|
|
|
|
|
unsigned long long pullup_copy_count;
|
|
|
|
unsigned long long fixup_copy_count;
|
|
|
|
unsigned long hardway_register_count;
|
|
|
|
unsigned long failed_marshal_count;
|
|
|
|
unsigned long bad_reply_count;
|
2015-08-03 19:04:45 +02:00
|
|
|
unsigned long nomsg_call_count;
|
2015-10-24 23:28:08 +02:00
|
|
|
unsigned long bcall_count;
|
2016-06-29 19:52:54 +02:00
|
|
|
unsigned long mrs_recovered;
|
|
|
|
unsigned long mrs_orphaned;
|
2016-06-29 19:54:00 +02:00
|
|
|
unsigned long mrs_allocated;
|
2016-09-15 16:57:16 +02:00
|
|
|
unsigned long local_inv_needed;
|
2007-09-10 19:50:12 +02:00
|
|
|
};
|
|
|
|
|
2015-03-30 20:34:21 +02:00
|
|
|
/*
|
|
|
|
* Per-registration mode operations
|
|
|
|
*/
|
2015-03-30 20:34:30 +02:00
|
|
|
struct rpcrdma_xprt;
|
2015-03-30 20:34:21 +02:00
|
|
|
struct rpcrdma_memreg_ops {
|
2015-03-30 20:34:39 +02:00
|
|
|
int (*ro_map)(struct rpcrdma_xprt *,
|
2016-06-29 19:54:16 +02:00
|
|
|
struct rpcrdma_mr_seg *, int, bool,
|
|
|
|
struct rpcrdma_mw **);
|
2015-12-16 23:22:39 +01:00
|
|
|
void (*ro_unmap_sync)(struct rpcrdma_xprt *,
|
|
|
|
struct rpcrdma_req *);
|
2016-05-02 20:42:46 +02:00
|
|
|
void (*ro_unmap_safe)(struct rpcrdma_xprt *,
|
|
|
|
struct rpcrdma_req *, bool);
|
2016-06-29 19:52:54 +02:00
|
|
|
void (*ro_recover_mr)(struct rpcrdma_mw *);
|
2015-03-30 20:35:26 +02:00
|
|
|
int (*ro_open)(struct rpcrdma_ia *,
|
|
|
|
struct rpcrdma_ep *,
|
|
|
|
struct rpcrdma_create_data_internal *);
|
2015-03-30 20:34:30 +02:00
|
|
|
size_t (*ro_maxpages)(struct rpcrdma_xprt *);
|
2016-06-29 19:54:00 +02:00
|
|
|
int (*ro_init_mr)(struct rpcrdma_ia *,
|
|
|
|
struct rpcrdma_mw *);
|
|
|
|
void (*ro_release_mr)(struct rpcrdma_mw *);
|
2015-03-30 20:34:21 +02:00
|
|
|
const char *ro_displayname;
|
2016-09-15 16:57:16 +02:00
|
|
|
const int ro_send_w_inv_ok;
|
2015-03-30 20:34:21 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
extern const struct rpcrdma_memreg_ops rpcrdma_fmr_memreg_ops;
|
|
|
|
extern const struct rpcrdma_memreg_ops rpcrdma_frwr_memreg_ops;
|
|
|
|
|
2007-09-10 19:50:12 +02:00
|
|
|
/*
|
|
|
|
* RPCRDMA transport -- encapsulates the structures above for
|
|
|
|
* integration with RPC.
|
|
|
|
*
|
|
|
|
* The contained structures are embedded, not pointers,
|
|
|
|
* for convenience. This structure need not be visible externally.
|
|
|
|
*
|
|
|
|
* It is allocated and initialized during mount, and released
|
|
|
|
* during unmount.
|
|
|
|
*/
|
|
|
|
struct rpcrdma_xprt {
|
2015-01-21 17:02:37 +01:00
|
|
|
struct rpc_xprt rx_xprt;
|
2007-09-10 19:50:12 +02:00
|
|
|
struct rpcrdma_ia rx_ia;
|
|
|
|
struct rpcrdma_ep rx_ep;
|
|
|
|
struct rpcrdma_buffer rx_buf;
|
|
|
|
struct rpcrdma_create_data_internal rx_data;
|
2015-01-21 17:02:37 +01:00
|
|
|
struct delayed_work rx_connect_worker;
|
2007-09-10 19:50:12 +02:00
|
|
|
struct rpcrdma_stats rx_stats;
|
|
|
|
};
|
|
|
|
|
2015-01-21 17:02:37 +01:00
|
|
|
#define rpcx_to_rdmax(x) container_of(x, struct rpcrdma_xprt, rx_xprt)
|
2007-09-10 19:50:12 +02:00
|
|
|
#define rpcx_to_rdmad(x) (rpcx_to_rdmax(x)->rx_data)
|
|
|
|
|
2008-10-09 21:01:11 +02:00
|
|
|
/* Setting this to 0 ensures interoperability with early servers.
|
|
|
|
* Setting this to 1 enhances certain unaligned read/write performance.
|
|
|
|
* Default is 0, see sysctl entry and rpc_rdma.c rpcrdma_convert_iovs() */
|
|
|
|
extern int xprt_rdma_pad_optimize;
|
|
|
|
|
2007-09-10 19:50:12 +02:00
|
|
|
/*
|
|
|
|
* Interface Adapter calls - xprtrdma/verbs.c
|
|
|
|
*/
|
|
|
|
int rpcrdma_ia_open(struct rpcrdma_xprt *, struct sockaddr *, int);
|
|
|
|
void rpcrdma_ia_close(struct rpcrdma_ia *);
|
2016-06-29 19:53:27 +02:00
|
|
|
bool frwr_is_supported(struct rpcrdma_ia *);
|
|
|
|
bool fmr_is_supported(struct rpcrdma_ia *);
|
2007-09-10 19:50:12 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Endpoint calls - xprtrdma/verbs.c
|
|
|
|
*/
|
|
|
|
int rpcrdma_ep_create(struct rpcrdma_ep *, struct rpcrdma_ia *,
|
|
|
|
struct rpcrdma_create_data_internal *);
|
2014-05-28 16:33:16 +02:00
|
|
|
void rpcrdma_ep_destroy(struct rpcrdma_ep *, struct rpcrdma_ia *);
|
2007-09-10 19:50:12 +02:00
|
|
|
int rpcrdma_ep_connect(struct rpcrdma_ep *, struct rpcrdma_ia *);
|
2014-07-29 23:25:55 +02:00
|
|
|
void rpcrdma_ep_disconnect(struct rpcrdma_ep *, struct rpcrdma_ia *);
|
2007-09-10 19:50:12 +02:00
|
|
|
|
|
|
|
int rpcrdma_ep_post(struct rpcrdma_ia *, struct rpcrdma_ep *,
|
|
|
|
struct rpcrdma_req *);
|
2016-09-15 16:56:35 +02:00
|
|
|
int rpcrdma_ep_post_recv(struct rpcrdma_ia *, struct rpcrdma_rep *);
|
2007-09-10 19:50:12 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Buffer calls - xprtrdma/verbs.c
|
|
|
|
*/
|
2015-10-24 23:27:43 +02:00
|
|
|
struct rpcrdma_req *rpcrdma_create_req(struct rpcrdma_xprt *);
|
|
|
|
struct rpcrdma_rep *rpcrdma_create_rep(struct rpcrdma_xprt *);
|
2016-09-15 16:56:26 +02:00
|
|
|
void rpcrdma_destroy_req(struct rpcrdma_req *);
|
2015-01-21 17:03:44 +01:00
|
|
|
int rpcrdma_buffer_create(struct rpcrdma_xprt *);
|
2007-09-10 19:50:12 +02:00
|
|
|
void rpcrdma_buffer_destroy(struct rpcrdma_buffer *);
|
|
|
|
|
2015-05-26 17:52:06 +02:00
|
|
|
struct rpcrdma_mw *rpcrdma_get_mw(struct rpcrdma_xprt *);
|
|
|
|
void rpcrdma_put_mw(struct rpcrdma_xprt *, struct rpcrdma_mw *);
|
2007-09-10 19:50:12 +02:00
|
|
|
struct rpcrdma_req *rpcrdma_buffer_get(struct rpcrdma_buffer *);
|
|
|
|
void rpcrdma_buffer_put(struct rpcrdma_req *);
|
|
|
|
void rpcrdma_recv_buffer_get(struct rpcrdma_req *);
|
|
|
|
void rpcrdma_recv_buffer_put(struct rpcrdma_rep *);
|
|
|
|
|
2016-06-29 19:52:54 +02:00
|
|
|
void rpcrdma_defer_mr_recovery(struct rpcrdma_mw *);
|
|
|
|
|
2016-09-15 16:56:26 +02:00
|
|
|
struct rpcrdma_regbuf *rpcrdma_alloc_regbuf(size_t, enum dma_data_direction,
|
2016-09-15 16:56:10 +02:00
|
|
|
gfp_t);
|
xprtrdma: Delay DMA mapping Send and Receive buffers
Currently, each regbuf is allocated and DMA mapped at the same time.
This is done during transport creation.
When a device driver is unloaded, every DMA-mapped buffer in use by
a transport has to be unmapped, and then remapped to the new
device if the driver is loaded again. Remapping will have to be done
_after_ the connect worker has set up the new device.
But there's an ordering problem:
call_allocate, which invokes xprt_rdma_allocate which calls
rpcrdma_alloc_regbuf to allocate Send buffers, happens _before_
the connect worker can run to set up the new device.
Instead, at transport creation, allocate each buffer, but leave it
unmapped. Once the RPC carries these buffers into ->send_request, by
which time a transport connection should have been established,
check to see that the RPC's buffers have been DMA mapped. If not,
map them there.
When device driver unplug support is added, it will simply unmap all
the transport's regbufs, but it doesn't have to deallocate the
underlying memory.
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>
2016-09-15 16:56:18 +02:00
|
|
|
bool __rpcrdma_dma_map_regbuf(struct rpcrdma_ia *, struct rpcrdma_regbuf *);
|
2016-09-15 16:56:26 +02:00
|
|
|
void rpcrdma_free_regbuf(struct rpcrdma_regbuf *);
|
2015-01-21 17:04:00 +01:00
|
|
|
|
xprtrdma: Delay DMA mapping Send and Receive buffers
Currently, each regbuf is allocated and DMA mapped at the same time.
This is done during transport creation.
When a device driver is unloaded, every DMA-mapped buffer in use by
a transport has to be unmapped, and then remapped to the new
device if the driver is loaded again. Remapping will have to be done
_after_ the connect worker has set up the new device.
But there's an ordering problem:
call_allocate, which invokes xprt_rdma_allocate which calls
rpcrdma_alloc_regbuf to allocate Send buffers, happens _before_
the connect worker can run to set up the new device.
Instead, at transport creation, allocate each buffer, but leave it
unmapped. Once the RPC carries these buffers into ->send_request, by
which time a transport connection should have been established,
check to see that the RPC's buffers have been DMA mapped. If not,
map them there.
When device driver unplug support is added, it will simply unmap all
the transport's regbufs, but it doesn't have to deallocate the
underlying memory.
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>
2016-09-15 16:56:18 +02:00
|
|
|
static inline bool
|
|
|
|
rpcrdma_regbuf_is_mapped(struct rpcrdma_regbuf *rb)
|
|
|
|
{
|
|
|
|
return rb->rg_device != NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool
|
|
|
|
rpcrdma_dma_map_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb)
|
|
|
|
{
|
|
|
|
if (likely(rpcrdma_regbuf_is_mapped(rb)))
|
|
|
|
return true;
|
|
|
|
return __rpcrdma_dma_map_regbuf(ia, rb);
|
|
|
|
}
|
|
|
|
|
2015-10-24 23:27:43 +02:00
|
|
|
int rpcrdma_ep_post_extra_recv(struct rpcrdma_xprt *, unsigned int);
|
2015-03-30 20:35:44 +02:00
|
|
|
|
2015-10-24 23:27:10 +02:00
|
|
|
int rpcrdma_alloc_wq(void);
|
|
|
|
void rpcrdma_destroy_wq(void);
|
|
|
|
|
2015-03-30 20:35:44 +02:00
|
|
|
/*
|
|
|
|
* Wrappers for chunk registration, shared by read/write chunk code.
|
|
|
|
*/
|
|
|
|
|
|
|
|
static inline enum dma_data_direction
|
|
|
|
rpcrdma_data_dir(bool writing)
|
|
|
|
{
|
|
|
|
return writing ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
|
|
|
|
}
|
|
|
|
|
2007-09-10 19:50:12 +02:00
|
|
|
/*
|
|
|
|
* RPC/RDMA connection management calls - xprtrdma/rpc_rdma.c
|
|
|
|
*/
|
2014-05-28 16:32:17 +02:00
|
|
|
void rpcrdma_connect_worker(struct work_struct *);
|
2007-09-10 19:50:12 +02:00
|
|
|
void rpcrdma_conn_func(struct rpcrdma_ep *);
|
2016-09-15 16:57:57 +02:00
|
|
|
void rpcrdma_reply_handler(struct work_struct *);
|
2007-09-10 19:50:12 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* RPC/RDMA protocol calls - xprtrdma/rpc_rdma.c
|
|
|
|
*/
|
xprtrdma: Use gathered Send for large inline messages
An RPC Call message that is sent inline but that has a data payload
(ie, one or more items in rq_snd_buf's page list) must be "pulled
up:"
- call_allocate has to reserve enough RPC Call buffer space to
accommodate the data payload
- call_transmit has to memcopy the rq_snd_buf's page list and tail
into its head iovec before it is sent
As the inline threshold is increased beyond its current 1KB default,
however, this means data payloads of more than a few KB are copied
by the host CPU. For example, if the inline threshold is increased
just to 4KB, then NFS WRITE requests up to 4KB would involve a
memcpy of the NFS WRITE's payload data into the RPC Call buffer.
This is an undesirable amount of participation by the host CPU.
The inline threshold may be much larger than 4KB in the future,
after negotiation with a peer server.
Instead of copying the components of rq_snd_buf into its head iovec,
construct a gather list of these components, and send them all in
place. The same approach is already used in the Linux server's
RPC-over-RDMA reply path.
This mechanism also eliminates the need for rpcrdma_tail_pullup,
which is used to manage the XDR pad and trailing inline content when
a Read list is present.
This requires that the pages in rq_snd_buf's page list be DMA-mapped
during marshaling, and unmapped when a data-bearing RPC is
completed. This is slightly less efficient for very small I/O
payloads, but significantly more efficient as data payload size and
inline threshold increase past a kilobyte.
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>
2016-09-15 16:57:24 +02:00
|
|
|
|
|
|
|
enum rpcrdma_chunktype {
|
|
|
|
rpcrdma_noch = 0,
|
|
|
|
rpcrdma_readch,
|
|
|
|
rpcrdma_areadch,
|
|
|
|
rpcrdma_writech,
|
|
|
|
rpcrdma_replych
|
|
|
|
};
|
|
|
|
|
|
|
|
bool rpcrdma_prepare_send_sges(struct rpcrdma_ia *, struct rpcrdma_req *,
|
|
|
|
u32, struct xdr_buf *, enum rpcrdma_chunktype);
|
|
|
|
void rpcrdma_unmap_sges(struct rpcrdma_ia *, struct rpcrdma_req *);
|
2007-09-10 19:50:12 +02:00
|
|
|
int rpcrdma_marshal_req(struct rpc_rqst *);
|
2016-09-15 16:57:07 +02:00
|
|
|
void rpcrdma_set_max_header_sizes(struct rpcrdma_xprt *);
|
2007-09-10 19:50:12 +02:00
|
|
|
|
2015-06-04 17:21:42 +02:00
|
|
|
/* RPC/RDMA module init - xprtrdma/transport.c
|
|
|
|
*/
|
2016-01-07 20:50:10 +01:00
|
|
|
extern unsigned int xprt_rdma_max_inline_read;
|
|
|
|
void xprt_rdma_format_addresses(struct rpc_xprt *xprt, struct sockaddr *sap);
|
|
|
|
void xprt_rdma_free_addresses(struct rpc_xprt *xprt);
|
|
|
|
void xprt_rdma_print_stats(struct rpc_xprt *xprt, struct seq_file *seq);
|
2015-06-04 17:21:42 +02:00
|
|
|
int xprt_rdma_init(void);
|
|
|
|
void xprt_rdma_cleanup(void);
|
|
|
|
|
2015-10-24 23:27:43 +02:00
|
|
|
/* Backchannel calls - xprtrdma/backchannel.c
|
|
|
|
*/
|
|
|
|
#if defined(CONFIG_SUNRPC_BACKCHANNEL)
|
|
|
|
int xprt_rdma_bc_setup(struct rpc_xprt *, unsigned int);
|
2015-10-24 23:28:32 +02:00
|
|
|
int xprt_rdma_bc_up(struct svc_serv *, struct net *);
|
2016-05-02 20:40:40 +02:00
|
|
|
size_t xprt_rdma_bc_maxpayload(struct rpc_xprt *);
|
2015-10-24 23:27:43 +02:00
|
|
|
int rpcrdma_bc_post_recv(struct rpcrdma_xprt *, unsigned int);
|
2015-10-24 23:28:08 +02:00
|
|
|
void rpcrdma_bc_receive_call(struct rpcrdma_xprt *, struct rpcrdma_rep *);
|
2015-10-24 23:27:59 +02:00
|
|
|
int rpcrdma_bc_marshal_reply(struct rpc_rqst *);
|
2015-10-24 23:27:43 +02:00
|
|
|
void xprt_rdma_bc_free_rqst(struct rpc_rqst *);
|
|
|
|
void xprt_rdma_bc_destroy(struct rpc_xprt *, unsigned int);
|
|
|
|
#endif /* CONFIG_SUNRPC_BACKCHANNEL */
|
|
|
|
|
2016-01-07 20:50:10 +01:00
|
|
|
extern struct xprt_class xprt_rdma_bc;
|
2012-02-15 18:30:00 +01:00
|
|
|
|
2007-09-10 19:50:12 +02:00
|
|
|
#endif /* _LINUX_SUNRPC_XPRT_RDMA_H */
|