net: use indirect call wrappers for skb_copy_datagram_iter()
commit29f3490ba9
upstream. TCP recvmsg() calls skb_copy_datagram_iter(), which calls an indirect function (cb pointing to simple_copy_to_iter()) for every MSS (fragment) present in the skb. CONFIG_RETPOLINE=y forces a very expensive operation that we can avoid thanks to indirect call wrappers. This patch gives a 13% increase of performance on a single flow, if the bottleneck is the thread reading the TCP socket. Fixes:950fcaecd5
("datagram: consolidate datagram copy to iter helpers") Signed-off-by: Eric Dumazet <edumazet@google.com> Acked-by: Paolo Abeni <pabeni@redhat.com> Acked-by: Willem de Bruijn <willemb@google.com> Signed-off-by: David S. Miller <davem@davemloft.net> Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
This commit is contained in:
parent
ec9cf8afcd
commit
f161048060
|
@ -51,6 +51,7 @@
|
|||
#include <linux/slab.h>
|
||||
#include <linux/pagemap.h>
|
||||
#include <linux/uio.h>
|
||||
#include <linux/indirect_call_wrapper.h>
|
||||
|
||||
#include <net/protocol.h>
|
||||
#include <linux/skbuff.h>
|
||||
|
@ -407,6 +408,11 @@ int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags)
|
|||
}
|
||||
EXPORT_SYMBOL(skb_kill_datagram);
|
||||
|
||||
INDIRECT_CALLABLE_DECLARE(static size_t simple_copy_to_iter(const void *addr,
|
||||
size_t bytes,
|
||||
void *data __always_unused,
|
||||
struct iov_iter *i));
|
||||
|
||||
static int __skb_datagram_iter(const struct sk_buff *skb, int offset,
|
||||
struct iov_iter *to, int len, bool fault_short,
|
||||
size_t (*cb)(const void *, size_t, void *,
|
||||
|
@ -420,7 +426,8 @@ static int __skb_datagram_iter(const struct sk_buff *skb, int offset,
|
|||
if (copy > 0) {
|
||||
if (copy > len)
|
||||
copy = len;
|
||||
n = cb(skb->data + offset, copy, data, to);
|
||||
n = INDIRECT_CALL_1(cb, simple_copy_to_iter,
|
||||
skb->data + offset, copy, data, to);
|
||||
offset += n;
|
||||
if (n != copy)
|
||||
goto short_copy;
|
||||
|
@ -442,8 +449,9 @@ static int __skb_datagram_iter(const struct sk_buff *skb, int offset,
|
|||
|
||||
if (copy > len)
|
||||
copy = len;
|
||||
n = cb(vaddr + skb_frag_off(frag) + offset - start,
|
||||
copy, data, to);
|
||||
n = INDIRECT_CALL_1(cb, simple_copy_to_iter,
|
||||
vaddr + skb_frag_off(frag) + offset - start,
|
||||
copy, data, to);
|
||||
kunmap(page);
|
||||
offset += n;
|
||||
if (n != copy)
|
||||
|
|
Loading…
Reference in New Issue