ipv6: factorize sk_wmem_alloc updates done by __ip6_append_data()
While testing my inet defrag changes, I found that the senders could spend ~20% of cpu cycles in skb_set_owner_w() updating sk->sk_wmem_alloc for every fragment they cook, competing with TX completion of prior skbs possibly happening on another cpus. The solution to this problem is to use alloc_skb() instead of sock_wmalloc() and manually perform a single sk_wmem_alloc change. Signed-off-by: Eric Dumazet <edumazet@google.com> Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
parent
694aba690d
commit
1f4c6eb240
|
@ -1259,6 +1259,7 @@ static int __ip6_append_data(struct sock *sk,
|
||||||
struct ipv6_txoptions *opt = v6_cork->opt;
|
struct ipv6_txoptions *opt = v6_cork->opt;
|
||||||
int csummode = CHECKSUM_NONE;
|
int csummode = CHECKSUM_NONE;
|
||||||
unsigned int maxnonfragsize, headersize;
|
unsigned int maxnonfragsize, headersize;
|
||||||
|
unsigned int wmem_alloc_delta = 0;
|
||||||
|
|
||||||
skb = skb_peek_tail(queue);
|
skb = skb_peek_tail(queue);
|
||||||
if (!skb) {
|
if (!skb) {
|
||||||
|
@ -1411,11 +1412,10 @@ alloc_new_skb:
|
||||||
(flags & MSG_DONTWAIT), &err);
|
(flags & MSG_DONTWAIT), &err);
|
||||||
} else {
|
} else {
|
||||||
skb = NULL;
|
skb = NULL;
|
||||||
if (refcount_read(&sk->sk_wmem_alloc) <=
|
if (refcount_read(&sk->sk_wmem_alloc) + wmem_alloc_delta <=
|
||||||
2 * sk->sk_sndbuf)
|
2 * sk->sk_sndbuf)
|
||||||
skb = sock_wmalloc(sk,
|
skb = alloc_skb(alloclen + hh_len,
|
||||||
alloclen + hh_len, 1,
|
sk->sk_allocation);
|
||||||
sk->sk_allocation);
|
|
||||||
if (unlikely(!skb))
|
if (unlikely(!skb))
|
||||||
err = -ENOBUFS;
|
err = -ENOBUFS;
|
||||||
}
|
}
|
||||||
|
@ -1474,6 +1474,11 @@ alloc_new_skb:
|
||||||
/*
|
/*
|
||||||
* Put the packet on the pending queue
|
* Put the packet on the pending queue
|
||||||
*/
|
*/
|
||||||
|
if (!skb->destructor) {
|
||||||
|
skb->destructor = sock_wfree;
|
||||||
|
skb->sk = sk;
|
||||||
|
wmem_alloc_delta += skb->truesize;
|
||||||
|
}
|
||||||
__skb_queue_tail(queue, skb);
|
__skb_queue_tail(queue, skb);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
@ -1520,12 +1525,13 @@ alloc_new_skb:
|
||||||
skb->len += copy;
|
skb->len += copy;
|
||||||
skb->data_len += copy;
|
skb->data_len += copy;
|
||||||
skb->truesize += copy;
|
skb->truesize += copy;
|
||||||
refcount_add(copy, &sk->sk_wmem_alloc);
|
wmem_alloc_delta += copy;
|
||||||
}
|
}
|
||||||
offset += copy;
|
offset += copy;
|
||||||
length -= copy;
|
length -= copy;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
error_efault:
|
error_efault:
|
||||||
|
@ -1533,6 +1539,7 @@ error_efault:
|
||||||
error:
|
error:
|
||||||
cork->length -= length;
|
cork->length -= length;
|
||||||
IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS);
|
IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS);
|
||||||
|
refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
|
||||||
return err;
|
return err;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue