netfilter: use jump_label for nf_hooks
On configs where CONFIG_JUMP_LABEL=y, we can replace in fast path a load/compare/conditional jump by a single jump with no dcache reference. Jump target is modified as soon as nf_hooks[pf][hook] switches from empty state to non empty states. jump_label state is kept outside of nf_hooks array so has no cost on cpu caches. This patch removes the test on CONFIG_NETFILTER_DEBUG : No need to call nf_hook_slow() at all if nf_hooks[pf][hook] is empty, this didnt give useful information, but slowed down things a lot. Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com> CC: Patrick McHardy <kaber@trash.net> CC: Pablo Neira Ayuso <pablo@netfilter.org> Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
parent
9205fd9cca
commit
a2d7ec58ac
|
@ -162,6 +162,24 @@ extern struct ctl_path nf_net_ipv4_netfilter_sysctl_path[];
|
||||||
|
|
||||||
extern struct list_head nf_hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
|
extern struct list_head nf_hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
|
||||||
|
|
||||||
|
#if defined(CONFIG_JUMP_LABEL)
|
||||||
|
#include <linux/jump_label.h>
|
||||||
|
extern struct jump_label_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
|
||||||
|
static inline bool nf_hooks_active(u_int8_t pf, unsigned int hook)
|
||||||
|
{
|
||||||
|
if (__builtin_constant_p(pf) &&
|
||||||
|
__builtin_constant_p(hook))
|
||||||
|
return static_branch(&nf_hooks_needed[pf][hook]);
|
||||||
|
|
||||||
|
return !list_empty(&nf_hooks[pf][hook]);
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
static inline bool nf_hooks_active(u_int8_t pf, unsigned int hook)
|
||||||
|
{
|
||||||
|
return !list_empty(&nf_hooks[pf][hook]);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
int nf_hook_slow(u_int8_t pf, unsigned int hook, struct sk_buff *skb,
|
int nf_hook_slow(u_int8_t pf, unsigned int hook, struct sk_buff *skb,
|
||||||
struct net_device *indev, struct net_device *outdev,
|
struct net_device *indev, struct net_device *outdev,
|
||||||
int (*okfn)(struct sk_buff *), int thresh);
|
int (*okfn)(struct sk_buff *), int thresh);
|
||||||
|
@ -179,11 +197,9 @@ static inline int nf_hook_thresh(u_int8_t pf, unsigned int hook,
|
||||||
struct net_device *outdev,
|
struct net_device *outdev,
|
||||||
int (*okfn)(struct sk_buff *), int thresh)
|
int (*okfn)(struct sk_buff *), int thresh)
|
||||||
{
|
{
|
||||||
#ifndef CONFIG_NETFILTER_DEBUG
|
if (nf_hooks_active(pf, hook))
|
||||||
if (list_empty(&nf_hooks[pf][hook]))
|
return nf_hook_slow(pf, hook, skb, indev, outdev, okfn, thresh);
|
||||||
return 1;
|
return 1;
|
||||||
#endif
|
|
||||||
return nf_hook_slow(pf, hook, skb, indev, outdev, okfn, thresh);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline int nf_hook(u_int8_t pf, unsigned int hook, struct sk_buff *skb,
|
static inline int nf_hook(u_int8_t pf, unsigned int hook, struct sk_buff *skb,
|
||||||
|
|
|
@ -54,6 +54,12 @@ EXPORT_SYMBOL_GPL(nf_unregister_afinfo);
|
||||||
|
|
||||||
struct list_head nf_hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS] __read_mostly;
|
struct list_head nf_hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS] __read_mostly;
|
||||||
EXPORT_SYMBOL(nf_hooks);
|
EXPORT_SYMBOL(nf_hooks);
|
||||||
|
|
||||||
|
#if defined(CONFIG_JUMP_LABEL)
|
||||||
|
struct jump_label_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
|
||||||
|
EXPORT_SYMBOL(nf_hooks_needed);
|
||||||
|
#endif
|
||||||
|
|
||||||
static DEFINE_MUTEX(nf_hook_mutex);
|
static DEFINE_MUTEX(nf_hook_mutex);
|
||||||
|
|
||||||
int nf_register_hook(struct nf_hook_ops *reg)
|
int nf_register_hook(struct nf_hook_ops *reg)
|
||||||
|
@ -70,6 +76,9 @@ int nf_register_hook(struct nf_hook_ops *reg)
|
||||||
}
|
}
|
||||||
list_add_rcu(®->list, elem->list.prev);
|
list_add_rcu(®->list, elem->list.prev);
|
||||||
mutex_unlock(&nf_hook_mutex);
|
mutex_unlock(&nf_hook_mutex);
|
||||||
|
#if defined(CONFIG_JUMP_LABEL)
|
||||||
|
jump_label_inc(&nf_hooks_needed[reg->pf][reg->hooknum]);
|
||||||
|
#endif
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL(nf_register_hook);
|
EXPORT_SYMBOL(nf_register_hook);
|
||||||
|
@ -79,7 +88,9 @@ void nf_unregister_hook(struct nf_hook_ops *reg)
|
||||||
mutex_lock(&nf_hook_mutex);
|
mutex_lock(&nf_hook_mutex);
|
||||||
list_del_rcu(®->list);
|
list_del_rcu(®->list);
|
||||||
mutex_unlock(&nf_hook_mutex);
|
mutex_unlock(&nf_hook_mutex);
|
||||||
|
#if defined(CONFIG_JUMP_LABEL)
|
||||||
|
jump_label_dec(&nf_hooks_needed[reg->pf][reg->hooknum]);
|
||||||
|
#endif
|
||||||
synchronize_net();
|
synchronize_net();
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL(nf_unregister_hook);
|
EXPORT_SYMBOL(nf_unregister_hook);
|
||||||
|
|
Loading…
Reference in New Issue