[net-next,04/16] netfilter: nf_tables: prefer direct calls for set lookups

Message ID	20210601220629.18307-5-pablo@netfilter.org
State	New
Headers	show Return-Path: <netdev-owner@kernel.org> From: Pablo Neira Ayuso <pablo@netfilter.org> To: netfilter-devel@vger.kernel.org Cc: davem@davemloft.net, netdev@vger.kernel.org, kuba@kernel.org Subject: [PATCH net-next 04/16] netfilter: nf_tables: prefer direct calls for set lookups Date: Wed, 2 Jun 2021 00:06:17 +0200 Message-Id: <20210601220629.18307-5-pablo@netfilter.org> In-Reply-To: <20210601220629.18307-1-pablo@netfilter.org> References: <20210601220629.18307-1-pablo@netfilter.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Precedence: bulk
Series	Netfilter updates for net-next \| expand [net-next,00/16] Netfilter updates for net-next [net-next,01/16] netfilter: nft_exthdr: Support SCTP chunks [net-next,02/16] netfilter: nft_set_pipapo_avx2: Skip LDMXCSR, we don't need a valid MXCSR state [net-next,03/16] netfilter: add and use nft_set_do_lookup helper [net-next,04/16] netfilter: nf_tables: prefer direct calls for set lookups [net-next,05/16] netfilter: Remove leading spaces in Kconfig [net-next,06/16] netfilter: x_tables: improve limit_mt scalability [net-next,07/16] netfilter: xt_CT: Remove redundant assignment to ret [net-next,08/16] netfilter: use nfnetlink_unicast() [net-next,09/16] netfilter: x_tables: reduce xt_action_param by 8 byte [net-next,10/16] netfilter: reduce size of nf_hook_state on 32bit platforms [net-next,11/16] netfilter: nf_tables: add and use nft_sk helper [net-next,12/16] netfilter: nf_tables: add and use nft_thoff helper [net-next,13/16] netfilter: nf_tables: remove unused arg in nft_set_pktinfo_unspec() [net-next,14/16] netfilter: nf_tables: remove xt_action_param from nft_pktinfo [net-next,15/16] netfilter: nft_set_pipapo_avx2: fix up description warnings [net-next,16/16] netfilter: fix clang-12 fmt string warnings

diff --git a/include/net/netfilter/nf_tables_core.h b/include/net/netfilter/nf_tables_core.h index 5eb699454490..46c8d5bb5d8d 100644 --- a/include/net/netfilter/nf_tables_core.h +++ b/include/net/netfilter/nf_tables_core.h @@ -3,6 +3,7 @@ #define _NET_NF_TABLES_CORE_H #include <net/netfilter/nf_tables.h> +#include <linux/indirect_call_wrapper.h> extern struct nft_expr_type nft_imm_type; extern struct nft_expr_type nft_cmp_type; @@ -88,12 +89,35 @@ extern const struct nft_set_type nft_set_bitmap_type; extern const struct nft_set_type nft_set_pipapo_type; extern const struct nft_set_type nft_set_pipapo_avx2_type; +#ifdef CONFIG_RETPOLINE +bool nft_rhash_lookup(const struct net *net, const struct nft_set *set, + const u32 *key, const struct nft_set_ext **ext); +bool nft_rbtree_lookup(const struct net *net, const struct nft_set *set, + const u32 *key, const struct nft_set_ext **ext); +bool nft_bitmap_lookup(const struct net *net, const struct nft_set *set, + const u32 *key, const struct nft_set_ext **ext); +bool nft_hash_lookup_fast(const struct net *net, + const struct nft_set *set, + const u32 *key, const struct nft_set_ext **ext); +bool nft_hash_lookup(const struct net *net, const struct nft_set *set, + const u32 *key, const struct nft_set_ext **ext); +bool nft_set_do_lookup(const struct net *net, const struct nft_set *set, + const u32 *key, const struct nft_set_ext **ext); +#else static inline bool nft_set_do_lookup(const struct net *net, const struct nft_set *set, const u32 *key, const struct nft_set_ext **ext) { return set->ops->lookup(net, set, key, ext); } +#endif + +/* called from nft_pipapo_avx2.c */ +bool nft_pipapo_lookup(const struct net *net, const struct nft_set *set, + const u32 *key, const struct nft_set_ext **ext); +/* called from nft_set_pipapo.c */ +bool nft_pipapo_avx2_lookup(const struct net *net, const struct nft_set *set, + const u32 *key, const struct nft_set_ext **ext); struct nft_expr; struct nft_regs; diff --git a/net/netfilter/nft_lookup.c b/net/netfilter/nft_lookup.c index 1a8581879af5..90becbf5bff3 100644 --- a/net/netfilter/nft_lookup.c +++ b/net/netfilter/nft_lookup.c @@ -23,6 +23,37 @@ struct nft_lookup { struct nft_set_binding binding; }; +#ifdef CONFIG_RETPOLINE +bool nft_set_do_lookup(const struct net *net, const struct nft_set *set, + const u32 *key, const struct nft_set_ext **ext) +{ + if (set->ops == &nft_set_hash_fast_type.ops) + return nft_hash_lookup_fast(net, set, key, ext); + if (set->ops == &nft_set_hash_type.ops) + return nft_hash_lookup(net, set, key, ext); + + if (set->ops == &nft_set_rhash_type.ops) + return nft_rhash_lookup(net, set, key, ext); + + if (set->ops == &nft_set_bitmap_type.ops) + return nft_bitmap_lookup(net, set, key, ext); + + if (set->ops == &nft_set_pipapo_type.ops) + return nft_pipapo_lookup(net, set, key, ext); +#if defined(CONFIG_X86_64) && !defined(CONFIG_UML) + if (set->ops == &nft_set_pipapo_avx2_type.ops) + return nft_pipapo_avx2_lookup(net, set, key, ext); +#endif + + if (set->ops == &nft_set_rbtree_type.ops) + return nft_rbtree_lookup(net, set, key, ext); + + WARN_ON_ONCE(1); + return set->ops->lookup(net, set, key, ext); +} +EXPORT_SYMBOL_GPL(nft_set_do_lookup); +#endif + void nft_lookup_eval(const struct nft_expr *expr, struct nft_regs *regs, const struct nft_pktinfo *pkt) diff --git a/net/netfilter/nft_set_bitmap.c b/net/netfilter/nft_set_bitmap.c index 2a81ea421819..e7ae5914971e 100644 --- a/net/netfilter/nft_set_bitmap.c +++ b/net/netfilter/nft_set_bitmap.c @@ -73,8 +73,9 @@ nft_bitmap_active(const u8 *bitmap, u32 idx, u32 off, u8 genmask) return (bitmap[idx] & (0x3 << off)) & (genmask << off); } -static bool nft_bitmap_lookup(const struct net *net, const struct nft_set *set, - const u32 *key, const struct nft_set_ext **ext) +INDIRECT_CALLABLE_SCOPE +bool nft_bitmap_lookup(const struct net *net, const struct nft_set *set, + const u32 *key, const struct nft_set_ext **ext) { const struct nft_bitmap *priv = nft_set_priv(set); u8 genmask = nft_genmask_cur(net); diff --git a/net/netfilter/nft_set_hash.c b/net/netfilter/nft_set_hash.c index 7b3d0a78c569..df40314de21f 100644 --- a/net/netfilter/nft_set_hash.c +++ b/net/netfilter/nft_set_hash.c @@ -74,8 +74,9 @@ static const struct rhashtable_params nft_rhash_params = { .automatic_shrinking = true, }; -static bool nft_rhash_lookup(const struct net *net, const struct nft_set *set, - const u32 *key, const struct nft_set_ext **ext) +INDIRECT_CALLABLE_SCOPE +bool nft_rhash_lookup(const struct net *net, const struct nft_set *set, + const u32 *key, const struct nft_set_ext **ext) { struct nft_rhash *priv = nft_set_priv(set); const struct nft_rhash_elem *he; @@ -446,8 +447,9 @@ struct nft_hash_elem { struct nft_set_ext ext; }; -static bool nft_hash_lookup(const struct net *net, const struct nft_set *set, - const u32 *key, const struct nft_set_ext **ext) +INDIRECT_CALLABLE_SCOPE +bool nft_hash_lookup(const struct net *net, const struct nft_set *set, + const u32 *key, const struct nft_set_ext **ext) { struct nft_hash *priv = nft_set_priv(set); u8 genmask = nft_genmask_cur(net); @@ -484,9 +486,10 @@ static void *nft_hash_get(const struct net *net, const struct nft_set *set, return ERR_PTR(-ENOENT); } -static bool nft_hash_lookup_fast(const struct net *net, - const struct nft_set *set, - const u32 *key, const struct nft_set_ext **ext) +INDIRECT_CALLABLE_SCOPE +bool nft_hash_lookup_fast(const struct net *net, + const struct nft_set *set, + const u32 *key, const struct nft_set_ext **ext) { struct nft_hash *priv = nft_set_priv(set); u8 genmask = nft_genmask_cur(net); diff --git a/net/netfilter/nft_set_pipapo.h b/net/netfilter/nft_set_pipapo.h index d84afb8fa79a..25a75591583e 100644 --- a/net/netfilter/nft_set_pipapo.h +++ b/net/netfilter/nft_set_pipapo.h @@ -178,8 +178,6 @@ struct nft_pipapo_elem { int pipapo_refill(unsigned long *map, int len, int rules, unsigned long *dst, union nft_pipapo_map_bucket *mt, bool match_only); -bool nft_pipapo_lookup(const struct net *net, const struct nft_set *set, - const u32 *key, const struct nft_set_ext **ext); /** * pipapo_and_field_buckets_4bit() - Intersect 4-bit buckets diff --git a/net/netfilter/nft_set_pipapo_avx2.h b/net/netfilter/nft_set_pipapo_avx2.h index 394bcb704db7..dbb6aaca8a7a 100644 --- a/net/netfilter/nft_set_pipapo_avx2.h +++ b/net/netfilter/nft_set_pipapo_avx2.h @@ -5,8 +5,6 @@ #include <asm/fpu/xstate.h> #define NFT_PIPAPO_ALIGN (XSAVE_YMM_SIZE / BITS_PER_BYTE) -bool nft_pipapo_avx2_lookup(const struct net *net, const struct nft_set *set, - const u32 *key, const struct nft_set_ext **ext); bool nft_pipapo_avx2_estimate(const struct nft_set_desc *desc, u32 features, struct nft_set_estimate *est); #endif /* defined(CONFIG_X86_64) && !defined(CONFIG_UML) */ diff --git a/net/netfilter/nft_set_rbtree.c b/net/netfilter/nft_set_rbtree.c index 9e36eb4a7429..d600a566da32 100644 --- a/net/netfilter/nft_set_rbtree.c +++ b/net/netfilter/nft_set_rbtree.c @@ -107,8 +107,9 @@ static bool __nft_rbtree_lookup(const struct net *net, const struct nft_set *set return false; } -static bool nft_rbtree_lookup(const struct net *net, const struct nft_set *set, - const u32 *key, const struct nft_set_ext **ext) +INDIRECT_CALLABLE_SCOPE +bool nft_rbtree_lookup(const struct net *net, const struct nft_set *set, + const u32 *key, const struct nft_set_ext **ext) { struct nft_rbtree *priv = nft_set_priv(set); unsigned int seq = read_seqcount_begin(&priv->count);

[net-next,04/16] netfilter: nf_tables: prefer direct calls for set lookups

Commit Message

Patch