archive/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2025-08-15 14:11:42 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

netfilter: nft_set_pipapo: merge pipapo_get/lookup

Browse source 
The matching algorithm has implemented thrice:
1. data path lookup, generic version
2. data path lookup, avx2 version
3. control plane lookup

Merge 1 and 3 by refactoring pipapo_get as a common helper, then make
nft_pipapo_lookup and nft_pipapo_get both call the common helper.

Aside from the code savings this has the benefit that we no longer allocate
temporary scratch maps for each control plane get and insertion operation.

Signed-off-by: Florian Westphal <fw@strlen.de>
Reviewed-by: Stefano Brivio <sbrivio@redhat.com>
Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>
This commit is contained in:
Florian Westphal 2025-07-09 19:05:15 +02:00 • committed by Pablo Neira Ayuso
parent 531e613121
commit d8d871a35c
1 changed files with 58 additions and 130 deletions
Download patch file Download diff file Expand all files Collapse all files

186
net/netfilter/nft_set_pipapo.c
View file

@ -397,35 +397,36 @@ int pipapo_refill(unsigned long *map, unsigned int len, unsigned int rules,
} }
/** /**
* nft_pipapo_lookup() - Lookup function * pipapo_get() - Get matching element reference given key data
* @net: Network namespace * @m: storage containing the set elements
* @set: nftables API set representation * @data: Key data to be matched against existing elements
* @key: nftables API element representation containing key data * @genmask: If set, check that element is active in given genmask
* @ext: nftables API extension pointer, filled with matching reference * @tstamp: timestamp to check for expired elements
* *
* For more details, see DOC: Theory of Operation. * For more details, see DOC: Theory of Operation.
* *
* Return: true on match, false otherwise. * This is the main lookup function. It matches key data against either
* the working match set or the uncommitted copy, depending on what the
* caller passed to us.
* nft_pipapo_get (lookup from userspace/control plane) and nft_pipapo_lookup
* (datapath lookup) pass the active copy.
* The insertion path will pass the uncommitted working copy.
*
* Return: pointer to &struct nft_pipapo_elem on match, NULL otherwise.
*/ */
const struct nft_set_ext * static struct nft_pipapo_elem *pipapo_get(const struct nft_pipapo_match *m,
nft_pipapo_lookup(const struct net *net, const struct nft_set *set, const u8 *data, u8 genmask,
const u32 *key) u64 tstamp)
{ {
struct nft_pipapo *priv = nft_set_priv(set);
struct nft_pipapo_scratch *scratch; struct nft_pipapo_scratch *scratch;
unsigned long *res_map, *fill_map; unsigned long *res_map, *fill_map;
u8 genmask = nft_genmask_cur(net);
const struct nft_pipapo_match *m;
const struct nft_pipapo_field *f; const struct nft_pipapo_field *f;
const u8 *rp = (const u8 *)key;
bool map_index; bool map_index;
int i; int i;
local_bh_disable(); local_bh_disable();
m = rcu_dereference(priv->match); if (unlikely(!raw_cpu_ptr(m->scratch)))
if (unlikely(!m || !*raw_cpu_ptr(m->scratch)))
goto out; goto out;
scratch = *raw_cpu_ptr(m->scratch); scratch = *raw_cpu_ptr(m->scratch);
@ -445,12 +446,12 @@ nft_pipapo_lookup(const struct net *net, const struct nft_set *set,
* packet bytes value, then AND bucket value * packet bytes value, then AND bucket value
*/ */
if (likely(f->bb == 8)) if (likely(f->bb == 8))
pipapo_and_field_buckets_8bit(f, res_map, rp); pipapo_and_field_buckets_8bit(f, res_map, data);
else else
pipapo_and_field_buckets_4bit(f, res_map, rp); pipapo_and_field_buckets_4bit(f, res_map, data);
NFT_PIPAPO_GROUP_BITS_ARE_8_OR_4; NFT_PIPAPO_GROUP_BITS_ARE_8_OR_4;
rp += f->groups / NFT_PIPAPO_GROUPS_PER_BYTE(f); data += f->groups / NFT_PIPAPO_GROUPS_PER_BYTE(f);
/* Now populate the bitmap for the next field, unless this is /* Now populate the bitmap for the next field, unless this is
* the last field, in which case return the matched 'ext' * the last field, in which case return the matched 'ext'
@ -470,11 +471,11 @@ next_match:
} }
if (last) { if (last) {
const struct nft_set_ext *ext; struct nft_pipapo_elem *e;
ext = &f->mt[b].e->ext; e = f->mt[b].e;
if (unlikely(nft_set_elem_expired(ext) || if (unlikely(__nft_set_elem_expired(&e->ext, tstamp) ||
!nft_set_elem_active(ext, genmask))) !nft_set_elem_active(&e->ext, genmask)))
goto next_match; goto next_match;
/* Last field: we're just returning the key without /* Last field: we're just returning the key without
@ -484,8 +485,7 @@ next_match:
*/ */
scratch->map_index = map_index; scratch->map_index = map_index;
local_bh_enable(); local_bh_enable();
return e;
return ext;
} }
/* Swap bitmap indices: res_map is the initial bitmap for the /* Swap bitmap indices: res_map is the initial bitmap for the
@ -495,7 +495,7 @@ next_match:
map_index = !map_index; map_index = !map_index;
swap(res_map, fill_map); swap(res_map, fill_map);
rp += NFT_PIPAPO_GROUPS_PADDING(f); data += NFT_PIPAPO_GROUPS_PADDING(f);
} }
out: out:
@ -504,99 +504,29 @@ out:
} }
/** /**
* pipapo_get() - Get matching element reference given key data * nft_pipapo_lookup() - Dataplane fronted for main lookup function
* @m: storage containing active/existing elements * @net: Network namespace
* @data: Key data to be matched against existing elements * @set: nftables API set representation
* @genmask: If set, check that element is active in given genmask * @key: pointer to nft registers containing key data
* @tstamp: timestamp to check for expired elements
* @gfp: the type of memory to allocate (see kmalloc).
* *
* This is essentially the same as the lookup function, except that it matches * This function is called from the data path. It will search for
* key data against the uncommitted copy and doesn't use preallocated maps for * an element matching the given key in the current active copy.
* bitmap results.
* *
* Return: pointer to &struct nft_pipapo_elem on match, error pointer otherwise. * Return: ntables API extension pointer or NULL if no match.
*/ */
static struct nft_pipapo_elem *pipapo_get(const struct nft_pipapo_match *m, const struct nft_set_ext *
const u8 *data, u8 genmask, nft_pipapo_lookup(const struct net *net, const struct nft_set *set,
u64 tstamp, gfp_t gfp) const u32 *key)
{ {
struct nft_pipapo_elem *ret = ERR_PTR(-ENOENT); struct nft_pipapo *priv = nft_set_priv(set);
unsigned long *res_map, *fill_map = NULL; u8 genmask = nft_genmask_cur(net);
const struct nft_pipapo_field *f; const struct nft_pipapo_match *m;
int i; const struct nft_pipapo_elem *e;
if (m->bsize_max == 0) m = rcu_dereference(priv->match);
return ret; e = pipapo_get(m, (const u8 *)key, genmask, get_jiffies_64());
res_map = kmalloc_array(m->bsize_max, sizeof(*res_map), gfp); return e ? &e->ext : NULL;
if (!res_map) {
ret = ERR_PTR(-ENOMEM);
goto out;
}
fill_map = kcalloc(m->bsize_max, sizeof(*res_map), gfp);
if (!fill_map) {
ret = ERR_PTR(-ENOMEM);
goto out;
}
pipapo_resmap_init(m, res_map);
nft_pipapo_for_each_field(f, i, m) {
bool last = i == m->field_count - 1;
int b;
/* For each bit group: select lookup table bucket depending on
* packet bytes value, then AND bucket value
*/
if (f->bb == 8)
pipapo_and_field_buckets_8bit(f, res_map, data);
else if (f->bb == 4)
pipapo_and_field_buckets_4bit(f, res_map, data);
else
BUG();
data += f->groups / NFT_PIPAPO_GROUPS_PER_BYTE(f);
/* Now populate the bitmap for the next field, unless this is
* the last field, in which case return the matched 'ext'
* pointer if any.
*
* Now res_map contains the matching bitmap, and fill_map is the
* bitmap for the next field.
*/
next_match:
b = pipapo_refill(res_map, f->bsize, f->rules, fill_map, f->mt,
last);
if (b < 0)
goto out;
if (last) {
if (__nft_set_elem_expired(&f->mt[b].e->ext, tstamp))
goto next_match;
if ((genmask &&
!nft_set_elem_active(&f->mt[b].e->ext, genmask)))
goto next_match;
ret = f->mt[b].e;
goto out;
}
data += NFT_PIPAPO_GROUPS_PADDING(f);
/* Swap bitmap indices: fill_map will be the initial bitmap for
* the next field (i.e. the new res_map), and res_map is
* guaranteed to be all-zeroes at this point, ready to be filled
* according to the next mapping table.
*/
swap(res_map, fill_map);
}
out:
kfree(fill_map);
kfree(res_map);
return ret;
} }
/** /**
@ -605,6 +535,11 @@ out:
* @set: nftables API set representation * @set: nftables API set representation
* @elem: nftables API element representation containing key data * @elem: nftables API element representation containing key data
* @flags: Unused * @flags: Unused
*
* This function is called from the control plane path under
* RCU read lock.
*
* Return: set element private pointer or ERR_PTR(-ENOENT).
*/ */
static struct nft_elem_priv * static struct nft_elem_priv *
nft_pipapo_get(const struct net *net, const struct nft_set *set, nft_pipapo_get(const struct net *net, const struct nft_set *set,
@ -615,10 +550,9 @@ nft_pipapo_get(const struct net *net, const struct nft_set *set,
struct nft_pipapo_elem *e; struct nft_pipapo_elem *e;
e = pipapo_get(m, (const u8 *)elem->key.val.data, e = pipapo_get(m, (const u8 *)elem->key.val.data,
nft_genmask_cur(net), get_jiffies_64(), nft_genmask_cur(net), get_jiffies_64());
GFP_ATOMIC); if (!e)
if (IS_ERR(e)) return ERR_PTR(-ENOENT);
return ERR_CAST(e);
return &e->priv; return &e->priv;
} }
@ -1344,8 +1278,8 @@ static int nft_pipapo_insert(const struct net *net, const struct nft_set *set,
else else
end = start; end = start;
dup = pipapo_get(m, start, genmask, tstamp, GFP_KERNEL); dup = pipapo_get(m, start, genmask, tstamp);
if (!IS_ERR(dup)) { if (dup) {
/* Check if we already have the same exact entry */ /* Check if we already have the same exact entry */
const struct nft_data *dup_key, *dup_end; const struct nft_data *dup_key, *dup_end;
@ -1364,15 +1298,9 @@ static int nft_pipapo_insert(const struct net *net, const struct nft_set *set,
return -ENOTEMPTY; return -ENOTEMPTY;
} }
if (PTR_ERR(dup) == -ENOENT) {
/* Look for partially overlapping entries */ /* Look for partially overlapping entries */
dup = pipapo_get(m, end, nft_genmask_next(net), tstamp, dup = pipapo_get(m, end, nft_genmask_next(net), tstamp);
GFP_KERNEL); if (dup) {
}
if (PTR_ERR(dup) != -ENOENT) {
if (IS_ERR(dup))
return PTR_ERR(dup);
*elem_priv = &dup->priv; *elem_priv = &dup->priv;
return -ENOTEMPTY; return -ENOTEMPTY;
} }
@ -1914,8 +1842,8 @@ nft_pipapo_deactivate(const struct net *net, const struct nft_set *set,
return NULL; return NULL;
e = pipapo_get(m, (const u8 *)elem->key.val.data, e = pipapo_get(m, (const u8 *)elem->key.val.data,
nft_genmask_next(net), nft_net_tstamp(net), GFP_KERNEL); nft_genmask_next(net), nft_net_tstamp(net));
if (IS_ERR(e)) if (!e)
return NULL; return NULL;
nft_set_elem_change_active(net, set, &e->ext); nft_set_elem_change_active(net, set, &e->ext);