archive/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2025-08-15 14:11:42 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

bpf-next-6.17

Browse source 
-----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCAAdFiEE+soXsSLHKoYyzcli6rmadz2vbToFAmiINnEACgkQ6rmadz2v
 bToBnA/9F+A3R6rTwGk4HK3xpfc/nm2Tanl3oRN7S2ub/mskDOtWSIyG6cVFZ0UG
 1fK6IkByyRIpAF/5qhdlw8drRXHkQtGLA0lP2L9llm4X1mHLofB18y9OeLrDE1WN
 KwNP06+IGX9W802lCGSIXOY+VmRscVfXSMokyQt2ilHplKjOnDqJcYkWupi3T2rC
 mz79FY9aEl2YrIcpj9RXz+8nwP49pZBuW2P0IM5PAIj4BJBXShrUp8T1nz94okNe
 NFsnAyRxjWpUT0McEgtA9WvpD9lZqujYD8Qp0KlGZWmI3vNpV5d9S1+dBcEb1n7q
 dyNMkTF3oRrJhhg4VqoHc6fVpzSEoZ9ZxV5Hx4cs+ganH75D4YbdGqx/7mR3DUgH
 MZh6rHF1pGnK7TAm7h5gl3ZRAOkZOaahbe1i01NKo9CEe5fSh3AqMyzJYoyGHRKi
 xDN39eQdWBNA+hm1VkbK2Bv93Rbjrka2Kj+D3sSSO9Bo/u3ntcknr7LW39idKz62
 Q8dkKHcCEtun7gjk0YXPF013y81nEohj1C+52BmJ2l5JitM57xfr6YOaQpu7DPDE
 AJbHx6ASxKdyEETecd0b+cXUPQ349zmRXy0+CDMAGKpBicC0H0mHhL14cwOY1Hfu
 EIpIjmIJGI3JNF6T5kybcQGSBOYebdV0FFgwSllzPvuYt7YsHCs=
 =/O3j
 -----END PGP SIGNATURE-----

Merge tag 'bpf-next-6.17' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Pull bpf updates from Alexei Starovoitov:

 - Remove usermode driver (UMD) framework (Thomas Weißschuh)

 - Introduce Strongly Connected Component (SCC) in the verifier to
   detect loops and refine register liveness (Eduard Zingerman)

 - Allow 'void *' cast using bpf_rdonly_cast() and corresponding
   '__arg_untrusted' for global function parameters (Eduard Zingerman)

 - Improve precision for BPF_ADD and BPF_SUB operations in the verifier
   (Harishankar Vishwanathan)

 - Teach the verifier that constant pointer to a map cannot be NULL
   (Ihor Solodrai)

 - Introduce BPF streams for error reporting of various conditions
   detected by BPF runtime (Kumar Kartikeya Dwivedi)

 - Teach the verifier to insert runtime speculation barrier (lfence on
   x86) to mitigate speculative execution instead of rejecting the
   programs (Luis Gerhorst)

 - Various improvements for 'veristat' (Mykyta Yatsenko)

 - For CONFIG_DEBUG_KERNEL config warn on internal verifier errors to
   improve bug detection by syzbot (Paul Chaignon)

 - Support BPF private stack on arm64 (Puranjay Mohan)

 - Introduce bpf_cgroup_read_xattr() kfunc to read xattr of cgroup's
   node (Song Liu)

 - Introduce kfuncs for read-only string opreations (Viktor Malik)

 - Implement show_fdinfo() for bpf_links (Tao Chen)

 - Reduce verifier's stack consumption (Yonghong Song)

 - Implement mprog API for cgroup-bpf programs (Yonghong Song)

* tag 'bpf-next-6.17' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next: (192 commits)
  selftests/bpf: Migrate fexit_noreturns case into tracing_failure test suite
  selftests/bpf: Add selftest for attaching tracing programs to functions in deny list
  bpf: Add log for attaching tracing programs to functions in deny list
  bpf: Show precise rejected function when attaching fexit/fmod_ret to __noreturn functions
  bpf: Fix various typos in verifier.c comments
  bpf: Add third round of bounds deduction
  selftests/bpf: Test invariants on JSLT crossing sign
  selftests/bpf: Test cross-sign 64bits range refinement
  selftests/bpf: Update reg_bound range refinement logic
  bpf: Improve bounds when s64 crosses sign boundary
  bpf: Simplify bounds refinement from s32
  selftests/bpf: Enable private stack tests for arm64
  bpf, arm64: JIT support for private stack
  bpf: Move bpf_jit_get_prog_name() to core.c
  bpf, arm64: Fix fp initialization for exception boundary
  umd: Remove usermode driver framework
  bpf/preload: Don't select USERMODE_DRIVER
  selftests/bpf: Fix test dynptr/test_dynptr_memset_xdp_chunks failure
  selftests/bpf: Fix test dynptr/test_dynptr_copy_xdp failure
  selftests/bpf: Increase xdp data size for arm64 64K page size
  ...
This commit is contained in:
Linus Torvalds 2025-07-30 09:58:50 -07:00
commit d9104cec3e
178 changed files with 9775 additions and 2293 deletions
Download patch file Download diff file Expand all files Collapse all files

7
Documentation/bpf/bpf_devel_QA.rst
View file

@ -611,9 +611,10 @@ Q: I have added a new BPF instruction to the kernel, how can I integrate
it into LLVM?
A: LLVM has a ``-mcpu`` selector for the BPF back end in order to allow
the selection of BPF instruction set extensions. By default the
``generic`` processor target is used, which is the base instruction set
(v1) of BPF.
the selection of BPF instruction set extensions. Before llvm version 20,
the ``generic`` processor target is used, which is the base instruction
set (v1) of BPF. Since llvm 20, the default processor target has changed
to instruction set v3.
LLVM has an option to select ``-mcpu=probe`` where it will probe the host
kernel for supported BPF instruction set extensions and selects the

6
Documentation/bpf/standardization/instruction-set.rst
View file

@ -350,9 +350,9 @@ Underflow and overflow are allowed during arithmetic operations, meaning
the 64-bit or 32-bit value will wrap. If BPF program execution would
result in division by zero, the destination register is instead set to zero.
Otherwise, for ``ALU64``, if execution would result in ``LLONG_MIN``
dividing -1, the desination register is instead set to ``LLONG_MIN``. For
``ALU``, if execution would result in ``INT_MIN`` dividing -1, the
desination register is instead set to ``INT_MIN``.
divided by -1, the destination register is instead set to ``LLONG_MIN``. For
``ALU``, if execution would result in ``INT_MIN`` divided by -1, the
destination register is instead set to ``INT_MIN``.
If execution would result in modulo by zero, for ``ALU64`` the value of
the destination register is unchanged whereas for ``ALU`` the upper

5
arch/arm64/net/bpf_jit.h
View file

@ -325,4 +325,9 @@
#define A64_MRS_SP_EL0(Rt) \
aarch64_insn_gen_mrs(Rt, AARCH64_INSN_SYSREG_SP_EL0)
/* Barriers */
#define A64_SB aarch64_insn_get_sb_value()
#define A64_DSB_NSH (aarch64_insn_get_dsb_base_value() | 0x7 << 8)
#define A64_ISB aarch64_insn_get_isb_value()
#endif /* _BPF_JIT_H */

167
arch/arm64/net/bpf_jit_comp.c
View file

@ -30,6 +30,7 @@
#define TMP_REG_2 (MAX_BPF_JIT_REG + 1)
#define TCCNT_PTR (MAX_BPF_JIT_REG + 2)
#define TMP_REG_3 (MAX_BPF_JIT_REG + 3)
#define PRIVATE_SP (MAX_BPF_JIT_REG + 4)
#define ARENA_VM_START (MAX_BPF_JIT_REG + 5)
#define check_imm(bits, imm) do { \
@ -68,6 +69,8 @@ static const int bpf2a64[] = {
[TCCNT_PTR] = A64_R(26),
/* temporary register for blinding constants */
[BPF_REG_AX] = A64_R(9),
/* callee saved register for private stack pointer */
[PRIVATE_SP] = A64_R(27),
/* callee saved register for kern_vm_start address */
[ARENA_VM_START] = A64_R(28),
};
@ -86,6 +89,7 @@ struct jit_ctx {
u64 user_vm_start;
u64 arena_vm_start;
bool fp_used;
bool priv_sp_used;
bool write;
};
@ -98,6 +102,10 @@ struct bpf_plt {
#define PLT_TARGET_SIZE sizeof_field(struct bpf_plt, target)
#define PLT_TARGET_OFFSET offsetof(struct bpf_plt, target)
/* Memory size/value to protect private stack overflow/underflow */
#define PRIV_STACK_GUARD_SZ 16
#define PRIV_STACK_GUARD_VAL 0xEB9F12345678eb9fULL
static inline void emit(const u32 insn, struct jit_ctx *ctx)
{
if (ctx->image != NULL && ctx->write)
@ -387,8 +395,11 @@ static void find_used_callee_regs(struct jit_ctx *ctx)
if (reg_used & 8)
ctx->used_callee_reg[i++] = bpf2a64[BPF_REG_9];
if (reg_used & 16)
if (reg_used & 16) {
ctx->used_callee_reg[i++] = bpf2a64[BPF_REG_FP];
if (ctx->priv_sp_used)
ctx->used_callee_reg[i++] = bpf2a64[PRIVATE_SP];
}
if (ctx->arena_vm_start)
ctx->used_callee_reg[i++] = bpf2a64[ARENA_VM_START];
@ -412,6 +423,7 @@ static void push_callee_regs(struct jit_ctx *ctx)
emit(A64_PUSH(A64_R(23), A64_R(24), A64_SP), ctx);
emit(A64_PUSH(A64_R(25), A64_R(26), A64_SP), ctx);
emit(A64_PUSH(A64_R(27), A64_R(28), A64_SP), ctx);
ctx->fp_used = true;
} else {
find_used_callee_regs(ctx);
for (i = 0; i + 1 < ctx->nr_used_callee_reg; i += 2) {
@ -461,6 +473,19 @@ static void pop_callee_regs(struct jit_ctx *ctx)
}
}
static void emit_percpu_ptr(const u8 dst_reg, void __percpu *ptr,
struct jit_ctx *ctx)
{
const u8 tmp = bpf2a64[TMP_REG_1];
emit_a64_mov_i64(dst_reg, (__force const u64)ptr, ctx);
if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN))
emit(A64_MRS_TPIDR_EL2(tmp), ctx);
else
emit(A64_MRS_TPIDR_EL1(tmp), ctx);
emit(A64_ADD(1, dst_reg, dst_reg, tmp), ctx);
}
#define BTI_INSNS (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL) ? 1 : 0)
#define PAC_INSNS (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL) ? 1 : 0)
@ -476,6 +501,8 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
const bool is_main_prog = !bpf_is_subprog(prog);
const u8 fp = bpf2a64[BPF_REG_FP];
const u8 arena_vm_base = bpf2a64[ARENA_VM_START];
const u8 priv_sp = bpf2a64[PRIVATE_SP];
void __percpu *priv_stack_ptr;
const int idx0 = ctx->idx;
int cur_offset;
@ -551,15 +578,23 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
emit(A64_SUB_I(1, A64_SP, A64_FP, 96), ctx);
}
if (ctx->fp_used)
/* Set up BPF prog stack base register */
emit(A64_MOV(1, fp, A64_SP), ctx);
/* Stack must be multiples of 16B */
ctx->stack_size = round_up(prog->aux->stack_depth, 16);
if (ctx->fp_used) {
if (ctx->priv_sp_used) {
/* Set up private stack pointer */
priv_stack_ptr = prog->aux->priv_stack_ptr + PRIV_STACK_GUARD_SZ;
emit_percpu_ptr(priv_sp, priv_stack_ptr, ctx);
emit(A64_ADD_I(1, fp, priv_sp, ctx->stack_size), ctx);
} else {
/* Set up BPF prog stack base register */
emit(A64_MOV(1, fp, A64_SP), ctx);
}
}
/* Set up function call stack */
if (ctx->stack_size)
if (ctx->stack_size && !ctx->priv_sp_used)
emit(A64_SUB_I(1, A64_SP, A64_SP, ctx->stack_size), ctx);
if (ctx->arena_vm_start)
@ -623,7 +658,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
emit(A64_STR64I(tcc, ptr, 0), ctx);
/* restore SP */
if (ctx->stack_size)
if (ctx->stack_size && !ctx->priv_sp_used)
emit(A64_ADD_I(1, A64_SP, A64_SP, ctx->stack_size), ctx);
pop_callee_regs(ctx);
@ -991,7 +1026,7 @@ static void build_epilogue(struct jit_ctx *ctx, bool was_classic)
const u8 ptr = bpf2a64[TCCNT_PTR];
/* We're done with BPF stack */
if (ctx->stack_size)
if (ctx->stack_size && !ctx->priv_sp_used)
emit(A64_ADD_I(1, A64_SP, A64_SP, ctx->stack_size), ctx);
pop_callee_regs(ctx);
@ -1120,6 +1155,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
const u8 tmp2 = bpf2a64[TMP_REG_2];
const u8 fp = bpf2a64[BPF_REG_FP];
const u8 arena_vm_base = bpf2a64[ARENA_VM_START];
const u8 priv_sp = bpf2a64[PRIVATE_SP];
const s16 off = insn->off;
const s32 imm = insn->imm;
const int i = insn - ctx->prog->insnsi;
@ -1564,7 +1600,7 @@ emit_cond_jmp:
src = tmp2;
}
if (src == fp) {
src_adj = A64_SP;
src_adj = ctx->priv_sp_used ? priv_sp : A64_SP;
off_adj = off + ctx->stack_size;
} else {
src_adj = src;
@ -1630,17 +1666,14 @@ emit_cond_jmp:
return ret;
break;
/* speculation barrier */
/* speculation barrier against v1 and v4 */
case BPF_ST | BPF_NOSPEC:
/*
* Nothing required here.
*
* In case of arm64, we rely on the firmware mitigation of
* Speculative Store Bypass as controlled via the ssbd kernel
* parameter. Whenever the mitigation is enabled, it works
* for all of the kernel code with no need to provide any
* additional instructions.
*/
if (alternative_has_cap_likely(ARM64_HAS_SB)) {
emit(A64_SB, ctx);
} else {
emit(A64_DSB_NSH, ctx);
emit(A64_ISB, ctx);
}
break;
/* ST: *(size *)(dst + off) = imm */
@ -1657,7 +1690,7 @@ emit_cond_jmp:
dst = tmp2;
}
if (dst == fp) {
dst_adj = A64_SP;
dst_adj = ctx->priv_sp_used ? priv_sp : A64_SP;
off_adj = off + ctx->stack_size;
} else {
dst_adj = dst;
@ -1719,7 +1752,7 @@ emit_cond_jmp:
dst = tmp2;
}
if (dst == fp) {
dst_adj = A64_SP;
dst_adj = ctx->priv_sp_used ? priv_sp : A64_SP;
off_adj = off + ctx->stack_size;
} else {
dst_adj = dst;
@ -1862,6 +1895,39 @@ static inline void bpf_flush_icache(void *start, void *end)
flush_icache_range((unsigned long)start, (unsigned long)end);
}
static void priv_stack_init_guard(void __percpu *priv_stack_ptr, int alloc_size)
{
int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3;
u64 *stack_ptr;
for_each_possible_cpu(cpu) {
stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu);
stack_ptr[0] = PRIV_STACK_GUARD_VAL;
stack_ptr[1] = PRIV_STACK_GUARD_VAL;
stack_ptr[underflow_idx] = PRIV_STACK_GUARD_VAL;
stack_ptr[underflow_idx + 1] = PRIV_STACK_GUARD_VAL;
}
}
static void priv_stack_check_guard(void __percpu *priv_stack_ptr, int alloc_size,
struct bpf_prog *prog)
{
int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3;
u64 *stack_ptr;
for_each_possible_cpu(cpu) {
stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu);
if (stack_ptr[0] != PRIV_STACK_GUARD_VAL ||
stack_ptr[1] != PRIV_STACK_GUARD_VAL ||
stack_ptr[underflow_idx] != PRIV_STACK_GUARD_VAL ||
stack_ptr[underflow_idx + 1] != PRIV_STACK_GUARD_VAL) {
pr_err("BPF private stack overflow/underflow detected for prog %sx\n",
bpf_jit_get_prog_name(prog));
break;
}
}
}
struct arm64_jit_data {
struct bpf_binary_header *header;
u8 *ro_image;
@ -1874,9 +1940,11 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
int image_size, prog_size, extable_size, extable_align, extable_offset;
struct bpf_prog *tmp, *orig_prog = prog;
struct bpf_binary_header *header;
struct bpf_binary_header *ro_header;
struct bpf_binary_header *ro_header = NULL;
struct arm64_jit_data *jit_data;
void __percpu *priv_stack_ptr = NULL;
bool was_classic = bpf_prog_was_classic(prog);
int priv_stack_alloc_sz;
bool tmp_blinded = false;
bool extra_pass = false;
struct jit_ctx ctx;
@ -1908,6 +1976,23 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
}
prog->aux->jit_data = jit_data;
}
priv_stack_ptr = prog->aux->priv_stack_ptr;
if (!priv_stack_ptr && prog->aux->jits_use_priv_stack) {
/* Allocate actual private stack size with verifier-calculated
* stack size plus two memory guards to protect overflow and
* underflow.
*/
priv_stack_alloc_sz = round_up(prog->aux->stack_depth, 16) +
2 * PRIV_STACK_GUARD_SZ;
priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_sz, 16, GFP_KERNEL);
if (!priv_stack_ptr) {
prog = orig_prog;
goto out_priv_stack;
}
priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_sz);
prog->aux->priv_stack_ptr = priv_stack_ptr;
}
if (jit_data->ctx.offset) {
ctx = jit_data->ctx;
ro_image_ptr = jit_data->ro_image;
@ -1931,6 +2016,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);
ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena);
if (priv_stack_ptr)
ctx.priv_sp_used = true;
/* Pass 1: Estimate the maximum image size.
*
* BPF line info needs ctx->offset[i] to be the offset of
@ -2070,7 +2158,12 @@ skip_init_ctx:
ctx.offset[i] *= AARCH64_INSN_SIZE;
bpf_prog_fill_jited_linfo(prog, ctx.offset + 1);
out_off:
if (!ro_header && priv_stack_ptr) {
free_percpu(priv_stack_ptr);
prog->aux->priv_stack_ptr = NULL;
}
kvfree(ctx.offset);
out_priv_stack:
kfree(jit_data);
prog->aux->jit_data = NULL;
}
@ -2089,6 +2182,11 @@ out_free_hdr:
goto out_off;
}
bool bpf_jit_supports_private_stack(void)
{
return true;
}
bool bpf_jit_supports_kfunc_call(void)
{
return true;
@ -2243,11 +2341,6 @@ static int calc_arg_aux(const struct btf_func_model *m,
/* the rest arguments are passed through stack */
for (; i < m->nr_args; i++) {
/* We can not know for sure about exact alignment needs for
* struct passed on stack, so deny those
*/
if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG)
return -ENOTSUPP;
stack_slots = (m->arg_size[i] + 7) / 8;
a->bstack_for_args += stack_slots * 8;
a->ostack_for_args = a->ostack_for_args + stack_slots * 8;
@ -2911,6 +3004,17 @@ bool bpf_jit_supports_percpu_insn(void)
return true;
}
bool bpf_jit_bypass_spec_v4(void)
{
/* In case of arm64, we rely on the firmware mitigation of Speculative
* Store Bypass as controlled via the ssbd kernel parameter. Whenever
* the mitigation is enabled, it works for all of the kernel code with
* no need to provide any additional instructions. Therefore, skip
* inserting nospec insns against Spectre v4.
*/
return true;
}
bool bpf_jit_inlines_helper_call(s32 imm)
{
switch (imm) {
@ -2928,6 +3032,8 @@ void bpf_jit_free(struct bpf_prog *prog)
if (prog->jited) {
struct arm64_jit_data *jit_data = prog->aux->jit_data;
struct bpf_binary_header *hdr;
void __percpu *priv_stack_ptr;
int priv_stack_alloc_sz;
/*
* If we fail the final pass of JIT (from jit_subprogs),
@ -2941,6 +3047,13 @@ void bpf_jit_free(struct bpf_prog *prog)
}
hdr = bpf_jit_binary_pack_hdr(prog);
bpf_jit_binary_pack_free(hdr, NULL);
priv_stack_ptr = prog->aux->priv_stack_ptr;
if (priv_stack_ptr) {
priv_stack_alloc_sz = round_up(prog->aux->stack_depth, 16) +
2 * PRIV_STACK_GUARD_SZ;
priv_stack_check_guard(priv_stack_ptr, priv_stack_alloc_sz, prog);
free_percpu(prog->aux->priv_stack_ptr);
}
WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog));
}

79
arch/powerpc/net/bpf_jit_comp64.c
View file

@ -370,6 +370,23 @@ static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 o
return 0;
}
bool bpf_jit_bypass_spec_v1(void)
{
#if defined(CONFIG_PPC_E500) || defined(CONFIG_PPC_BOOK3S_64)
return !(security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) &&
security_ftr_enabled(SEC_FTR_BNDS_CHK_SPEC_BAR));
#else
return true;
#endif
}
bool bpf_jit_bypass_spec_v4(void)
{
return !(security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) &&
security_ftr_enabled(SEC_FTR_STF_BARRIER) &&
stf_barrier_type_get() != STF_BARRIER_NONE);
}
/*
* We spill into the redzone always, even if the bpf program has its own stackframe.
* Offsets hardcoded based on BPF_PPC_STACK_SAVE -- see bpf_jit_stack_local()
@ -397,6 +414,7 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code
u32 *addrs, int pass, bool extra_pass)
{
enum stf_barrier_type stf_barrier = stf_barrier_type_get();
bool sync_emitted, ori31_emitted;
const struct bpf_insn *insn = fp->insnsi;
int flen = fp->len;
int i, ret;
@ -789,30 +807,51 @@ emit_clear:
/*
* BPF_ST NOSPEC (speculation barrier)
*
* The following must act as a barrier against both Spectre v1
* and v4 if we requested both mitigations. Therefore, also emit
* 'isync; sync' on E500 or 'ori31' on BOOK3S_64 in addition to
* the insns needed for a Spectre v4 barrier.
*
* If we requested only !bypass_spec_v1 OR only !bypass_spec_v4,
* we can skip the respective other barrier type as an
* optimization.
*/
case BPF_ST | BPF_NOSPEC:
if (!security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) ||
!security_ftr_enabled(SEC_FTR_STF_BARRIER))
break;
switch (stf_barrier) {
case STF_BARRIER_EIEIO:
EMIT(PPC_RAW_EIEIO() | 0x02000000);
break;
case STF_BARRIER_SYNC_ORI:
sync_emitted = false;
ori31_emitted = false;
if (IS_ENABLED(CONFIG_PPC_E500) &&
!bpf_jit_bypass_spec_v1()) {
EMIT(PPC_RAW_ISYNC());
EMIT(PPC_RAW_SYNC());
EMIT(PPC_RAW_LD(tmp1_reg, _R13, 0));
EMIT(PPC_RAW_ORI(_R31, _R31, 0));
break;
case STF_BARRIER_FALLBACK:
ctx->seen |= SEEN_FUNC;
PPC_LI64(_R12, dereference_kernel_function_descriptor(bpf_stf_barrier));
EMIT(PPC_RAW_MTCTR(_R12));
EMIT(PPC_RAW_BCTRL());
break;
case STF_BARRIER_NONE:
break;
sync_emitted = true;
}
if (!bpf_jit_bypass_spec_v4()) {
switch (stf_barrier) {
case STF_BARRIER_EIEIO:
EMIT(PPC_RAW_EIEIO() | 0x02000000);
break;
case STF_BARRIER_SYNC_ORI:
if (!sync_emitted)
EMIT(PPC_RAW_SYNC());
EMIT(PPC_RAW_LD(tmp1_reg, _R13, 0));
EMIT(PPC_RAW_ORI(_R31, _R31, 0));
ori31_emitted = true;
break;
case STF_BARRIER_FALLBACK:
ctx->seen |= SEEN_FUNC;
PPC_LI64(_R12, dereference_kernel_function_descriptor(bpf_stf_barrier));
EMIT(PPC_RAW_MTCTR(_R12));
EMIT(PPC_RAW_BCTRL());
break;
case STF_BARRIER_NONE:
break;
}
}
if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) &&
!bpf_jit_bypass_spec_v1() &&
!ori31_emitted)
EMIT(PPC_RAW_ORI(_R31, _R31, 0));
break;
/*

55
arch/s390/net/bpf_jit.h
View file

@ -1,55 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* BPF Jit compiler defines
*
* Copyright IBM Corp. 2012,2015
*
* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
* Michael Holzheu <holzheu@linux.vnet.ibm.com>
*/
#ifndef __ARCH_S390_NET_BPF_JIT_H
#define __ARCH_S390_NET_BPF_JIT_H
#ifndef __ASSEMBLER__
#include <linux/filter.h>
#include <linux/types.h>
#endif /* __ASSEMBLER__ */
/*
* Stackframe layout (packed stack):
*
* ^ high
* +---------------+ |
* | old backchain | |
* +---------------+ |
* | r15 - r6 | |
* +---------------+ |
* | 4 byte align | |
* | tail_call_cnt | |
* BFP -> +===============+ |
* | | |
* | BPF stack | |
* | | |
* R15+160 -> +---------------+ |
* | new backchain | |
* R15+152 -> +---------------+ |
* | + 152 byte SA | |
* R15 -> +---------------+ + low
*
* We get 160 bytes stack space from calling function, but only use
* 12 * 8 byte for old backchain, r15..r6, and tail_call_cnt.
*
* The stack size used by the BPF program ("BPF stack" above) is passed
* via "aux->stack_depth".
*/
#define STK_SPACE_ADD (160)
#define STK_160_UNUSED (160 - 12 * 8)
#define STK_OFF (STK_SPACE_ADD - STK_160_UNUSED)
#define STK_OFF_R6 (160 - 11 * 8) /* Offset of r6 on stack */
#define STK_OFF_TCCNT (160 - 12 * 8) /* Offset of tail_call_cnt on stack */
#endif /* __ARCH_S390_NET_BPF_JIT_H */

113
arch/s390/net/bpf_jit_comp.c
View file

@ -32,7 +32,6 @@
#include <asm/set_memory.h>
#include <asm/text-patching.h>
#include <asm/unwind.h>
#include "bpf_jit.h"
struct bpf_jit {
u32 seen; /* Flags to remember seen eBPF instructions */
@ -54,6 +53,7 @@ struct bpf_jit {
int prologue_plt; /* Start of prologue hotpatch PLT */
int kern_arena; /* Pool offset of kernel arena address */
u64 user_arena; /* User arena address */
u32 frame_off; /* Offset of struct bpf_prog from %r15 */
};
#define SEEN_MEM BIT(0) /* use mem[] for temporary storage */
@ -425,12 +425,26 @@ static void jit_fill_hole(void *area, unsigned int size)
memset(area, 0, size);
}
/*
* Caller-allocated part of the frame.
* Thanks to packed stack, its otherwise unused initial part can be used for
* the BPF stack and for the next frame.
*/
struct prog_frame {
u64 unused[8];
/* BPF stack starts here and grows towards 0 */
u32 tail_call_cnt;
u32 pad;
u64 r6[10]; /* r6 - r15 */
u64 backchain;
} __packed;
/*
* Save registers from "rs" (register start) to "re" (register end) on stack
*/
static void save_regs(struct bpf_jit *jit, u32 rs, u32 re)
{
u32 off = STK_OFF_R6 + (rs - 6) * 8;
u32 off = offsetof(struct prog_frame, r6) + (rs - 6) * 8;
if (rs == re)
/* stg %rs,off(%r15) */
@ -443,12 +457,9 @@ static void save_regs(struct bpf_jit *jit, u32 rs, u32 re)
/*
* Restore registers from "rs" (register start) to "re" (register end) on stack
*/
static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re, u32 stack_depth)
static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re)
{
u32 off = STK_OFF_R6 + (rs - 6) * 8;
if (jit->seen & SEEN_STACK)
off += STK_OFF + stack_depth;
u32 off = jit->frame_off + offsetof(struct prog_frame, r6) + (rs - 6) * 8;
if (rs == re)
/* lg %rs,off(%r15) */
@ -492,8 +503,7 @@ static int get_end(u16 seen_regs, int start)
* Save and restore clobbered registers (6-15) on stack.
* We save/restore registers in chunks with gap >= 2 registers.
*/
static void save_restore_regs(struct bpf_jit *jit, int op, u32 stack_depth,
u16 extra_regs)
static void save_restore_regs(struct bpf_jit *jit, int op, u16 extra_regs)
{
u16 seen_regs = jit->seen_regs | extra_regs;
const int last = 15, save_restore_size = 6;
@ -516,7 +526,7 @@ static void save_restore_regs(struct bpf_jit *jit, int op, u32 stack_depth,
if (op == REGS_SAVE)
save_regs(jit, rs, re);
else
restore_regs(jit, rs, re, stack_depth);
restore_regs(jit, rs, re);
re++;
} while (re <= last);
}
@ -581,11 +591,12 @@ static void bpf_jit_plt(struct bpf_plt *plt, void *ret, void *target)
* Emit function prologue
*
* Save registers and create stack frame if necessary.
* See stack frame layout description in "bpf_jit.h"!
* Stack frame layout is described by struct prog_frame.
*/
static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp,
u32 stack_depth)
static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp)
{
BUILD_BUG_ON(sizeof(struct prog_frame) != STACK_FRAME_OVERHEAD);
/* No-op for hotpatching */
/* brcl 0,prologue_plt */
EMIT6_PCREL_RILC(0xc0040000, 0, jit->prologue_plt);
@ -593,8 +604,9 @@ static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp,
if (!bpf_is_subprog(fp)) {
/* Initialize the tail call counter in the main program. */
/* xc STK_OFF_TCCNT(4,%r15),STK_OFF_TCCNT(%r15) */
_EMIT6(0xd703f000 | STK_OFF_TCCNT, 0xf000 | STK_OFF_TCCNT);
/* xc tail_call_cnt(4,%r15),tail_call_cnt(%r15) */
_EMIT6(0xd703f000 | offsetof(struct prog_frame, tail_call_cnt),
0xf000 | offsetof(struct prog_frame, tail_call_cnt));
} else {
/*
* Skip the tail call counter initialization in subprograms.
@ -617,7 +629,7 @@ static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp,
jit->seen_regs |= NVREGS;
} else {
/* Save registers */
save_restore_regs(jit, REGS_SAVE, stack_depth,
save_restore_regs(jit, REGS_SAVE,
fp->aux->exception_boundary ? NVREGS : 0);
}
/* Setup literal pool */
@ -637,13 +649,15 @@ static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp,
if (is_first_pass(jit) || (jit->seen & SEEN_STACK)) {
/* lgr %w1,%r15 (backchain) */
EMIT4(0xb9040000, REG_W1, REG_15);
/* la %bfp,STK_160_UNUSED(%r15) (BPF frame pointer) */
EMIT4_DISP(0x41000000, BPF_REG_FP, REG_15, STK_160_UNUSED);
/* aghi %r15,-STK_OFF */
EMIT4_IMM(0xa70b0000, REG_15, -(STK_OFF + stack_depth));
/* stg %w1,152(%r15) (backchain) */
/* la %bfp,unused_end(%r15) (BPF frame pointer) */
EMIT4_DISP(0x41000000, BPF_REG_FP, REG_15,
offsetofend(struct prog_frame, unused));
/* aghi %r15,-frame_off */
EMIT4_IMM(0xa70b0000, REG_15, -jit->frame_off);
/* stg %w1,backchain(%r15) */
EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0,
REG_15, 152);
REG_15,
offsetof(struct prog_frame, backchain));
}
}
@ -677,13 +691,13 @@ static void call_r1(struct bpf_jit *jit)
/*
* Function epilogue
*/
static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth)
static void bpf_jit_epilogue(struct bpf_jit *jit)
{
jit->exit_ip = jit->prg;
/* Load exit code: lgr %r2,%b0 */
EMIT4(0xb9040000, REG_2, BPF_REG_0);
/* Restore registers */
save_restore_regs(jit, REGS_RESTORE, stack_depth, 0);
save_restore_regs(jit, REGS_RESTORE, 0);
EMIT_JUMP_REG(14);
jit->prg = ALIGN(jit->prg, 8);
@ -865,7 +879,7 @@ static int sign_extend(struct bpf_jit *jit, int r, u8 size, u8 flags)
* stack space for the large switch statement.
*/
static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
int i, bool extra_pass, u32 stack_depth)
int i, bool extra_pass)
{
struct bpf_insn *insn = &fp->insnsi[i];
s32 branch_oc_off = insn->off;
@ -1786,9 +1800,10 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
* Note 2: We assume that the verifier does not let us call the
* main program, which clears the tail call counter on entry.
*/
/* mvc STK_OFF_TCCNT(4,%r15),N(%r15) */
_EMIT6(0xd203f000 | STK_OFF_TCCNT,
0xf000 | (STK_OFF_TCCNT + STK_OFF + stack_depth));
/* mvc tail_call_cnt(4,%r15),frame_off+tail_call_cnt(%r15) */
_EMIT6(0xd203f000 | offsetof(struct prog_frame, tail_call_cnt),
0xf000 | (jit->frame_off +
offsetof(struct prog_frame, tail_call_cnt)));
/* Sign-extend the kfunc arguments. */
if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
@ -1839,10 +1854,8 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
* goto out;
*/
if (jit->seen & SEEN_STACK)
off = STK_OFF_TCCNT + STK_OFF + stack_depth;
else
off = STK_OFF_TCCNT;
off = jit->frame_off +
offsetof(struct prog_frame, tail_call_cnt);
/* lhi %w0,1 */
EMIT4_IMM(0xa7080000, REG_W0, 1);
/* laal %w1,%w0,off(%r15) */
@ -1872,7 +1885,7 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
/*
* Restore registers before calling function
*/
save_restore_regs(jit, REGS_RESTORE, stack_depth, 0);
save_restore_regs(jit, REGS_RESTORE, 0);
/*
* goto *(prog->bpf_func + tail_call_start);
@ -2165,7 +2178,7 @@ static int bpf_set_addr(struct bpf_jit *jit, int i)
* Compile eBPF program into s390x code
*/
static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp,
bool extra_pass, u32 stack_depth)
bool extra_pass)
{
int i, insn_count, lit32_size, lit64_size;
u64 kern_arena;
@ -2174,24 +2187,30 @@ static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp,
jit->lit64 = jit->lit64_start;
jit->prg = 0;
jit->excnt = 0;
if (is_first_pass(jit) || (jit->seen & SEEN_STACK))
jit->frame_off = sizeof(struct prog_frame) -
offsetofend(struct prog_frame, unused) +
round_up(fp->aux->stack_depth, 8);
else
jit->frame_off = 0;
kern_arena = bpf_arena_get_kern_vm_start(fp->aux->arena);
if (kern_arena)
jit->kern_arena = _EMIT_CONST_U64(kern_arena);
jit->user_arena = bpf_arena_get_user_vm_start(fp->aux->arena);
bpf_jit_prologue(jit, fp, stack_depth);
bpf_jit_prologue(jit, fp);
if (bpf_set_addr(jit, 0) < 0)
return -1;
for (i = 0; i < fp->len; i += insn_count) {
insn_count = bpf_jit_insn(jit, fp, i, extra_pass, stack_depth);
insn_count = bpf_jit_insn(jit, fp, i, extra_pass);
if (insn_count < 0)
return -1;
/* Next instruction address */
if (bpf_set_addr(jit, i + insn_count) < 0)
return -1;
}
bpf_jit_epilogue(jit, stack_depth);
bpf_jit_epilogue(jit);
lit32_size = jit->lit32 - jit->lit32_start;
lit64_size = jit->lit64 - jit->lit64_start;
@ -2267,7 +2286,6 @@ static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit,
*/
struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
{
u32 stack_depth = round_up(fp->aux->stack_depth, 8);
struct bpf_prog *tmp, *orig_fp = fp;
struct bpf_binary_header *header;
struct s390_jit_data *jit_data;
@ -2320,7 +2338,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
* - 3: Calculate program size and addrs array
*/
for (pass = 1; pass <= 3; pass++) {
if (bpf_jit_prog(&jit, fp, extra_pass, stack_depth)) {
if (bpf_jit_prog(&jit, fp, extra_pass)) {
fp = orig_fp;
goto free_addrs;
}
@ -2334,7 +2352,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
goto free_addrs;
}
skip_init_ctx:
if (bpf_jit_prog(&jit, fp, extra_pass, stack_depth)) {
if (bpf_jit_prog(&jit, fp, extra_pass)) {
bpf_jit_binary_free(header);
fp = orig_fp;
goto free_addrs;
@ -2654,9 +2672,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
/* stg %r1,backchain_off(%r15) */
EMIT6_DISP_LH(0xe3000000, 0x0024, REG_1, REG_0, REG_15,
tjit->backchain_off);
/* mvc tccnt_off(4,%r15),stack_size+STK_OFF_TCCNT(%r15) */
/* mvc tccnt_off(4,%r15),stack_size+tail_call_cnt(%r15) */
_EMIT6(0xd203f000 | tjit->tccnt_off,
0xf000 | (tjit->stack_size + STK_OFF_TCCNT));
0xf000 | (tjit->stack_size +
offsetof(struct prog_frame, tail_call_cnt)));
/* stmg %r2,%rN,fwd_reg_args_off(%r15) */
if (nr_reg_args)
EMIT6_DISP_LH(0xeb000000, 0x0024, REG_2,
@ -2793,8 +2812,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
(nr_stack_args * sizeof(u64) - 1) << 16 |
tjit->stack_args_off,
0xf000 | tjit->orig_stack_args_off);
/* mvc STK_OFF_TCCNT(4,%r15),tccnt_off(%r15) */
_EMIT6(0xd203f000 | STK_OFF_TCCNT, 0xf000 | tjit->tccnt_off);
/* mvc tail_call_cnt(4,%r15),tccnt_off(%r15) */
_EMIT6(0xd203f000 | offsetof(struct prog_frame, tail_call_cnt),
0xf000 | tjit->tccnt_off);
/* lgr %r1,%r8 */
EMIT4(0xb9040000, REG_1, REG_8);
/* %r1() */
@ -2851,8 +2871,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
if (flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET))
EMIT6_DISP_LH(0xe3000000, 0x0004, REG_2, REG_0, REG_15,
tjit->retval_off);
/* mvc stack_size+STK_OFF_TCCNT(4,%r15),tccnt_off(%r15) */
_EMIT6(0xd203f000 | (tjit->stack_size + STK_OFF_TCCNT),
/* mvc stack_size+tail_call_cnt(4,%r15),tccnt_off(%r15) */
_EMIT6(0xd203f000 | (tjit->stack_size +
offsetof(struct prog_frame, tail_call_cnt)),
0xf000 | tjit->tccnt_off);
/* aghi %r15,stack_size */
EMIT4_IMM(0xa70b0000, REG_15, tjit->stack_size);

10
arch/x86/net/bpf_jit_comp.c
View file

@ -3501,13 +3501,6 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func
return emit_bpf_dispatcher(&prog, 0, num_funcs - 1, funcs, image, buf);
}
static const char *bpf_get_prog_name(struct bpf_prog *prog)
{
if (prog->aux->ksym.prog)
return prog->aux->ksym.name;
return prog->aux->name;
}
static void priv_stack_init_guard(void __percpu *priv_stack_ptr, int alloc_size)
{
int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3;
@ -3531,7 +3524,7 @@ static void priv_stack_check_guard(void __percpu *priv_stack_ptr, int alloc_size
if (stack_ptr[0] != PRIV_STACK_GUARD_VAL ||
stack_ptr[underflow_idx] != PRIV_STACK_GUARD_VAL) {
pr_err("BPF private stack overflow/underflow detected for prog %sx\n",
bpf_get_prog_name(prog));
bpf_jit_get_prog_name(prog));
break;
}
}
@ -3845,7 +3838,6 @@ void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp
}
return;
#endif
WARN(1, "verification of programs using bpf_throw should have failed\n");
}
void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke,

10
drivers/net/netkit.c
View file

@ -32,7 +32,6 @@ struct netkit {
struct netkit_link {
struct bpf_link link;
struct net_device *dev;
u32 location;
};
static __always_inline int
@ -733,8 +732,8 @@ static void netkit_link_fdinfo(const struct bpf_link *link, struct seq_file *seq
seq_printf(seq, "ifindex:\t%u\n", ifindex);
seq_printf(seq, "attach_type:\t%u (%s)\n",
nkl->location,
nkl->location == BPF_NETKIT_PRIMARY ? "primary" : "peer");
link->attach_type,
link->attach_type == BPF_NETKIT_PRIMARY ? "primary" : "peer");
}
static int netkit_link_fill_info(const struct bpf_link *link,
@ -749,7 +748,7 @@ static int netkit_link_fill_info(const struct bpf_link *link,
rtnl_unlock();
info->netkit.ifindex = ifindex;
info->netkit.attach_type = nkl->location;
info->netkit.attach_type = link->attach_type;
return 0;
}
@ -775,8 +774,7 @@ static int netkit_link_init(struct netkit_link *nkl,
struct bpf_prog *prog)
{
bpf_link_init(&nkl->link, BPF_LINK_TYPE_NETKIT,
&netkit_link_lops, prog);
nkl->location = attr->link_create.attach_type;
&netkit_link_lops, prog, attr->link_create.attach_type);
nkl->dev = dev;
return bpf_link_prime(&nkl->link, link_primer);
}

4
fs/kernfs/inode.c
View file

@ -142,9 +142,9 @@ ssize_t kernfs_iop_listxattr(struct dentry *dentry, char *buf, size_t size)
struct kernfs_node *kn = kernfs_dentry_node(dentry);
struct kernfs_iattrs *attrs;
attrs = kernfs_iattrs(kn);
attrs = kernfs_iattrs_noalloc(kn);
if (!attrs)
return -ENOMEM;
return -ENODATA;
return simple_xattr_list(d_inode(dentry), &attrs->xattrs, buf, size);
}

1
include/linux/bpf-cgroup-defs.h
View file

@ -63,6 +63,7 @@ struct cgroup_bpf {
*/
struct hlist_head progs[MAX_CGROUP_BPF_ATTACH_TYPE];
u8 flags[MAX_CGROUP_BPF_ATTACH_TYPE];
u64 revisions[MAX_CGROUP_BPF_ATTACH_TYPE];
/* list of cgroup shared storages */
struct list_head storages;

1
include/linux/bpf-cgroup.h
View file

@ -103,7 +103,6 @@ struct bpf_cgroup_storage {
struct bpf_cgroup_link {
struct bpf_link link;
struct cgroup *cgroup;
enum bpf_attach_type type;
};
struct bpf_prog_list {

125
include/linux/bpf.h
View file

@ -1538,6 +1538,37 @@ struct btf_mod_pair {
struct bpf_kfunc_desc_tab;
enum bpf_stream_id {
BPF_STDOUT = 1,
BPF_STDERR = 2,
};
struct bpf_stream_elem {
struct llist_node node;
int total_len;
int consumed_len;
char str[];
};
enum {
/* 100k bytes */
BPF_STREAM_MAX_CAPACITY = 100000ULL,
};
struct bpf_stream {
atomic_t capacity;
struct llist_head log; /* list of in-flight stream elements in LIFO order */
struct mutex lock; /* lock protecting backlog_{head,tail} */
struct llist_node *backlog_head; /* list of in-flight stream elements in FIFO order */
struct llist_node *backlog_tail; /* tail of the list above */
};
struct bpf_stream_stage {
struct llist_head log;
int len;
};
struct bpf_prog_aux {
atomic64_t refcnt;
u32 used_map_cnt;
@ -1646,6 +1677,7 @@ struct bpf_prog_aux {
struct work_struct work;
struct rcu_head rcu;
};
struct bpf_stream stream[2];
};
struct bpf_prog {
@ -1697,11 +1729,10 @@ struct bpf_link {
enum bpf_link_type type;
const struct bpf_link_ops *ops;
struct bpf_prog *prog;
/* whether BPF link itself has "sleepable" semantics, which can differ
* from underlying BPF program having a "sleepable" semantics, as BPF
* link's semantics is determined by target attach hook
*/
bool sleepable;
u32 flags;
enum bpf_attach_type attach_type;
/* rcu is used before freeing, work can be used to schedule that
* RCU-based freeing before that, so they never overlap
*/
@ -1709,6 +1740,11 @@ struct bpf_link {
struct rcu_head rcu;
struct work_struct work;
};
/* whether BPF link itself has "sleepable" semantics, which can differ
* from underlying BPF program having a "sleepable" semantics, as BPF
* link's semantics is determined by target attach hook
*/
bool sleepable;
};
struct bpf_link_ops {
@ -1748,7 +1784,6 @@ struct bpf_shim_tramp_link {
struct bpf_tracing_link {
struct bpf_tramp_link link;
enum bpf_attach_type attach_type;
struct bpf_trampoline *trampoline;
struct bpf_prog *tgt_prog;
};
@ -2001,11 +2036,13 @@ int bpf_prog_ctx_arg_info_init(struct bpf_prog *prog,
#if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM)
int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
int cgroup_atype);
int cgroup_atype,
enum bpf_attach_type attach_type);
void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog);
#else
static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
int cgroup_atype)
int cgroup_atype,
enum bpf_attach_type attach_type)
{
return -EOPNOTSUPP;
}
@ -2288,6 +2325,9 @@ bpf_prog_run_array_uprobe(const struct bpf_prog_array *array,
return ret;
}
bool bpf_jit_bypass_spec_v1(void);
bool bpf_jit_bypass_spec_v4(void);
#ifdef CONFIG_BPF_SYSCALL
DECLARE_PER_CPU(int, bpf_prog_active);
extern struct mutex bpf_stats_enabled_mutex;
@ -2314,6 +2354,7 @@ extern const struct super_operations bpf_super_ops;
extern const struct file_operations bpf_map_fops;
extern const struct file_operations bpf_prog_fops;
extern const struct file_operations bpf_iter_fops;
extern const struct file_operations bpf_token_fops;
#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
extern const struct bpf_prog_ops _name ## _prog_ops; \
@ -2405,6 +2446,7 @@ int generic_map_delete_batch(struct bpf_map *map,
struct bpf_map *bpf_map_get_curr_or_next(u32 *id);
struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id);
int bpf_map_alloc_pages(const struct bpf_map *map, int nid,
unsigned long nr_pages, struct page **page_array);
#ifdef CONFIG_MEMCG
@ -2475,22 +2517,27 @@ static inline bool bpf_allow_uninit_stack(const struct bpf_token *token)
static inline bool bpf_bypass_spec_v1(const struct bpf_token *token)
{
return cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON);
return bpf_jit_bypass_spec_v1() ||
cpu_mitigations_off() ||
bpf_token_capable(token, CAP_PERFMON);
}
static inline bool bpf_bypass_spec_v4(const struct bpf_token *token)
{
return cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON);
return bpf_jit_bypass_spec_v4() ||
cpu_mitigations_off() ||
bpf_token_capable(token, CAP_PERFMON);
}
int bpf_map_new_fd(struct bpf_map *map, int flags);
int bpf_prog_new_fd(struct bpf_prog *prog);
void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
const struct bpf_link_ops *ops, struct bpf_prog *prog);
const struct bpf_link_ops *ops, struct bpf_prog *prog,
enum bpf_attach_type attach_type);
void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type,
const struct bpf_link_ops *ops, struct bpf_prog *prog,
bool sleepable);
enum bpf_attach_type attach_type, bool sleepable);
int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer);
int bpf_link_settle(struct bpf_link_primer *primer);
void bpf_link_cleanup(struct bpf_link_primer *primer);
@ -2505,6 +2552,9 @@ void bpf_token_inc(struct bpf_token *token);
void bpf_token_put(struct bpf_token *token);
int bpf_token_create(union bpf_attr *attr);
struct bpf_token *bpf_token_get_from_fd(u32 ufd);
int bpf_token_get_info_by_fd(struct bpf_token *token,
const union bpf_attr *attr,
union bpf_attr __user *uattr);
bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd);
bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type);
@ -2842,13 +2892,13 @@ bpf_prog_inc_not_zero(struct bpf_prog *prog)
static inline void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
const struct bpf_link_ops *ops,
struct bpf_prog *prog)
struct bpf_prog *prog, enum bpf_attach_type attach_type)
{
}
static inline void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type,
const struct bpf_link_ops *ops, struct bpf_prog *prog,
bool sleepable)
enum bpf_attach_type attach_type, bool sleepable)
{
}
@ -2903,6 +2953,13 @@ static inline struct bpf_token *bpf_token_get_from_fd(u32 ufd)
return ERR_PTR(-EOPNOTSUPP);
}
static inline int bpf_token_get_info_by_fd(struct bpf_token *token,
const union bpf_attr *attr,
union bpf_attr __user *uattr)
{
return -EOPNOTSUPP;
}
static inline void __dev_flush(struct list_head *flush_list)
{
}
@ -3543,6 +3600,16 @@ bool btf_id_set_contains(const struct btf_id_set *set, u32 id);
#define MAX_BPRINTF_VARARGS 12
#define MAX_BPRINTF_BUF 1024
/* Per-cpu temp buffers used by printf-like helpers to store the bprintf binary
* arguments representation.
*/
#define MAX_BPRINTF_BIN_ARGS 512
struct bpf_bprintf_buffers {
char bin_args[MAX_BPRINTF_BIN_ARGS];
char buf[MAX_BPRINTF_BUF];
};
struct bpf_bprintf_data {
u32 *bin_args;
char *buf;
@ -3550,9 +3617,33 @@ struct bpf_bprintf_data {
bool get_buf;
};
int bpf_bprintf_prepare(char *fmt, u32 fmt_size, const u64 *raw_args,
int bpf_bprintf_prepare(const char *fmt, u32 fmt_size, const u64 *raw_args,
u32 num_args, struct bpf_bprintf_data *data);
void bpf_bprintf_cleanup(struct bpf_bprintf_data *data);
int bpf_try_get_buffers(struct bpf_bprintf_buffers **bufs);
void bpf_put_buffers(void);
void bpf_prog_stream_init(struct bpf_prog *prog);
void bpf_prog_stream_free(struct bpf_prog *prog);
int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len);
void bpf_stream_stage_init(struct bpf_stream_stage *ss);
void bpf_stream_stage_free(struct bpf_stream_stage *ss);
__printf(2, 3)
int bpf_stream_stage_printk(struct bpf_stream_stage *ss, const char *fmt, ...);
int bpf_stream_stage_commit(struct bpf_stream_stage *ss, struct bpf_prog *prog,
enum bpf_stream_id stream_id);
int bpf_stream_stage_dump_stack(struct bpf_stream_stage *ss);
#define bpf_stream_printk(ss, ...) bpf_stream_stage_printk(&ss, __VA_ARGS__)
#define bpf_stream_dump_stack(ss) bpf_stream_stage_dump_stack(&ss)
#define bpf_stream_stage(ss, prog, stream_id, expr) \
({ \
bpf_stream_stage_init(&ss); \
(expr); \
bpf_stream_stage_commit(&ss, prog, stream_id); \
bpf_stream_stage_free(&ss); \
})
#ifdef CONFIG_BPF_LSM
void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype);
@ -3588,4 +3679,8 @@ static inline bool bpf_is_subprog(const struct bpf_prog *prog)
return prog->aux->func_idx != 0;
}
int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep,
const char **linep, int *nump);
struct bpf_prog *bpf_prog_find_from_stack(void);
#endif /* _LINUX_BPF_H */

81
include/linux/bpf_verifier.h
View file

@ -344,7 +344,7 @@ struct bpf_func_state {
#define MAX_CALL_FRAMES 8
/* instruction history flags, used in bpf_insn_hist_entry.flags field */
/* instruction history flags, used in bpf_jmp_history_entry.flags field */
enum {
/* instruction references stack slot through PTR_TO_STACK register;
* we also store stack's frame number in lower 3 bits (MAX_CALL_FRAMES is 8)
@ -366,7 +366,7 @@ enum {
static_assert(INSN_F_FRAMENO_MASK + 1 >= MAX_CALL_FRAMES);
static_assert(INSN_F_SPI_MASK + 1 >= MAX_BPF_STACK / 8);
struct bpf_insn_hist_entry {
struct bpf_jmp_history_entry {
u32 idx;
/* insn idx can't be bigger than 1 million */
u32 prev_idx : 20;
@ -449,32 +449,20 @@ struct bpf_verifier_state {
/* first and last insn idx of this verifier state */
u32 first_insn_idx;
u32 last_insn_idx;
/* If this state is a part of states loop this field points to some
* parent of this state such that:
* - it is also a member of the same states loop;
* - DFS states traversal starting from initial state visits loop_entry
* state before this state.
* Used to compute topmost loop entry for state loops.
* State loops might appear because of open coded iterators logic.
* See get_loop_entry() for more information.
/* if this state is a backedge state then equal_state
* records cached state to which this state is equal.
*/
struct bpf_verifier_state *loop_entry;
/* Sub-range of env->insn_hist[] corresponding to this state's
* instruction history.
* Backtracking is using it to go from last to first.
* For most states instruction history is short, 0-3 instructions.
struct bpf_verifier_state *equal_state;
/* jmp history recorded from first to last.
* backtracking is using it to go from last to first.
* For most states jmp_history_cnt is [0-3].
* For loops can go up to ~40.
*/
u32 insn_hist_start;
u32 insn_hist_end;
struct bpf_jmp_history_entry *jmp_history;
u32 jmp_history_cnt;
u32 dfs_depth;
u32 callback_unroll_depth;
u32 may_goto_depth;
/* If this state was ever pointed-to by other state's loop_entry field
* this flag would be set to true. Used to avoid freeing such states
* while they are still in use.
*/
u32 used_as_loop_entry;
};
#define bpf_get_spilled_reg(slot, frame, mask) \
@ -580,7 +568,8 @@ struct bpf_insn_aux_data {
u64 map_key_state; /* constant (32 bit) key tracking for maps */
int ctx_field_size; /* the ctx field size for load insn, maybe 0 */
u32 seen; /* this insn was processed by the verifier at env->pass_cnt */
bool sanitize_stack_spill; /* subject to Spectre v4 sanitation */
bool nospec; /* do not execute this instruction speculatively */
bool nospec_result; /* result is unsafe under speculation, nospec must follow */
bool zext_dst; /* this insn zero extends dst reg */
bool needs_zext; /* alu op needs to clear upper bits */
bool storage_get_func_atomic; /* bpf_*_storage_get() with atomic memory alloc */
@ -609,6 +598,11 @@ struct bpf_insn_aux_data {
* accepts callback function as a parameter.
*/
bool calls_callback;
/*
* CFG strongly connected component this instruction belongs to,
* zero if it is a singleton SCC.
*/
u32 scc;
/* registers alive before this instruction. */
u16 live_regs_before;
};
@ -718,6 +712,38 @@ struct bpf_idset {
u32 ids[BPF_ID_MAP_SIZE];
};
/* see verifier.c:compute_scc_callchain() */
struct bpf_scc_callchain {
/* call sites from bpf_verifier_state->frame[*]->callsite leading to this SCC */
u32 callsites[MAX_CALL_FRAMES - 1];
/* last frame in a chain is identified by SCC id */
u32 scc;
};
/* verifier state waiting for propagate_backedges() */
struct bpf_scc_backedge {
struct bpf_scc_backedge *next;
struct bpf_verifier_state state;
};
struct bpf_scc_visit {
struct bpf_scc_callchain callchain;
/* first state in current verification path that entered SCC
* identified by the callchain
*/
struct bpf_verifier_state *entry_state;
struct bpf_scc_backedge *backedges; /* list of backedges */
u32 num_backedges;
};
/* An array of bpf_scc_visit structs sharing tht same bpf_scc_callchain->scc
* but having different bpf_scc_callchain->callsites.
*/
struct bpf_scc_info {
u32 num_visits;
struct bpf_scc_visit visits[];
};
/* single container for all structs
* one verifier_env per bpf_check() call
*/
@ -775,9 +801,7 @@ struct bpf_verifier_env {
int cur_postorder;
} cfg;
struct backtrack_state bt;
struct bpf_insn_hist_entry *insn_hist;
struct bpf_insn_hist_entry *cur_hist_ent;
u32 insn_hist_cap;
struct bpf_jmp_history_entry *cur_hist_ent;
u32 pass_cnt; /* number of times do_check() was called */
u32 subprog_cnt;
/* number of instructions analyzed by the verifier */
@ -799,6 +823,7 @@ struct bpf_verifier_env {
u32 longest_mark_read_walk;
u32 free_list_size;
u32 explored_states_size;
u32 num_backedges;
bpfptr_t fd_array;
/* bit mask to keep track of whether a register has been accessed
@ -816,6 +841,10 @@ struct bpf_verifier_env {
char tmp_str_buf[TMP_STR_BUF_LEN];
struct bpf_insn insn_buf[INSN_BUF_SIZE];
struct bpf_insn epilogue_buf[INSN_BUF_SIZE];
struct bpf_scc_callchain callchain_buf;
/* array of pointers to bpf_scc_info indexed by SCC id */
struct bpf_scc_info **scc_info;
u32 scc_cnt;
};
static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog)

3
include/linux/btf.h
View file

@ -221,6 +221,9 @@ bool btf_is_vmlinux(const struct btf *btf);
struct module *btf_try_get_module(const struct btf *btf);
u32 btf_nr_types(const struct btf *btf);
struct btf *btf_base_btf(const struct btf *btf);
bool btf_type_is_i32(const struct btf_type *t);
bool btf_type_is_i64(const struct btf_type *t);
bool btf_type_is_primitive(const struct btf_type *t);
bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s,
const struct btf_member *m,
u32 expected_offset, u32 expected_size);

4
include/linux/filter.h
View file

@ -82,7 +82,7 @@ struct ctl_table_header;
#define BPF_CALL_ARGS 0xe0
/* unused opcode to mark speculation barrier for mitigating
* Speculative Store Bypass
* Spectre v1 and v4
*/
#define BPF_NOSPEC 0xc0
@ -1288,6 +1288,8 @@ int bpf_jit_get_func_addr(const struct bpf_prog *prog,
const struct bpf_insn *insn, bool extra_pass,
u64 *func_addr, bool *func_addr_fixed);
const char *bpf_jit_get_prog_name(struct bpf_prog *prog);
struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *fp);
void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other);

2
include/linux/tnum.h
View file

@ -40,6 +40,8 @@ struct tnum tnum_arshift(struct tnum a, u8 min_shift, u8 insn_bitness);
struct tnum tnum_add(struct tnum a, struct tnum b);
/* Subtract two tnums, return @a - @b */
struct tnum tnum_sub(struct tnum a, struct tnum b);
/* Neg of a tnum, return 0 - @a */
struct tnum tnum_neg(struct tnum a);
/* Bitwise-AND, return @a & @b */
struct tnum tnum_and(struct tnum a, struct tnum b);
/* Bitwise-OR, return @a | @b */

2
include/linux/uaccess.h
View file

@ -296,6 +296,8 @@ static inline bool pagefault_disabled(void)
*/
#define faulthandler_disabled() (pagefault_disabled() || in_atomic())
DEFINE_LOCK_GUARD_0(pagefault, pagefault_disable(), pagefault_enable())
#ifndef CONFIG_ARCH_HAS_SUBPAGE_FAULTS
/**

19
include/linux/usermode_driver.h
View file

@ -1,19 +0,0 @@
#ifndef __LINUX_USERMODE_DRIVER_H__
#define __LINUX_USERMODE_DRIVER_H__
#include <linux/umh.h>
#include <linux/path.h>
struct umd_info {
const char *driver_name;
struct file *pipe_to_umh;
struct file *pipe_from_umh;
struct path wd;
struct pid *tgid;
};
int umd_load_blob(struct umd_info *info, const void *data, size_t len);
int umd_unload_blob(struct umd_info *info);
int fork_usermode_driver(struct umd_info *info);
void umd_cleanup_helper(struct umd_info *info);
#endif /* __LINUX_USERMODE_DRIVER_H__ */

1
include/net/tcx.h
View file

@ -20,7 +20,6 @@ struct tcx_entry {
struct tcx_link {
struct bpf_link link;
struct net_device *dev;
u32 location;
};
static inline void tcx_set_ingress(struct sk_buff *skb, bool ingress)

45
include/uapi/linux/bpf.h
View file

@ -450,6 +450,7 @@ union bpf_iter_link_info {
* * **struct bpf_map_info**
* * **struct bpf_btf_info**
* * **struct bpf_link_info**
* * **struct bpf_token_info**
*
* Return
* Returns zero on success. On error, -1 is returned and *errno*
@ -906,6 +907,17 @@ union bpf_iter_link_info {
* A new file descriptor (a nonnegative integer), or -1 if an
* error occurred (in which case, *errno* is set appropriately).
*
* BPF_PROG_STREAM_READ_BY_FD
* Description
* Read data of a program's BPF stream. The program is identified
* by *prog_fd*, and the stream is identified by the *stream_id*.
* The data is copied to a buffer pointed to by *stream_buf*, and
* filled less than or equal to *stream_buf_len* bytes.
*
* Return
* Number of bytes read from the stream on success, or -1 if an
* error occurred (in which case, *errno* is set appropriately).
*
* NOTES
* eBPF objects (maps and programs) can be shared between processes.
*
@ -961,6 +973,7 @@ enum bpf_cmd {
BPF_LINK_DETACH,
BPF_PROG_BIND_MAP,
BPF_TOKEN_CREATE,
BPF_PROG_STREAM_READ_BY_FD,
__MAX_BPF_CMD,
};
@ -1463,6 +1476,11 @@ struct bpf_stack_build_id {
#define BPF_OBJ_NAME_LEN 16U
enum {
BPF_STREAM_STDOUT = 1,
BPF_STREAM_STDERR = 2,
};
union bpf_attr {
struct { /* anonymous struct used by BPF_MAP_CREATE command */
__u32 map_type; /* one of enum bpf_map_type */
@ -1794,6 +1812,13 @@ union bpf_attr {
};
__u64 expected_revision;
} netkit;
struct {
union {
__u32 relative_fd;
__u32 relative_id;
};
__u64 expected_revision;
} cgroup;
};
} link_create;
@ -1842,6 +1867,13 @@ union bpf_attr {
__u32 bpffs_fd;
} token_create;
struct {
__aligned_u64 stream_buf;
__u32 stream_buf_len;
__u32 stream_id;
__u32 prog_fd;
} prog_stream_read;
} __attribute__((aligned(8)));
/* The description below is an attempt at providing documentation to eBPF
@ -2403,7 +2435,7 @@ union bpf_attr {
* into it. An example is available in file
* *samples/bpf/trace_output_user.c* in the Linux kernel source
* tree (the eBPF program counterpart is in
* *samples/bpf/trace_output_kern.c*).
* *samples/bpf/trace_output.bpf.c*).
*
* **bpf_perf_event_output**\ () achieves better performance
* than **bpf_trace_printk**\ () for sharing data with user
@ -6653,11 +6685,15 @@ struct bpf_link_info {
struct {
__aligned_u64 tp_name; /* in/out: tp_name buffer ptr */
__u32 tp_name_len; /* in/out: tp_name buffer len */
__u32 :32;
__u64 cookie;
} raw_tracepoint;
struct {
__u32 attach_type;
__u32 target_obj_id; /* prog_id for PROG_EXT, otherwise btf object id */
__u32 target_btf_id; /* BTF type id inside the object */
__u32 :32;
__u64 cookie;
} tracing;
struct {
__u64 cgroup_id;
@ -6768,6 +6804,13 @@ struct bpf_link_info {
};
} __attribute__((aligned(8)));
struct bpf_token_info {
__u64 allowed_cmds;
__u64 allowed_maps;
__u64 allowed_progs;
__u64 allowed_attachs;
} __attribute__((aligned(8)));
/* User bpf_sock_addr struct to access socket fields and sockaddr struct passed
* by user and intended to be used by socket (e.g. to bind to, depends on
* attach type).

1
kernel/Makefile
View file

@ -12,7 +12,6 @@ obj-y = fork.o exec_domain.o panic.o \
notifier.o ksysfs.o cred.o reboot.o \
async.o range.o smpboot.o ucount.o regset.o ksyms_common.o
obj-$(CONFIG_USERMODE_DRIVER) += usermode_driver.o
obj-$(CONFIG_MULTIUSER) += groups.o
obj-$(CONFIG_VHOST_TASK) += vhost_task.o

2
kernel/bpf/Makefile
View file

@ -14,7 +14,7 @@ obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o
obj-${CONFIG_BPF_LSM} += bpf_inode_storage.o
obj-$(CONFIG_BPF_SYSCALL) += disasm.o mprog.o
obj-$(CONFIG_BPF_JIT) += trampoline.o
obj-$(CONFIG_BPF_SYSCALL) += btf.o memalloc.o rqspinlock.o
obj-$(CONFIG_BPF_SYSCALL) += btf.o memalloc.o rqspinlock.o stream.o
ifeq ($(CONFIG_MMU)$(CONFIG_64BIT),yy)
obj-$(CONFIG_BPF_SYSCALL) += arena.o range_tree.o
endif

43
kernel/bpf/arena.c
View file

@ -550,6 +550,34 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt)
}
}
/*
* Reserve an arena virtual address range without populating it. This call stops
* bpf_arena_alloc_pages from adding pages to this range.
*/
static int arena_reserve_pages(struct bpf_arena *arena, long uaddr, u32 page_cnt)
{
long page_cnt_max = (arena->user_vm_end - arena->user_vm_start) >> PAGE_SHIFT;
long pgoff;
int ret;
if (uaddr & ~PAGE_MASK)
return 0;
pgoff = compute_pgoff(arena, uaddr);
if (pgoff + page_cnt > page_cnt_max)
return -EINVAL;
guard(mutex)(&arena->lock);
/* Cannot guard already allocated pages. */
ret = is_range_tree_set(&arena->rt, pgoff, page_cnt);
if (ret)
return -EBUSY;
/* "Allocate" the region to prevent it from being allocated. */
return range_tree_clear(&arena->rt, pgoff, page_cnt);
}
__bpf_kfunc_start_defs();
__bpf_kfunc void *bpf_arena_alloc_pages(void *p__map, void *addr__ign, u32 page_cnt,
@ -573,11 +601,26 @@ __bpf_kfunc void bpf_arena_free_pages(void *p__map, void *ptr__ign, u32 page_cnt
return;
arena_free_pages(arena, (long)ptr__ign, page_cnt);
}
__bpf_kfunc int bpf_arena_reserve_pages(void *p__map, void *ptr__ign, u32 page_cnt)
{
struct bpf_map *map = p__map;
struct bpf_arena *arena = container_of(map, struct bpf_arena, map);
if (map->map_type != BPF_MAP_TYPE_ARENA)
return -EINVAL;
if (!page_cnt)
return 0;
return arena_reserve_pages(arena, (long)ptr__ign, page_cnt);
}
__bpf_kfunc_end_defs();
BTF_KFUNCS_START(arena_kfuncs)
BTF_ID_FLAGS(func, bpf_arena_alloc_pages, KF_TRUSTED_ARGS | KF_SLEEPABLE | KF_ARENA_RET | KF_ARENA_ARG2)
BTF_ID_FLAGS(func, bpf_arena_free_pages, KF_TRUSTED_ARGS | KF_SLEEPABLE | KF_ARENA_ARG2)
BTF_ID_FLAGS(func, bpf_arena_reserve_pages, KF_TRUSTED_ARGS | KF_SLEEPABLE | KF_ARENA_ARG2)
BTF_KFUNCS_END(arena_kfuncs)
static const struct btf_kfunc_id_set common_kfunc_set = {

11
kernel/bpf/arraymap.c
View file

@ -530,8 +530,6 @@ static int array_map_check_btf(const struct bpf_map *map,
const struct btf_type *key_type,
const struct btf_type *value_type)
{
u32 int_data;
/* One exception for keyless BTF: .bss/.data/.rodata map */
if (btf_type_is_void(key_type)) {
if (map->map_type != BPF_MAP_TYPE_ARRAY ||
@ -544,14 +542,11 @@ static int array_map_check_btf(const struct bpf_map *map,
return 0;
}
if (BTF_INFO_KIND(key_type->info) != BTF_KIND_INT)
return -EINVAL;
int_data = *(u32 *)(key_type + 1);
/* bpf array can only take a u32 key. This check makes sure
/*
* Bpf array can only take a u32 key. This check makes sure
* that the btf matches the attr used during map_create.
*/
if (BTF_INT_BITS(int_data) != 32 || BTF_INT_OFFSET(int_data))
if (!btf_type_is_i32(key_type))
return -EINVAL;
return 0;

14
kernel/bpf/bpf_iter.c
View file

@ -38,8 +38,7 @@ static DEFINE_MUTEX(link_mutex);
/* incremented on every opened seq_file */
static atomic64_t session_id;
static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
const struct bpf_iter_seq_info *seq_info);
static int prepare_seq_file(struct file *file, struct bpf_iter_link *link);
static void bpf_iter_inc_seq_num(struct seq_file *seq)
{
@ -257,7 +256,7 @@ static int iter_open(struct inode *inode, struct file *file)
{
struct bpf_iter_link *link = inode->i_private;
return prepare_seq_file(file, link, __get_seq_info(link));
return prepare_seq_file(file, link);
}
static int iter_release(struct inode *inode, struct file *file)
@ -553,7 +552,8 @@ int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr,
if (!link)
return -ENOMEM;
bpf_link_init(&link->link, BPF_LINK_TYPE_ITER, &bpf_iter_link_lops, prog);
bpf_link_init(&link->link, BPF_LINK_TYPE_ITER, &bpf_iter_link_lops, prog,
attr->link_create.attach_type);
link->tinfo = tinfo;
err = bpf_link_prime(&link->link, &link_primer);
@ -586,9 +586,9 @@ static void init_seq_meta(struct bpf_iter_priv_data *priv_data,
priv_data->done_stop = false;
}
static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
const struct bpf_iter_seq_info *seq_info)
static int prepare_seq_file(struct file *file, struct bpf_iter_link *link)
{
const struct bpf_iter_seq_info *seq_info = __get_seq_info(link);
struct bpf_iter_priv_data *priv_data;
struct bpf_iter_target_info *tinfo;
struct bpf_prog *prog;
@ -653,7 +653,7 @@ int bpf_iter_new_fd(struct bpf_link *link)
}
iter_link = container_of(link, struct bpf_iter_link, link);
err = prepare_seq_file(file, iter_link, __get_seq_info(iter_link));
err = prepare_seq_file(file, iter_link);
if (err)
goto free_file;

8
kernel/bpf/bpf_local_storage.c
View file

@ -722,13 +722,7 @@ int bpf_local_storage_map_check_btf(const struct bpf_map *map,
const struct btf_type *key_type,
const struct btf_type *value_type)
{
u32 int_data;
if (BTF_INFO_KIND(key_type->info) != BTF_KIND_INT)
return -EINVAL;
int_data = *(u32 *)(key_type + 1);
if (BTF_INT_BITS(int_data) != 32 || BTF_INT_OFFSET(int_data))
if (!btf_type_is_i32(key_type))
return -EINVAL;
return 0;

5
kernel/bpf/bpf_struct_ops.c
View file

@ -808,7 +808,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
goto reset_unlock;
}
bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS,
&bpf_struct_ops_link_lops, prog);
&bpf_struct_ops_link_lops, prog, prog->expected_attach_type);
*plink++ = &link->link;
ksym = kzalloc(sizeof(*ksym), GFP_USER);
@ -1351,7 +1351,8 @@ int bpf_struct_ops_link_create(union bpf_attr *attr)
err = -ENOMEM;
goto err_out;
}
bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS, &bpf_struct_ops_map_lops, NULL);
bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS, &bpf_struct_ops_map_lops, NULL,
attr->link_create.attach_type);
err = bpf_link_prime(&link->link, &link_primer);
if (err)

116
kernel/bpf/btf.c
View file

@ -858,26 +858,43 @@ const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id)
EXPORT_SYMBOL_GPL(btf_type_by_id);
/*
* Regular int is not a bit field and it must be either
* u8/u16/u32/u64 or __int128.
* Check that the type @t is a regular int. This means that @t is not
* a bit field and it has the same size as either of u8/u16/u32/u64
* or __int128. If @expected_size is not zero, then size of @t should
* be the same. A caller should already have checked that the type @t
* is an integer.
*/
static bool __btf_type_int_is_regular(const struct btf_type *t, size_t expected_size)
{
u32 int_data = btf_type_int(t);
u8 nr_bits = BTF_INT_BITS(int_data);
u8 nr_bytes = BITS_ROUNDUP_BYTES(nr_bits);
return BITS_PER_BYTE_MASKED(nr_bits) == 0 &&
BTF_INT_OFFSET(int_data) == 0 &&
(nr_bytes <= 16 && is_power_of_2(nr_bytes)) &&
(expected_size == 0 || nr_bytes == expected_size);
}
static bool btf_type_int_is_regular(const struct btf_type *t)
{
u8 nr_bits, nr_bytes;
u32 int_data;
return __btf_type_int_is_regular(t, 0);
}
int_data = btf_type_int(t);
nr_bits = BTF_INT_BITS(int_data);
nr_bytes = BITS_ROUNDUP_BYTES(nr_bits);
if (BITS_PER_BYTE_MASKED(nr_bits) ||
BTF_INT_OFFSET(int_data) ||
(nr_bytes != sizeof(u8) && nr_bytes != sizeof(u16) &&
nr_bytes != sizeof(u32) && nr_bytes != sizeof(u64) &&
nr_bytes != (2 * sizeof(u64)))) {
return false;
}
bool btf_type_is_i32(const struct btf_type *t)
{
return btf_type_is_int(t) && __btf_type_int_is_regular(t, 4);
}
return true;
bool btf_type_is_i64(const struct btf_type *t)
{
return btf_type_is_int(t) && __btf_type_int_is_regular(t, 8);
}
bool btf_type_is_primitive(const struct btf_type *t)
{
return (btf_type_is_int(t) && btf_type_int_is_regular(t)) ||
btf_is_any_enum(t);
}
/*
@ -3443,7 +3460,8 @@ btf_find_graph_root(const struct btf *btf, const struct btf_type *pt,
node_field_name = strstr(value_type, ":");
if (!node_field_name)
return -EINVAL;
value_type = kstrndup(value_type, node_field_name - value_type, GFP_KERNEL | __GFP_NOWARN);
value_type = kstrndup(value_type, node_field_name - value_type,
GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
if (!value_type)
return -ENOMEM;
id = btf_find_by_name_kind(btf, value_type, BTF_KIND_STRUCT);
@ -3958,7 +3976,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type
/* This needs to be kzalloc to zero out padding and unused fields, see
* comment in btf_record_equal.
*/
rec = kzalloc(struct_size(rec, fields, cnt), GFP_KERNEL | __GFP_NOWARN);
rec = kzalloc(struct_size(rec, fields, cnt), GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
if (!rec)
return ERR_PTR(-ENOMEM);
@ -6182,8 +6200,7 @@ int get_kern_ctx_btf_id(struct bpf_verifier_log *log, enum bpf_prog_type prog_ty
return kctx_type_id;
}
BTF_ID_LIST(bpf_ctx_convert_btf_id)
BTF_ID(struct, bpf_ctx_convert)
BTF_ID_LIST_SINGLE(bpf_ctx_convert_btf_id, struct, bpf_ctx_convert)
static struct btf *btf_parse_base(struct btf_verifier_env *env, const char *name,
void *data, unsigned int data_size)
@ -6903,6 +6920,7 @@ enum bpf_struct_walk_result {
/* < 0 error */
WALK_SCALAR = 0,
WALK_PTR,
WALK_PTR_UNTRUSTED,
WALK_STRUCT,
};
@ -7144,6 +7162,8 @@ error:
*field_name = mname;
return WALK_PTR;
}
return WALK_PTR_UNTRUSTED;
}
/* Allow more flexible access within an int as long as
@ -7216,6 +7236,9 @@ int btf_struct_access(struct bpf_verifier_log *log,
*next_btf_id = id;
*flag = tmp_flag;
return PTR_TO_BTF_ID;
case WALK_PTR_UNTRUSTED:
*flag = MEM_RDONLY | PTR_UNTRUSTED;
return PTR_TO_MEM;
case WALK_SCALAR:
return SCALAR_VALUE;
case WALK_STRUCT:
@ -7628,11 +7651,12 @@ cand_cache_unlock:
}
enum btf_arg_tag {
ARG_TAG_CTX = BIT_ULL(0),
ARG_TAG_NONNULL = BIT_ULL(1),
ARG_TAG_TRUSTED = BIT_ULL(2),
ARG_TAG_NULLABLE = BIT_ULL(3),
ARG_TAG_ARENA = BIT_ULL(4),
ARG_TAG_CTX = BIT_ULL(0),
ARG_TAG_NONNULL = BIT_ULL(1),
ARG_TAG_TRUSTED = BIT_ULL(2),
ARG_TAG_UNTRUSTED = BIT_ULL(3),
ARG_TAG_NULLABLE = BIT_ULL(4),
ARG_TAG_ARENA = BIT_ULL(5),
};
/* Process BTF of a function to produce high-level expectation of function
@ -7740,6 +7764,8 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
tags |= ARG_TAG_CTX;
} else if (strcmp(tag, "trusted") == 0) {
tags |= ARG_TAG_TRUSTED;
} else if (strcmp(tag, "untrusted") == 0) {
tags |= ARG_TAG_UNTRUSTED;
} else if (strcmp(tag, "nonnull") == 0) {
tags |= ARG_TAG_NONNULL;
} else if (strcmp(tag, "nullable") == 0) {
@ -7800,6 +7826,38 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
sub->args[i].btf_id = kern_type_id;
continue;
}
if (tags & ARG_TAG_UNTRUSTED) {
struct btf *vmlinux_btf;
int kern_type_id;
if (tags & ~ARG_TAG_UNTRUSTED) {
bpf_log(log, "arg#%d untrusted cannot be combined with any other tags\n", i);
return -EINVAL;
}
ref_t = btf_type_skip_modifiers(btf, t->type, NULL);
if (btf_type_is_void(ref_t) || btf_type_is_primitive(ref_t)) {
sub->args[i].arg_type = ARG_PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED;
sub->args[i].mem_size = 0;
continue;
}
kern_type_id = btf_get_ptr_to_btf_id(log, i, btf, t);
if (kern_type_id < 0)
return kern_type_id;
vmlinux_btf = bpf_get_btf_vmlinux();
ref_t = btf_type_by_id(vmlinux_btf, kern_type_id);
if (!btf_type_is_struct(ref_t)) {
tname = __btf_name_by_offset(vmlinux_btf, t->name_off);
bpf_log(log, "arg#%d has type %s '%s', but only struct or primitive types are allowed\n",
i, btf_type_str(ref_t), tname);
return -EINVAL;
}
sub->args[i].arg_type = ARG_PTR_TO_BTF_ID | PTR_UNTRUSTED;
sub->args[i].btf_id = kern_type_id;
continue;
}
if (tags & ARG_TAG_ARENA) {
if (tags & ~ARG_TAG_ARENA) {
bpf_log(log, "arg#%d arena cannot be combined with any other tags\n", i);
@ -9019,7 +9077,7 @@ static struct bpf_cand_cache *populate_cand_cache(struct bpf_cand_cache *cands,
bpf_free_cands_from_cache(*cc);
*cc = NULL;
}
new_cands = kmemdup(cands, sizeof_cands(cands->cnt), GFP_KERNEL);
new_cands = kmemdup(cands, sizeof_cands(cands->cnt), GFP_KERNEL_ACCOUNT);
if (!new_cands) {
bpf_free_cands(cands);
return ERR_PTR(-ENOMEM);
@ -9027,7 +9085,7 @@ static struct bpf_cand_cache *populate_cand_cache(struct bpf_cand_cache *cands,
/* strdup the name, since it will stay in cache.
* the cands->name points to strings in prog's BTF and the prog can be unloaded.
*/
new_cands->name = kmemdup_nul(cands->name, cands->name_len, GFP_KERNEL);
new_cands->name = kmemdup_nul(cands->name, cands->name_len, GFP_KERNEL_ACCOUNT);
bpf_free_cands(cands);
if (!new_cands->name) {
kfree(new_cands);
@ -9111,7 +9169,7 @@ bpf_core_add_cands(struct bpf_cand_cache *cands, const struct btf *targ_btf,
continue;
/* most of the time there is only one candidate for a given kind+name pair */
new_cands = kmalloc(sizeof_cands(cands->cnt + 1), GFP_KERNEL);
new_cands = kmalloc(sizeof_cands(cands->cnt + 1), GFP_KERNEL_ACCOUNT);
if (!new_cands) {
bpf_free_cands(cands);
return ERR_PTR(-ENOMEM);
@ -9228,7 +9286,7 @@ int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo,
/* ~4k of temp memory necessary to convert LLVM spec like "0:1:0:5"
* into arrays of btf_ids of struct fields and array indices.
*/
specs = kcalloc(3, sizeof(*specs), GFP_KERNEL);
specs = kcalloc(3, sizeof(*specs), GFP_KERNEL_ACCOUNT);
if (!specs)
return -ENOMEM;
@ -9253,7 +9311,7 @@ int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo,
goto out;
}
if (cc->cnt) {
cands.cands = kcalloc(cc->cnt, sizeof(*cands.cands), GFP_KERNEL);
cands.cands = kcalloc(cc->cnt, sizeof(*cands.cands), GFP_KERNEL_ACCOUNT);
if (!cands.cands) {
err = -ENOMEM;
goto out;

195
kernel/bpf/cgroup.c
View file

@ -658,6 +658,116 @@ static struct bpf_prog_list *find_attach_entry(struct hlist_head *progs,
return NULL;
}
static struct bpf_link *bpf_get_anchor_link(u32 flags, u32 id_or_fd)
{
struct bpf_link *link = ERR_PTR(-EINVAL);
if (flags & BPF_F_ID)
link = bpf_link_by_id(id_or_fd);
else if (id_or_fd)
link = bpf_link_get_from_fd(id_or_fd);
return link;
}
static struct bpf_prog *bpf_get_anchor_prog(u32 flags, u32 id_or_fd)
{
struct bpf_prog *prog = ERR_PTR(-EINVAL);
if (flags & BPF_F_ID)
prog = bpf_prog_by_id(id_or_fd);
else if (id_or_fd)
prog = bpf_prog_get(id_or_fd);
return prog;
}
static struct bpf_prog_list *get_prog_list(struct hlist_head *progs, struct bpf_prog *prog,
struct bpf_cgroup_link *link, u32 flags, u32 id_or_fd)
{
bool is_link = flags & BPF_F_LINK, is_id = flags & BPF_F_ID;
struct bpf_prog_list *pltmp, *pl = ERR_PTR(-EINVAL);
bool preorder = flags & BPF_F_PREORDER;
struct bpf_link *anchor_link = NULL;
struct bpf_prog *anchor_prog = NULL;
bool is_before, is_after;
is_before = flags & BPF_F_BEFORE;
is_after = flags & BPF_F_AFTER;
if (is_link || is_id || id_or_fd) {
/* flags must have either BPF_F_BEFORE or BPF_F_AFTER */
if (is_before == is_after)
return ERR_PTR(-EINVAL);
if ((is_link && !link) || (!is_link && !prog))
return ERR_PTR(-EINVAL);
} else if (!hlist_empty(progs)) {
/* flags cannot have both BPF_F_BEFORE and BPF_F_AFTER */
if (is_before && is_after)
return ERR_PTR(-EINVAL);
}
if (is_link) {
anchor_link = bpf_get_anchor_link(flags, id_or_fd);
if (IS_ERR(anchor_link))
return ERR_CAST(anchor_link);
} else if (is_id || id_or_fd) {
anchor_prog = bpf_get_anchor_prog(flags, id_or_fd);
if (IS_ERR(anchor_prog))
return ERR_CAST(anchor_prog);
}
if (!anchor_prog && !anchor_link) {
/* if there is no anchor_prog/anchor_link, then BPF_F_PREORDER
* doesn't matter since either prepend or append to a combined
* list of progs will end up with correct result.
*/
hlist_for_each_entry(pltmp, progs, node) {
if (is_before)
return pltmp;
if (pltmp->node.next)
continue;
return pltmp;
}
return NULL;
}
hlist_for_each_entry(pltmp, progs, node) {
if ((anchor_prog && anchor_prog == pltmp->prog) ||
(anchor_link && anchor_link == &pltmp->link->link)) {
if (!!(pltmp->flags & BPF_F_PREORDER) != preorder)
goto out;
pl = pltmp;
goto out;
}
}
pl = ERR_PTR(-ENOENT);
out:
if (anchor_link)
bpf_link_put(anchor_link);
else
bpf_prog_put(anchor_prog);
return pl;
}
static int insert_pl_to_hlist(struct bpf_prog_list *pl, struct hlist_head *progs,
struct bpf_prog *prog, struct bpf_cgroup_link *link,
u32 flags, u32 id_or_fd)
{
struct bpf_prog_list *pltmp;
pltmp = get_prog_list(progs, prog, link, flags, id_or_fd);
if (IS_ERR(pltmp))
return PTR_ERR(pltmp);
if (!pltmp)
hlist_add_head(&pl->node, progs);
else if (flags & BPF_F_BEFORE)
hlist_add_before(&pl->node, &pltmp->node);
else
hlist_add_behind(&pl->node, &pltmp->node);
return 0;
}
/**
* __cgroup_bpf_attach() - Attach the program or the link to a cgroup, and
* propagate the change to descendants
@ -667,6 +777,8 @@ static struct bpf_prog_list *find_attach_entry(struct hlist_head *progs,
* @replace_prog: Previously attached program to replace if BPF_F_REPLACE is set
* @type: Type of attach operation
* @flags: Option flags
* @id_or_fd: Relative prog id or fd
* @revision: bpf_prog_list revision
*
* Exactly one of @prog or @link can be non-null.
* Must be called with cgroup_mutex held.
@ -674,7 +786,8 @@ static struct bpf_prog_list *find_attach_entry(struct hlist_head *progs,
static int __cgroup_bpf_attach(struct cgroup *cgrp,
struct bpf_prog *prog, struct bpf_prog *replace_prog,
struct bpf_cgroup_link *link,
enum bpf_attach_type type, u32 flags)
enum bpf_attach_type type, u32 flags, u32 id_or_fd,
u64 revision)
{
u32 saved_flags = (flags & (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI));
struct bpf_prog *old_prog = NULL;
@ -690,6 +803,9 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp,
((flags & BPF_F_REPLACE) && !(flags & BPF_F_ALLOW_MULTI)))
/* invalid combination */
return -EINVAL;
if ((flags & BPF_F_REPLACE) && (flags & (BPF_F_BEFORE | BPF_F_AFTER)))
/* only either replace or insertion with before/after */
return -EINVAL;
if (link && (prog || replace_prog))
/* only either link or prog/replace_prog can be specified */
return -EINVAL;
@ -700,6 +816,8 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp,
atype = bpf_cgroup_atype_find(type, new_prog->aux->attach_btf_id);
if (atype < 0)
return -EINVAL;
if (revision && revision != cgrp->bpf.revisions[atype])
return -ESTALE;
progs = &cgrp->bpf.progs[atype];
@ -728,22 +846,18 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp,
if (pl) {
old_prog = pl->prog;
} else {
struct hlist_node *last = NULL;
pl = kmalloc(sizeof(*pl), GFP_KERNEL);
if (!pl) {
bpf_cgroup_storages_free(new_storage);
return -ENOMEM;
}
if (hlist_empty(progs))
hlist_add_head(&pl->node, progs);
else
hlist_for_each(last, progs) {
if (last->next)
continue;
hlist_add_behind(&pl->node, last);
break;
}
err = insert_pl_to_hlist(pl, progs, prog, link, flags, id_or_fd);
if (err) {
kfree(pl);
bpf_cgroup_storages_free(new_storage);
return err;
}
}
pl->prog = prog;
@ -753,7 +867,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp,
cgrp->bpf.flags[atype] = saved_flags;
if (type == BPF_LSM_CGROUP) {
err = bpf_trampoline_link_cgroup_shim(new_prog, atype);
err = bpf_trampoline_link_cgroup_shim(new_prog, atype, type);
if (err)
goto cleanup;
}
@ -762,6 +876,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp,
if (err)
goto cleanup_trampoline;
cgrp->bpf.revisions[atype] += 1;
if (old_prog) {
if (type == BPF_LSM_CGROUP)
bpf_trampoline_unlink_cgroup_shim(old_prog);
@ -793,12 +908,13 @@ static int cgroup_bpf_attach(struct cgroup *cgrp,
struct bpf_prog *prog, struct bpf_prog *replace_prog,
struct bpf_cgroup_link *link,
enum bpf_attach_type type,
u32 flags)
u32 flags, u32 id_or_fd, u64 revision)
{
int ret;
cgroup_lock();
ret = __cgroup_bpf_attach(cgrp, prog, replace_prog, link, type, flags);
ret = __cgroup_bpf_attach(cgrp, prog, replace_prog, link, type, flags,
id_or_fd, revision);
cgroup_unlock();
return ret;
}
@ -868,7 +984,7 @@ static int __cgroup_bpf_replace(struct cgroup *cgrp,
struct hlist_head *progs;
bool found = false;
atype = bpf_cgroup_atype_find(link->type, new_prog->aux->attach_btf_id);
atype = bpf_cgroup_atype_find(link->link.attach_type, new_prog->aux->attach_btf_id);
if (atype < 0)
return -EINVAL;
@ -886,6 +1002,7 @@ static int __cgroup_bpf_replace(struct cgroup *cgrp,
if (!found)
return -ENOENT;
cgrp->bpf.revisions[atype] += 1;
old_prog = xchg(&link->link.prog, new_prog);
replace_effective_prog(cgrp, atype, link);
bpf_prog_put(old_prog);
@ -1011,12 +1128,14 @@ found:
* @prog: A program to detach or NULL
* @link: A link to detach or NULL
* @type: Type of detach operation
* @revision: bpf_prog_list revision
*
* At most one of @prog or @link can be non-NULL.
* Must be called with cgroup_mutex held.
*/
static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
struct bpf_cgroup_link *link, enum bpf_attach_type type)
struct bpf_cgroup_link *link, enum bpf_attach_type type,
u64 revision)
{
enum cgroup_bpf_attach_type atype;
struct bpf_prog *old_prog;
@ -1034,6 +1153,9 @@ static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
if (atype < 0)
return -EINVAL;
if (revision && revision != cgrp->bpf.revisions[atype])
return -ESTALE;
progs = &cgrp->bpf.progs[atype];
flags = cgrp->bpf.flags[atype];
@ -1059,6 +1181,7 @@ static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
/* now can actually delete it from this cgroup list */
hlist_del(&pl->node);
cgrp->bpf.revisions[atype] += 1;
kfree(pl);
if (hlist_empty(progs))
@ -1074,12 +1197,12 @@ static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
}
static int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
enum bpf_attach_type type)
enum bpf_attach_type type, u64 revision)
{
int ret;
cgroup_lock();
ret = __cgroup_bpf_detach(cgrp, prog, NULL, type);
ret = __cgroup_bpf_detach(cgrp, prog, NULL, type, revision);
cgroup_unlock();
return ret;
}
@ -1097,6 +1220,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
struct bpf_prog_array *effective;
int cnt, ret = 0, i;
int total_cnt = 0;
u64 revision = 0;
u32 flags;
if (effective_query && prog_attach_flags)
@ -1134,6 +1258,10 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
return -EFAULT;
if (copy_to_user(&uattr->query.prog_cnt, &total_cnt, sizeof(total_cnt)))
return -EFAULT;
if (!effective_query && from_atype == to_atype)
revision = cgrp->bpf.revisions[from_atype];
if (copy_to_user(&uattr->query.revision, &revision, sizeof(revision)))
return -EFAULT;
if (attr->query.prog_cnt == 0 || !prog_ids || !total_cnt)
/* return early if user requested only program count + flags */
return 0;
@ -1216,7 +1344,8 @@ int cgroup_bpf_prog_attach(const union bpf_attr *attr,
}
ret = cgroup_bpf_attach(cgrp, prog, replace_prog, NULL,
attr->attach_type, attr->attach_flags);
attr->attach_type, attr->attach_flags,
attr->relative_fd, attr->expected_revision);
if (replace_prog)
bpf_prog_put(replace_prog);
@ -1238,7 +1367,7 @@ int cgroup_bpf_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype)
if (IS_ERR(prog))
prog = NULL;
ret = cgroup_bpf_detach(cgrp, prog, attr->attach_type);
ret = cgroup_bpf_detach(cgrp, prog, attr->attach_type, attr->expected_revision);
if (prog)
bpf_prog_put(prog);
@ -1267,8 +1396,8 @@ static void bpf_cgroup_link_release(struct bpf_link *link)
}
WARN_ON(__cgroup_bpf_detach(cg_link->cgroup, NULL, cg_link,
cg_link->type));
if (cg_link->type == BPF_LSM_CGROUP)
link->attach_type, 0));
if (link->attach_type == BPF_LSM_CGROUP)
bpf_trampoline_unlink_cgroup_shim(cg_link->link.prog);
cg = cg_link->cgroup;
@ -1310,7 +1439,7 @@ static void bpf_cgroup_link_show_fdinfo(const struct bpf_link *link,
"cgroup_id:\t%llu\n"
"attach_type:\t%d\n",
cg_id,
cg_link->type);
link->attach_type);
}
static int bpf_cgroup_link_fill_link_info(const struct bpf_link *link,
@ -1326,7 +1455,7 @@ static int bpf_cgroup_link_fill_link_info(const struct bpf_link *link,
cgroup_unlock();
info->cgroup.cgroup_id = cg_id;
info->cgroup.attach_type = cg_link->type;
info->cgroup.attach_type = link->attach_type;
return 0;
}
@ -1339,6 +1468,13 @@ static const struct bpf_link_ops bpf_cgroup_link_lops = {
.fill_link_info = bpf_cgroup_link_fill_link_info,
};
#define BPF_F_LINK_ATTACH_MASK \
(BPF_F_ID | \
BPF_F_BEFORE | \
BPF_F_AFTER | \
BPF_F_PREORDER | \
BPF_F_LINK)
int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
{
struct bpf_link_primer link_primer;
@ -1346,7 +1482,7 @@ int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
struct cgroup *cgrp;
int err;
if (attr->link_create.flags)
if (attr->link_create.flags & (~BPF_F_LINK_ATTACH_MASK))
return -EINVAL;
cgrp = cgroup_get_from_fd(attr->link_create.target_fd);
@ -1359,9 +1495,8 @@ int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
goto out_put_cgroup;
}
bpf_link_init(&link->link, BPF_LINK_TYPE_CGROUP, &bpf_cgroup_link_lops,
prog);
prog, attr->link_create.attach_type);
link->cgroup = cgrp;
link->type = attr->link_create.attach_type;
err = bpf_link_prime(&link->link, &link_primer);
if (err) {
@ -1370,7 +1505,9 @@ int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
}
err = cgroup_bpf_attach(cgrp, NULL, NULL, link,
link->type, BPF_F_ALLOW_MULTI);
link->link.attach_type, BPF_F_ALLOW_MULTI | attr->link_create.flags,
attr->link_create.cgroup.relative_fd,
attr->link_create.cgroup.expected_revision);
if (err) {
bpf_link_cleanup(&link_primer);
goto out_put_cgroup;

151
kernel/bpf/core.c
View file

@ -134,6 +134,10 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag
mutex_init(&fp->aux->ext_mutex);
mutex_init(&fp->aux->dst_mutex);
#ifdef CONFIG_BPF_SYSCALL
bpf_prog_stream_init(fp);
#endif
return fp;
}
@ -778,7 +782,10 @@ bool is_bpf_text_address(unsigned long addr)
struct bpf_prog *bpf_prog_ksym_find(unsigned long addr)
{
struct bpf_ksym *ksym = bpf_ksym_find(addr);
struct bpf_ksym *ksym;
WARN_ON_ONCE(!rcu_read_lock_held());
ksym = bpf_ksym_find(addr);
return ksym && ksym->prog ?
container_of(ksym, struct bpf_prog_aux, ksym)->prog :
@ -1290,6 +1297,13 @@ int bpf_jit_get_func_addr(const struct bpf_prog *prog,
return 0;
}
const char *bpf_jit_get_prog_name(struct bpf_prog *prog)
{
if (prog->aux->ksym.prog)
return prog->aux->ksym.name;
return prog->aux->name;
}
static int bpf_jit_blind_insn(const struct bpf_insn *from,
const struct bpf_insn *aux,
struct bpf_insn *to_buff,
@ -2102,14 +2116,15 @@ out:
#undef COND_JMP
/* ST, STX and LDX*/
ST_NOSPEC:
/* Speculation barrier for mitigating Speculative Store Bypass.
* In case of arm64, we rely on the firmware mitigation as
* controlled via the ssbd kernel parameter. Whenever the
* mitigation is enabled, it works for all of the kernel code
* with no need to provide any additional instructions here.
* In case of x86, we use 'lfence' insn for mitigation. We
* reuse preexisting logic from Spectre v1 mitigation that
* happens to produce the required code on x86 for v4 as well.
/* Speculation barrier for mitigating Speculative Store Bypass,
* Bounds-Check Bypass and Type Confusion. In case of arm64, we
* rely on the firmware mitigation as controlled via the ssbd
* kernel parameter. Whenever the mitigation is enabled, it
* works for all of the kernel code with no need to provide any
* additional instructions here. In case of x86, we use 'lfence'
* insn for mitigation. We reuse preexisting logic from Spectre
* v1 mitigation that happens to produce the required code on
* x86 for v4 as well.
*/
barrier_nospec();
CONT;
@ -2861,6 +2876,7 @@ static void bpf_prog_free_deferred(struct work_struct *work)
aux = container_of(work, struct bpf_prog_aux, work);
#ifdef CONFIG_BPF_SYSCALL
bpf_free_kfunc_btf_tab(aux->kfunc_btf_tab);
bpf_prog_stream_free(aux->prog);
#endif
#ifdef CONFIG_CGROUP_BPF
if (aux->cgroup_atype != CGROUP_BPF_ATTACH_TYPE_INVALID)
@ -3034,6 +3050,21 @@ bool __weak bpf_jit_needs_zext(void)
return false;
}
/* By default, enable the verifier's mitigations against Spectre v1 and v4 for
* all archs. The value returned must not change at runtime as there is
* currently no support for reloading programs that were loaded without
* mitigations.
*/
bool __weak bpf_jit_bypass_spec_v1(void)
{
return false;
}
bool __weak bpf_jit_bypass_spec_v4(void)
{
return false;
}
/* Return true if the JIT inlines the call to the helper corresponding to
* the imm.
*
@ -3144,6 +3175,22 @@ u64 __weak arch_bpf_timed_may_goto(void)
return 0;
}
static noinline void bpf_prog_report_may_goto_violation(void)
{
#ifdef CONFIG_BPF_SYSCALL
struct bpf_stream_stage ss;
struct bpf_prog *prog;
prog = bpf_prog_find_from_stack();
if (!prog)
return;
bpf_stream_stage(ss, prog, BPF_STDERR, ({
bpf_stream_printk(ss, "ERROR: Timeout detected for may_goto instruction\n");
bpf_stream_dump_stack(ss);
}));
#endif
}
u64 bpf_check_timed_may_goto(struct bpf_timed_may_goto *p)
{
u64 time = ktime_get_mono_fast_ns();
@ -3154,8 +3201,10 @@ u64 bpf_check_timed_may_goto(struct bpf_timed_may_goto *p)
return BPF_MAX_TIMED_LOOPS;
}
/* Check if we've exhausted our time slice, and zero count. */
if (time - p->timestamp >= (NSEC_PER_SEC / 4))
if (unlikely(time - p->timestamp >= (NSEC_PER_SEC / 4))) {
bpf_prog_report_may_goto_violation();
return 0;
}
/* Refresh the count for the stack frame. */
return BPF_MAX_TIMED_LOOPS;
}
@ -3192,3 +3241,85 @@ EXPORT_SYMBOL(bpf_stats_enabled_key);
EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_exception);
EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_bulk_tx);
#ifdef CONFIG_BPF_SYSCALL
int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep,
const char **linep, int *nump)
{
int idx = -1, insn_start, insn_end, len;
struct bpf_line_info *linfo;
void **jited_linfo;
struct btf *btf;
int nr_linfo;
btf = prog->aux->btf;
linfo = prog->aux->linfo;
jited_linfo = prog->aux->jited_linfo;
if (!btf || !linfo || !jited_linfo)
return -EINVAL;
len = prog->aux->func ? prog->aux->func[prog->aux->func_idx]->len : prog->len;
linfo = &prog->aux->linfo[prog->aux->linfo_idx];
jited_linfo = &prog->aux->jited_linfo[prog->aux->linfo_idx];
insn_start = linfo[0].insn_off;
insn_end = insn_start + len;
nr_linfo = prog->aux->nr_linfo - prog->aux->linfo_idx;
for (int i = 0; i < nr_linfo &&
linfo[i].insn_off >= insn_start && linfo[i].insn_off < insn_end; i++) {
if (jited_linfo[i] >= (void *)ip)
break;
idx = i;
}
if (idx == -1)
return -ENOENT;
/* Get base component of the file path. */
*filep = btf_name_by_offset(btf, linfo[idx].file_name_off);
*filep = kbasename(*filep);
/* Obtain the source line, and strip whitespace in prefix. */
*linep = btf_name_by_offset(btf, linfo[idx].line_off);
while (isspace(**linep))
*linep += 1;
*nump = BPF_LINE_INFO_LINE_NUM(linfo[idx].line_col);
return 0;
}
struct walk_stack_ctx {
struct bpf_prog *prog;
};
static bool find_from_stack_cb(void *cookie, u64 ip, u64 sp, u64 bp)
{
struct walk_stack_ctx *ctxp = cookie;
struct bpf_prog *prog;
/*
* The RCU read lock is held to safely traverse the latch tree, but we
* don't need its protection when accessing the prog, since it has an
* active stack frame on the current stack trace, and won't disappear.
*/
rcu_read_lock();
prog = bpf_prog_ksym_find(ip);
rcu_read_unlock();
if (!prog)
return true;
if (bpf_is_subprog(prog))
return true;
ctxp->prog = prog;
return false;
}
struct bpf_prog *bpf_prog_find_from_stack(void)
{
struct walk_stack_ctx ctx = {};
arch_bpf_stack_walk(find_from_stack_cb, &ctx);
return ctx.prog;
}
#endif

469
kernel/bpf/helpers.c
View file

@ -24,6 +24,7 @@
#include <linux/bpf_mem_alloc.h>
#include <linux/kasan.h>
#include <linux/bpf_verifier.h>
#include <linux/uaccess.h>
#include "../../lib/kstrtox.h"
@ -763,22 +764,13 @@ static int bpf_trace_copy_string(char *buf, void *unsafe_ptr, char fmt_ptype,
return -EINVAL;
}
/* Per-cpu temp buffers used by printf-like helpers to store the bprintf binary
* arguments representation.
*/
#define MAX_BPRINTF_BIN_ARGS 512
/* Support executing three nested bprintf helper calls on a given CPU */
#define MAX_BPRINTF_NEST_LEVEL 3
struct bpf_bprintf_buffers {
char bin_args[MAX_BPRINTF_BIN_ARGS];
char buf[MAX_BPRINTF_BUF];
};
static DEFINE_PER_CPU(struct bpf_bprintf_buffers[MAX_BPRINTF_NEST_LEVEL], bpf_bprintf_bufs);
static DEFINE_PER_CPU(int, bpf_bprintf_nest_level);
static int try_get_buffers(struct bpf_bprintf_buffers **bufs)
int bpf_try_get_buffers(struct bpf_bprintf_buffers **bufs)
{
int nest_level;
@ -794,16 +786,21 @@ static int try_get_buffers(struct bpf_bprintf_buffers **bufs)
return 0;
}
void bpf_bprintf_cleanup(struct bpf_bprintf_data *data)
void bpf_put_buffers(void)
{
if (!data->bin_args && !data->buf)
return;
if (WARN_ON_ONCE(this_cpu_read(bpf_bprintf_nest_level) == 0))
return;
this_cpu_dec(bpf_bprintf_nest_level);
preempt_enable();
}
void bpf_bprintf_cleanup(struct bpf_bprintf_data *data)
{
if (!data->bin_args && !data->buf)
return;
bpf_put_buffers();
}
/*
* bpf_bprintf_prepare - Generic pass on format strings for bprintf-like helpers
*
@ -818,7 +815,7 @@ void bpf_bprintf_cleanup(struct bpf_bprintf_data *data)
* In argument preparation mode, if 0 is returned, safe temporary buffers are
* allocated and bpf_bprintf_cleanup should be called to free them after use.
*/
int bpf_bprintf_prepare(char *fmt, u32 fmt_size, const u64 *raw_args,
int bpf_bprintf_prepare(const char *fmt, u32 fmt_size, const u64 *raw_args,
u32 num_args, struct bpf_bprintf_data *data)
{
bool get_buffers = (data->get_bin_args && num_args) || data->get_buf;
@ -834,7 +831,7 @@ int bpf_bprintf_prepare(char *fmt, u32 fmt_size, const u64 *raw_args,
return -EINVAL;
fmt_size = fmt_end - fmt;
if (get_buffers && try_get_buffers(&buffers))
if (get_buffers && bpf_try_get_buffers(&buffers))
return -EBUSY;
if (data->get_bin_args) {
@ -2911,6 +2908,52 @@ __bpf_kfunc int bpf_dynptr_copy(struct bpf_dynptr *dst_ptr, u32 dst_off,
return 0;
}
/**
* bpf_dynptr_memset() - Fill dynptr memory with a constant byte.
* @p: Destination dynptr - where data will be filled
* @offset: Offset into the dynptr to start filling from
* @size: Number of bytes to fill
* @val: Constant byte to fill the memory with
*
* Fills the @size bytes of the memory area pointed to by @p
* at @offset with the constant byte @val.
* Returns 0 on success; negative error, otherwise.
*/
__bpf_kfunc int bpf_dynptr_memset(struct bpf_dynptr *p, u32 offset, u32 size, u8 val)
{
struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p;
u32 chunk_sz, write_off;
char buf[256];
void* slice;
int err;
slice = bpf_dynptr_slice_rdwr(p, offset, NULL, size);
if (likely(slice)) {
memset(slice, val, size);
return 0;
}
if (__bpf_dynptr_is_rdonly(ptr))
return -EINVAL;
err = bpf_dynptr_check_off_len(ptr, offset, size);
if (err)
return err;
/* Non-linear data under the dynptr, write from a local buffer */
chunk_sz = min_t(u32, sizeof(buf), size);
memset(buf, val, chunk_sz);
for (write_off = 0; write_off < size; write_off += chunk_sz) {
chunk_sz = min_t(u32, sizeof(buf), size - write_off);
err = __bpf_dynptr_write(ptr, offset + write_off, buf, chunk_sz, 0);
if (err)
return err;
}
return 0;
}
__bpf_kfunc void *bpf_cast_to_kern_ctx(void *obj)
{
return obj;
@ -2943,9 +2986,16 @@ static bool bpf_stack_walker(void *cookie, u64 ip, u64 sp, u64 bp)
struct bpf_throw_ctx *ctx = cookie;
struct bpf_prog *prog;
if (!is_bpf_text_address(ip))
return !ctx->cnt;
/*
* The RCU read lock is held to safely traverse the latch tree, but we
* don't need its protection when accessing the prog, since it has an
* active stack frame on the current stack trace, and won't disappear.
*/
rcu_read_lock();
prog = bpf_prog_ksym_find(ip);
rcu_read_unlock();
if (!prog)
return !ctx->cnt;
ctx->cnt++;
if (bpf_is_subprog(prog))
return true;
@ -3283,6 +3333,376 @@ __bpf_kfunc void __bpf_trap(void)
{
}
/*
* Kfuncs for string operations.
*
* Since strings are not necessarily %NUL-terminated, we cannot directly call
* in-kernel implementations. Instead, we open-code the implementations using
* __get_kernel_nofault instead of plain dereference to make them safe.
*/
/**
* bpf_strcmp - Compare two strings
* @s1__ign: One string
* @s2__ign: Another string
*
* Return:
* * %0 - Strings are equal
* * %-1 - @s1__ign is smaller
* * %1 - @s2__ign is smaller
* * %-EFAULT - Cannot read one of the strings
* * %-E2BIG - One of strings is too large
* * %-ERANGE - One of strings is outside of kernel address space
*/
__bpf_kfunc int bpf_strcmp(const char *s1__ign, const char *s2__ign)
{
char c1, c2;
int i;
if (!copy_from_kernel_nofault_allowed(s1__ign, 1) ||
!copy_from_kernel_nofault_allowed(s2__ign, 1)) {
return -ERANGE;
}
guard(pagefault)();
for (i = 0; i < XATTR_SIZE_MAX; i++) {
__get_kernel_nofault(&c1, s1__ign, char, err_out);
__get_kernel_nofault(&c2, s2__ign, char, err_out);
if (c1 != c2)
return c1 < c2 ? -1 : 1;
if (c1 == '\0')
return 0;
s1__ign++;
s2__ign++;
}
return -E2BIG;
err_out:
return -EFAULT;
}
/**
* bpf_strnchr - Find a character in a length limited string
* @s__ign: The string to be searched
* @count: The number of characters to be searched
* @c: The character to search for
*
* Note that the %NUL-terminator is considered part of the string, and can
* be searched for.
*
* Return:
* * >=0 - Index of the first occurrence of @c within @s__ign
* * %-ENOENT - @c not found in the first @count characters of @s__ign
* * %-EFAULT - Cannot read @s__ign
* * %-E2BIG - @s__ign is too large
* * %-ERANGE - @s__ign is outside of kernel address space
*/
__bpf_kfunc int bpf_strnchr(const char *s__ign, size_t count, char c)
{
char sc;
int i;
if (!copy_from_kernel_nofault_allowed(s__ign, 1))
return -ERANGE;
guard(pagefault)();
for (i = 0; i < count && i < XATTR_SIZE_MAX; i++) {
__get_kernel_nofault(&sc, s__ign, char, err_out);
if (sc == c)
return i;
if (sc == '\0')
return -ENOENT;
s__ign++;
}
return i == XATTR_SIZE_MAX ? -E2BIG : -ENOENT;
err_out:
return -EFAULT;
}
/**
* bpf_strchr - Find the first occurrence of a character in a string
* @s__ign: The string to be searched
* @c: The character to search for
*
* Note that the %NUL-terminator is considered part of the string, and can
* be searched for.
*
* Return:
* * >=0 - The index of the first occurrence of @c within @s__ign
* * %-ENOENT - @c not found in @s__ign
* * %-EFAULT - Cannot read @s__ign
* * %-E2BIG - @s__ign is too large
* * %-ERANGE - @s__ign is outside of kernel address space
*/
__bpf_kfunc int bpf_strchr(const char *s__ign, char c)
{
return bpf_strnchr(s__ign, XATTR_SIZE_MAX, c);
}
/**
* bpf_strchrnul - Find and return a character in a string, or end of string
* @s__ign: The string to be searched
* @c: The character to search for
*
* Return:
* * >=0 - Index of the first occurrence of @c within @s__ign or index of
* the null byte at the end of @s__ign when @c is not found
* * %-EFAULT - Cannot read @s__ign
* * %-E2BIG - @s__ign is too large
* * %-ERANGE - @s__ign is outside of kernel address space
*/
__bpf_kfunc int bpf_strchrnul(const char *s__ign, char c)
{
char sc;
int i;
if (!copy_from_kernel_nofault_allowed(s__ign, 1))
return -ERANGE;
guard(pagefault)();
for (i = 0; i < XATTR_SIZE_MAX; i++) {
__get_kernel_nofault(&sc, s__ign, char, err_out);
if (sc == '\0' || sc == c)
return i;
s__ign++;
}
return -E2BIG;
err_out:
return -EFAULT;
}
/**
* bpf_strrchr - Find the last occurrence of a character in a string
* @s__ign: The string to be searched
* @c: The character to search for
*
* Return:
* * >=0 - Index of the last occurrence of @c within @s__ign
* * %-ENOENT - @c not found in @s__ign
* * %-EFAULT - Cannot read @s__ign
* * %-E2BIG - @s__ign is too large
* * %-ERANGE - @s__ign is outside of kernel address space
*/
__bpf_kfunc int bpf_strrchr(const char *s__ign, int c)
{
char sc;
int i, last = -ENOENT;
if (!copy_from_kernel_nofault_allowed(s__ign, 1))
return -ERANGE;
guard(pagefault)();
for (i = 0; i < XATTR_SIZE_MAX; i++) {
__get_kernel_nofault(&sc, s__ign, char, err_out);
if (sc == c)
last = i;
if (sc == '\0')
return last;
s__ign++;
}
return -E2BIG;
err_out:
return -EFAULT;
}
/**
* bpf_strnlen - Calculate the length of a length-limited string
* @s__ign: The string
* @count: The maximum number of characters to count
*
* Return:
* * >=0 - The length of @s__ign
* * %-EFAULT - Cannot read @s__ign
* * %-E2BIG - @s__ign is too large
* * %-ERANGE - @s__ign is outside of kernel address space
*/
__bpf_kfunc int bpf_strnlen(const char *s__ign, size_t count)
{
char c;
int i;
if (!copy_from_kernel_nofault_allowed(s__ign, 1))
return -ERANGE;
guard(pagefault)();
for (i = 0; i < count && i < XATTR_SIZE_MAX; i++) {
__get_kernel_nofault(&c, s__ign, char, err_out);
if (c == '\0')
return i;
s__ign++;
}
return i == XATTR_SIZE_MAX ? -E2BIG : i;
err_out:
return -EFAULT;
}
/**
* bpf_strlen - Calculate the length of a string
* @s__ign: The string
*
* Return:
* * >=0 - The length of @s__ign
* * %-EFAULT - Cannot read @s__ign
* * %-E2BIG - @s__ign is too large
* * %-ERANGE - @s__ign is outside of kernel address space
*/
__bpf_kfunc int bpf_strlen(const char *s__ign)
{
return bpf_strnlen(s__ign, XATTR_SIZE_MAX);
}
/**
* bpf_strspn - Calculate the length of the initial substring of @s__ign which
* only contains letters in @accept__ign
* @s__ign: The string to be searched
* @accept__ign: The string to search for
*
* Return:
* * >=0 - The length of the initial substring of @s__ign which only
* contains letters from @accept__ign
* * %-EFAULT - Cannot read one of the strings
* * %-E2BIG - One of the strings is too large
* * %-ERANGE - One of the strings is outside of kernel address space
*/
__bpf_kfunc int bpf_strspn(const char *s__ign, const char *accept__ign)
{
char cs, ca;
int i, j;
if (!copy_from_kernel_nofault_allowed(s__ign, 1) ||
!copy_from_kernel_nofault_allowed(accept__ign, 1)) {
return -ERANGE;
}
guard(pagefault)();
for (i = 0; i < XATTR_SIZE_MAX; i++) {
__get_kernel_nofault(&cs, s__ign, char, err_out);
if (cs == '\0')
return i;
for (j = 0; j < XATTR_SIZE_MAX; j++) {
__get_kernel_nofault(&ca, accept__ign + j, char, err_out);
if (cs == ca || ca == '\0')
break;
}
if (j == XATTR_SIZE_MAX)
return -E2BIG;
if (ca == '\0')
return i;
s__ign++;
}
return -E2BIG;
err_out:
return -EFAULT;
}
/**
* bpf_strcspn - Calculate the length of the initial substring of @s__ign which
* does not contain letters in @reject__ign
* @s__ign: The string to be searched
* @reject__ign: The string to search for
*
* Return:
* * >=0 - The length of the initial substring of @s__ign which does not
* contain letters from @reject__ign
* * %-EFAULT - Cannot read one of the strings
* * %-E2BIG - One of the strings is too large
* * %-ERANGE - One of the strings is outside of kernel address space
*/
__bpf_kfunc int bpf_strcspn(const char *s__ign, const char *reject__ign)
{
char cs, cr;
int i, j;
if (!copy_from_kernel_nofault_allowed(s__ign, 1) ||
!copy_from_kernel_nofault_allowed(reject__ign, 1)) {
return -ERANGE;
}
guard(pagefault)();
for (i = 0; i < XATTR_SIZE_MAX; i++) {
__get_kernel_nofault(&cs, s__ign, char, err_out);
if (cs == '\0')
return i;
for (j = 0; j < XATTR_SIZE_MAX; j++) {
__get_kernel_nofault(&cr, reject__ign + j, char, err_out);
if (cs == cr || cr == '\0')
break;
}
if (j == XATTR_SIZE_MAX)
return -E2BIG;
if (cr != '\0')
return i;
s__ign++;
}
return -E2BIG;
err_out:
return -EFAULT;
}
/**
* bpf_strnstr - Find the first substring in a length-limited string
* @s1__ign: The string to be searched
* @s2__ign: The string to search for
* @len: the maximum number of characters to search
*
* Return:
* * >=0 - Index of the first character of the first occurrence of @s2__ign
* within the first @len characters of @s1__ign
* * %-ENOENT - @s2__ign not found in the first @len characters of @s1__ign
* * %-EFAULT - Cannot read one of the strings
* * %-E2BIG - One of the strings is too large
* * %-ERANGE - One of the strings is outside of kernel address space
*/
__bpf_kfunc int bpf_strnstr(const char *s1__ign, const char *s2__ign, size_t len)
{
char c1, c2;
int i, j;
if (!copy_from_kernel_nofault_allowed(s1__ign, 1) ||
!copy_from_kernel_nofault_allowed(s2__ign, 1)) {
return -ERANGE;
}
guard(pagefault)();
for (i = 0; i < XATTR_SIZE_MAX; i++) {
for (j = 0; i + j < len && j < XATTR_SIZE_MAX; j++) {
__get_kernel_nofault(&c2, s2__ign + j, char, err_out);
if (c2 == '\0')
return i;
__get_kernel_nofault(&c1, s1__ign + j, char, err_out);
if (c1 == '\0')
return -ENOENT;
if (c1 != c2)
break;
}
if (j == XATTR_SIZE_MAX)
return -E2BIG;
if (i + j == len)
return -ENOENT;
s1__ign++;
}
return -E2BIG;
err_out:
return -EFAULT;
}
/**
* bpf_strstr - Find the first substring in a string
* @s1__ign: The string to be searched
* @s2__ign: The string to search for
*
* Return:
* * >=0 - Index of the first character of the first occurrence of @s2__ign
* within @s1__ign
* * %-ENOENT - @s2__ign is not a substring of @s1__ign
* * %-EFAULT - Cannot read one of the strings
* * %-E2BIG - One of the strings is too large
* * %-ERANGE - One of the strings is outside of kernel address space
*/
__bpf_kfunc int bpf_strstr(const char *s1__ign, const char *s2__ign)
{
return bpf_strnstr(s1__ign, s2__ign, XATTR_SIZE_MAX);
}
__bpf_kfunc_end_defs();
BTF_KFUNCS_START(generic_btf_ids)
@ -3369,6 +3789,7 @@ BTF_ID_FLAGS(func, bpf_dynptr_is_rdonly)
BTF_ID_FLAGS(func, bpf_dynptr_size)
BTF_ID_FLAGS(func, bpf_dynptr_clone)
BTF_ID_FLAGS(func, bpf_dynptr_copy)
BTF_ID_FLAGS(func, bpf_dynptr_memset)
#ifdef CONFIG_NET
BTF_ID_FLAGS(func, bpf_modify_return_test_tp)
#endif
@ -3402,9 +3823,21 @@ BTF_ID_FLAGS(func, bpf_iter_dmabuf_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPAB
BTF_ID_FLAGS(func, bpf_iter_dmabuf_destroy, KF_ITER_DESTROY | KF_SLEEPABLE)
#endif
BTF_ID_FLAGS(func, __bpf_trap)
#ifdef CONFIG_CGROUPS
BTF_ID_FLAGS(func, bpf_strcmp);
BTF_ID_FLAGS(func, bpf_strchr);
BTF_ID_FLAGS(func, bpf_strchrnul);
BTF_ID_FLAGS(func, bpf_strnchr);
BTF_ID_FLAGS(func, bpf_strrchr);
BTF_ID_FLAGS(func, bpf_strlen);
BTF_ID_FLAGS(func, bpf_strnlen);
BTF_ID_FLAGS(func, bpf_strspn);
BTF_ID_FLAGS(func, bpf_strcspn);
BTF_ID_FLAGS(func, bpf_strstr);
BTF_ID_FLAGS(func, bpf_strnstr);
#if defined(CONFIG_BPF_LSM) && defined(CONFIG_CGROUPS)
BTF_ID_FLAGS(func, bpf_cgroup_read_xattr, KF_RCU)
#endif
BTF_ID_FLAGS(func, bpf_stream_vprintk, KF_TRUSTED_ARGS)
BTF_KFUNCS_END(common_btf_ids)
static const struct btf_kfunc_id_set common_kfunc_set = {

3
kernel/bpf/link_iter.c
View file

@ -78,8 +78,7 @@ static const struct seq_operations bpf_link_seq_ops = {
.show = bpf_link_seq_show,
};
BTF_ID_LIST(btf_bpf_link_id)
BTF_ID(struct, bpf_link)
BTF_ID_LIST_SINGLE(btf_bpf_link_id, struct, bpf_link)
static const struct bpf_iter_seq_info bpf_link_seq_info = {
.seq_ops = &bpf_link_seq_ops,

9
kernel/bpf/local_storage.c
View file

@ -394,17 +394,10 @@ static int cgroup_storage_check_btf(const struct bpf_map *map,
if (!btf_member_is_reg_int(btf, key_type, m, offset, size))
return -EINVAL;
} else {
u32 int_data;
/*
* Key is expected to be u64, which stores the cgroup_inode_id
*/
if (BTF_INFO_KIND(key_type->info) != BTF_KIND_INT)
return -EINVAL;
int_data = *(u32 *)(key_type + 1);
if (BTF_INT_BITS(int_data) != 64 || BTF_INT_OFFSET(int_data))
if (!btf_type_is_i64(key_type))
return -EINVAL;
}

10
kernel/bpf/net_namespace.c
View file

@ -11,8 +11,6 @@
struct bpf_netns_link {
struct bpf_link link;
enum bpf_attach_type type;
enum netns_bpf_attach_type netns_type;
/* We don't hold a ref to net in order to auto-detach the link
* when netns is going away. Instead we rely on pernet
@ -21,6 +19,7 @@ struct bpf_netns_link {
*/
struct net *net;
struct list_head node; /* node in list of links attached to net */
enum netns_bpf_attach_type netns_type;
};
/* Protects updates to netns_bpf */
@ -216,7 +215,7 @@ static int bpf_netns_link_fill_info(const struct bpf_link *link,
mutex_unlock(&netns_bpf_mutex);
info->netns.netns_ino = inum;
info->netns.attach_type = net_link->type;
info->netns.attach_type = link->attach_type;
return 0;
}
@ -230,7 +229,7 @@ static void bpf_netns_link_show_fdinfo(const struct bpf_link *link,
"netns_ino:\t%u\n"
"attach_type:\t%u\n",
info.netns.netns_ino,
info.netns.attach_type);
link->attach_type);
}
static const struct bpf_link_ops bpf_netns_link_ops = {
@ -501,9 +500,8 @@ int netns_bpf_link_create(const union bpf_attr *attr, struct bpf_prog *prog)
goto out_put_net;
}
bpf_link_init(&net_link->link, BPF_LINK_TYPE_NETNS,
&bpf_netns_link_ops, prog);
&bpf_netns_link_ops, prog, type);
net_link->net = net;
net_link->type = type;
net_link->netns_type = netns_type;
err = bpf_link_prime(&net_link->link, &link_primer);

5
kernel/bpf/preload/Kconfig
View file

@ -1,8 +1,4 @@
# SPDX-License-Identifier: GPL-2.0-only
config USERMODE_DRIVER
bool
default n
menuconfig BPF_PRELOAD
bool "Preload BPF file system with kernel specific program and map iterators"
depends on BPF
@ -10,7 +6,6 @@ menuconfig BPF_PRELOAD
# The dependency on !COMPILE_TEST prevents it from being enabled
# in allmodconfig or allyesconfig configurations
depends on !COMPILE_TEST
select USERMODE_DRIVER
help
This builds kernel module with several embedded BPF programs that are
pinned into BPF FS mount point as human readable files that are

492
kernel/bpf/preload/iterators/iterators.lskel-big-endian.h
View file

@ -89,10 +89,7 @@ iterators_bpf__load(struct iterators_bpf *skel)
{
struct bpf_load_and_run_opts opts = {};
int err;
opts.ctx = (struct bpf_loader_ctx *)skel;
opts.data_sz = 6008;
opts.data = (void *)"\
static const char opts_data[] __attribute__((__aligned__(8))) = "\
\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
@ -126,190 +123,196 @@ iterators_bpf__load(struct iterators_bpf *skel)
\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xeb\x9f\x01\0\
\0\0\0\x18\0\0\0\0\0\0\x04\x1c\0\0\x04\x1c\0\0\x05\x18\0\0\0\0\x02\0\0\0\0\0\0\
\0\0\0\x18\0\0\0\0\0\0\x04\x80\0\0\x04\x80\0\0\x05\x44\0\0\0\0\x02\0\0\0\0\0\0\
\x02\0\0\0\x01\x04\0\0\x02\0\0\0\x10\0\0\0\x13\0\0\0\x03\0\0\0\0\0\0\0\x18\0\0\
\0\x04\0\0\0\x40\0\0\0\0\x02\0\0\0\0\0\0\x08\0\0\0\0\x02\0\0\0\0\0\0\x0d\0\0\0\
\0\x0d\0\0\x01\0\0\0\x06\0\0\0\x1c\0\0\0\x01\0\0\0\x20\x01\0\0\0\0\0\0\x04\x01\
\0\0\x20\0\0\0\x24\x0c\0\0\x01\0\0\0\x05\0\0\0\xc2\x04\0\0\x03\0\0\0\x18\0\0\0\
\xd0\0\0\0\x09\0\0\0\0\0\0\0\xd4\0\0\0\x0b\0\0\0\x40\0\0\0\xdf\0\0\0\x0b\0\0\0\
\x80\0\0\0\0\x02\0\0\0\0\0\0\x0a\0\0\0\xe7\x07\0\0\0\0\0\0\0\0\0\0\xf0\x08\0\0\
\0\0\0\0\x0c\0\0\0\xf6\x01\0\0\0\0\0\0\x08\0\0\0\x40\0\0\x01\xb3\x04\0\0\x03\0\
\0\0\x18\0\0\x01\xbb\0\0\0\x0e\0\0\0\0\0\0\x01\xbe\0\0\0\x11\0\0\0\x20\0\0\x01\
\xc3\0\0\0\x0e\0\0\0\xa0\0\0\x01\xcf\x08\0\0\0\0\0\0\x0f\0\0\x01\xd5\x01\0\0\0\
\0\0\0\x04\0\0\0\x20\0\0\x01\xe2\x01\0\0\0\0\0\0\x01\x01\0\0\x08\0\0\0\0\x03\0\
\0\0\0\0\0\0\0\0\0\x10\0\0\0\x12\0\0\0\x10\0\0\x01\xe7\x01\0\0\0\0\0\0\x04\0\0\
\0\x20\0\0\0\0\x02\0\0\0\0\0\0\x14\0\0\x02\x4b\x04\0\0\x02\0\0\0\x10\0\0\0\x13\
\0\0\0\x03\0\0\0\0\0\0\x02\x5e\0\0\0\x15\0\0\0\x40\0\0\0\0\x02\0\0\0\0\0\0\x18\
\0\0\0\0\x0d\0\0\x01\0\0\0\x06\0\0\0\x1c\0\0\0\x13\0\0\x02\x63\x0c\0\0\x01\0\0\
\0\x16\0\0\x02\xaf\x04\0\0\x01\0\0\0\x08\0\0\x02\xb8\0\0\0\x19\0\0\0\0\0\0\0\0\
\x02\0\0\0\0\0\0\x1a\0\0\x03\x09\x04\0\0\x06\0\0\0\x38\0\0\x01\xbb\0\0\0\x0e\0\
\0\0\0\0\0\x01\xbe\0\0\0\x11\0\0\0\x20\0\0\x03\x16\0\0\0\x1b\0\0\0\xc0\0\0\x03\
\x27\0\0\0\x15\0\0\x01\0\0\0\x03\x30\0\0\0\x1d\0\0\x01\x40\0\0\x03\x3a\0\0\0\
\x1e\0\0\x01\x80\0\0\0\0\x02\0\0\0\0\0\0\x1c\0\0\0\0\x0a\0\0\0\0\0\0\x10\0\0\0\
\0\x02\0\0\0\0\0\0\x1f\0\0\0\0\x02\0\0\0\0\0\0\x20\0\0\x03\x84\x04\0\0\x02\0\0\
\0\x08\0\0\x03\x92\0\0\0\x0e\0\0\0\0\0\0\x03\x9b\0\0\0\x0e\0\0\0\x20\0\0\x03\
\x3a\x04\0\0\x03\0\0\0\x18\0\0\x03\xa5\0\0\0\x1b\0\0\0\0\0\0\x03\xad\0\0\0\x21\
\0\0\0\x40\0\0\x03\xb3\0\0\0\x23\0\0\0\x80\0\0\0\0\x02\0\0\0\0\0\0\x22\0\0\0\0\
\x02\0\0\0\0\0\0\x24\0\0\x03\xb7\x04\0\0\x01\0\0\0\x04\0\0\x03\xc2\0\0\0\x0e\0\
\0\0\0\0\0\x04\x2b\x04\0\0\x01\0\0\0\x04\0\0\x04\x34\0\0\0\x0e\0\0\0\0\0\0\0\0\
\x03\0\0\0\0\0\0\0\0\0\0\x1c\0\0\0\x12\0\0\0\x23\0\0\x04\xaa\x0e\0\0\0\0\0\0\
\x25\0\0\0\0\0\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\x1c\0\0\0\x12\0\0\0\x0e\0\0\x04\
\xbe\x0e\0\0\0\0\0\0\x27\0\0\0\0\0\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\x1c\0\0\0\x12\
\0\0\0\x20\0\0\x04\xd4\x0e\0\0\0\0\0\0\x29\0\0\0\0\0\0\0\0\x03\0\0\0\0\0\0\0\0\
\0\0\x1c\0\0\0\x12\0\0\0\x11\0\0\x04\xe9\x0e\0\0\0\0\0\0\x2b\0\0\0\0\0\0\0\0\
\x03\0\0\0\0\0\0\0\0\0\0\x10\0\0\0\x12\0\0\0\x04\0\0\x05\0\x0e\0\0\0\0\0\0\x2d\
\0\0\0\x01\0\0\x05\x08\x0f\0\0\x04\0\0\0\x62\0\0\0\x26\0\0\0\0\0\0\0\x23\0\0\0\
\x28\0\0\0\x23\0\0\0\x0e\0\0\0\x2a\0\0\0\x31\0\0\0\x20\0\0\0\x2c\0\0\0\x51\0\0\
\0\x11\0\0\x05\x10\x0f\0\0\x01\0\0\0\x04\0\0\0\x2e\0\0\0\0\0\0\0\x04\0\x62\x70\
\x66\x5f\x69\x74\x65\x72\x5f\x5f\x62\x70\x66\x5f\x6d\x61\x70\0\x6d\x65\x74\x61\
\0\x6d\x61\x70\0\x63\x74\x78\0\x69\x6e\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\
\x5f\x6d\x61\x70\0\x69\x74\x65\x72\x2f\x62\x70\x66\x5f\x6d\x61\x70\0\x30\x3a\
\x30\0\x2f\x68\x6f\x6d\x65\x2f\x69\x69\x69\x2f\x6c\x69\x6e\x75\x78\x2d\x6b\x65\
\x72\x6e\x65\x6c\x2d\x74\x6f\x6f\x6c\x63\x68\x61\x69\x6e\x2f\x73\x72\x63\x2f\
\x6c\x69\x6e\x75\x78\x2f\x6b\x65\x72\x6e\x65\x6c\x2f\x62\x70\x66\x2f\x70\x72\
\x65\x6c\x6f\x61\x64\x2f\x69\x74\x65\x72\x61\x74\x6f\x72\x73\x2f\x69\x74\x65\
\x72\x61\x74\x6f\x72\x73\x2e\x62\x70\x66\x2e\x63\0\x09\x73\x74\x72\x75\x63\x74\
\x20\x73\x65\x71\x5f\x66\x69\x6c\x65\x20\x2a\x73\x65\x71\x20\x3d\x20\x63\x74\
\x78\x2d\x3e\x6d\x65\x74\x61\x2d\x3e\x73\x65\x71\x3b\0\x62\x70\x66\x5f\x69\x74\
\x65\x72\x5f\x6d\x65\x74\x61\0\x73\x65\x71\0\x73\x65\x73\x73\x69\x6f\x6e\x5f\
\x69\x64\0\x73\x65\x71\x5f\x6e\x75\x6d\0\x73\x65\x71\x5f\x66\x69\x6c\x65\0\x5f\
\x5f\x75\x36\x34\0\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x6c\x6f\x6e\x67\x20\x6c\
\x6f\x6e\x67\0\x30\x3a\x31\0\x09\x73\x74\x72\x75\x63\x74\x20\x62\x70\x66\x5f\
\x6d\x61\x70\x20\x2a\x6d\x61\x70\x20\x3d\x20\x63\x74\x78\x2d\x3e\x6d\x61\x70\
\x3b\0\x09\x69\x66\x20\x28\x21\x6d\x61\x70\x29\0\x30\x3a\x32\0\x09\x5f\x5f\x75\
\x36\x34\x20\x73\x65\x71\x5f\x6e\x75\x6d\x20\x3d\x20\x63\x74\x78\x2d\x3e\x6d\
\x65\x74\x61\x2d\x3e\x73\x65\x71\x5f\x6e\x75\x6d\x3b\0\x09\x69\x66\x20\x28\x73\
\x65\x71\x5f\x6e\x75\x6d\x20\x3d\x3d\x20\x30\x29\0\x09\x09\x42\x50\x46\x5f\x53\
\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x20\x20\x69\
\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\x5c\x6e\x22\x29\x3b\0\x62\x70\x66\
\x5f\x6d\x61\x70\0\x69\x64\0\x6e\x61\x6d\x65\0\x6d\x61\x78\x5f\x65\x6e\x74\x72\
\x69\x65\x73\0\x5f\x5f\x75\x33\x32\0\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x69\
\x6e\x74\0\x63\x68\x61\x72\0\x5f\x5f\x41\x52\x52\x41\x59\x5f\x53\x49\x5a\x45\
\x5f\x54\x59\x50\x45\x5f\x5f\0\x09\x42\x50\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\
\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x25\x34\x75\x20\x25\x2d\x31\x36\x73\
\x25\x36\x64\x5c\x6e\x22\x2c\x20\x6d\x61\x70\x2d\x3e\x69\x64\x2c\x20\x6d\x61\
\x70\x2d\x3e\x6e\x61\x6d\x65\x2c\x20\x6d\x61\x70\x2d\x3e\x6d\x61\x78\x5f\x65\
\x6e\x74\x72\x69\x65\x73\x29\x3b\0\x7d\0\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\
\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x70\x72\x6f\x67\0\x64\x75\x6d\x70\x5f\
\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x69\x74\x65\x72\x2f\x62\x70\x66\x5f\x70\x72\
\x6f\x67\0\x09\x73\x74\x72\x75\x63\x74\x20\x62\x70\x66\x5f\x70\x72\x6f\x67\x20\
\x2a\x70\x72\x6f\x67\x20\x3d\x20\x63\x74\x78\x2d\x3e\x70\x72\x6f\x67\x3b\0\x09\
\x69\x66\x20\x28\x21\x70\x72\x6f\x67\x29\0\x62\x70\x66\x5f\x70\x72\x6f\x67\0\
\x61\x75\x78\0\x09\x61\x75\x78\x20\x3d\x20\x70\x72\x6f\x67\x2d\x3e\x61\x75\x78\
\x3b\0\x09\x09\x42\x50\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\
\x65\x71\x2c\x20\x22\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\
\x20\x20\x20\x20\x20\x20\x20\x20\x61\x74\x74\x61\x63\x68\x65\x64\x5c\x6e\x22\
\x29\x3b\0\x62\x70\x66\x5f\x70\x72\x6f\x67\x5f\x61\x75\x78\0\x61\x74\x74\x61\
\x63\x68\x5f\x66\x75\x6e\x63\x5f\x6e\x61\x6d\x65\0\x64\x73\x74\x5f\x70\x72\x6f\
\x67\0\x66\x75\x6e\x63\x5f\x69\x6e\x66\x6f\0\x62\x74\x66\0\x09\x42\x50\x46\x5f\
\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x25\x34\
\x75\x20\x25\x2d\x31\x36\x73\x20\x25\x73\x20\x25\x73\x5c\x6e\x22\x2c\x20\x61\
\x75\x78\x2d\x3e\x69\x64\x2c\0\x30\x3a\x34\0\x30\x3a\x35\0\x09\x69\x66\x20\x28\
\x21\x62\x74\x66\x29\0\x62\x70\x66\x5f\x66\x75\x6e\x63\x5f\x69\x6e\x66\x6f\0\
\x69\x6e\x73\x6e\x5f\x6f\x66\x66\0\x74\x79\x70\x65\x5f\x69\x64\0\x30\0\x73\x74\
\x72\x69\x6e\x67\x73\0\x74\x79\x70\x65\x73\0\x68\x64\x72\0\x62\x74\x66\x5f\x68\
\x65\x61\x64\x65\x72\0\x73\x74\x72\x5f\x6c\x65\x6e\0\x09\x74\x79\x70\x65\x73\
\x20\x3d\x20\x62\x74\x66\x2d\x3e\x74\x79\x70\x65\x73\x3b\0\x09\x62\x70\x66\x5f\
\x70\x72\x6f\x62\x65\x5f\x72\x65\x61\x64\x5f\x6b\x65\x72\x6e\x65\x6c\x28\x26\
\x74\x2c\x20\x73\x69\x7a\x65\x6f\x66\x28\x74\x29\x2c\x20\x74\x79\x70\x65\x73\
\x20\x2b\x20\x62\x74\x66\x5f\x69\x64\x29\x3b\0\x09\x73\x74\x72\x20\x3d\x20\x62\
\x74\x66\x2d\x3e\x73\x74\x72\x69\x6e\x67\x73\x3b\0\x62\x74\x66\x5f\x74\x79\x70\
\x65\0\x6e\x61\x6d\x65\x5f\x6f\x66\x66\0\x09\x6e\x61\x6d\x65\x5f\x6f\x66\x66\
\x20\x3d\x20\x42\x50\x46\x5f\x43\x4f\x52\x45\x5f\x52\x45\x41\x44\x28\x74\x2c\
\x20\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x29\x3b\0\x30\x3a\x32\x3a\x30\0\x09\x69\
\x66\x20\x28\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x20\x3e\x3d\x20\x62\x74\x66\x2d\
\x3e\x68\x64\x72\x2e\x73\x74\x72\x5f\x6c\x65\x6e\x29\0\x09\x72\x65\x74\x75\x72\
\x6e\x20\x73\x74\x72\x20\x2b\x20\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x3b\0\x30\x3a\
\x33\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\
\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\
\x74\x2e\x31\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\
\x5f\x66\x6d\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\x2e\x5f\
\x5f\x5f\x66\x6d\x74\x2e\x32\0\x4c\x49\x43\x45\x4e\x53\x45\0\x2e\x72\x6f\x64\
\x61\x74\x61\0\x6c\x69\x63\x65\x6e\x73\x65\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
\0\0\0\0\0\x09\x4c\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x02\0\0\0\x04\0\0\0\x62\0\0\0\
\x01\0\0\0\x80\0\0\0\0\0\0\0\0\x69\x74\x65\x72\x61\x74\x6f\x72\x2e\x72\x6f\x64\
\x61\x74\x61\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x2f\0\0\0\0\0\0\0\0\0\0\0\0\x20\
\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\
\x20\x20\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\x0a\0\x25\x34\x75\x20\x25\
\x2d\x31\x36\x73\x25\x36\x64\x0a\0\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\
\0\0\x20\0\0\0\x24\x0c\0\0\x01\0\0\0\x05\0\0\0\xc3\x04\0\0\x03\0\0\0\x18\0\0\0\
\xd1\0\0\0\x09\0\0\0\0\0\0\0\xd5\0\0\0\x0b\0\0\0\x40\0\0\0\xe0\0\0\0\x0b\0\0\0\
\x80\0\0\0\0\x02\0\0\0\0\0\0\x0a\0\0\0\xe8\x07\0\0\0\0\0\0\0\0\0\0\xf1\x08\0\0\
\0\0\0\0\x0c\0\0\0\xf7\x01\0\0\0\0\0\0\x08\0\0\0\x40\0\0\x01\xc1\x04\0\0\x03\0\
\0\0\x18\0\0\x01\xc9\0\0\0\x0e\0\0\0\0\0\0\x01\xcc\0\0\0\x11\0\0\0\x20\0\0\x01\
\xd1\0\0\0\x0e\0\0\0\xa0\0\0\x01\xdd\x08\0\0\0\0\0\0\x0f\0\0\x01\xe3\x01\0\0\0\
\0\0\0\x04\0\0\0\x20\0\0\x01\xf0\x01\0\0\0\0\0\0\x01\x01\0\0\x08\0\0\0\0\x03\0\
\0\0\0\0\0\0\0\0\0\x10\0\0\0\x12\0\0\0\x10\0\0\x01\xf5\x01\0\0\0\0\0\0\x04\0\0\
\0\x20\0\0\0\0\x0d\0\0\x01\0\0\0\x14\0\0\x05\x39\0\0\0\x04\0\0\x02\x3e\x08\0\0\
\0\0\0\0\x15\0\0\x02\x44\x01\0\0\0\0\0\0\x08\x01\0\0\x40\0\0\x02\x4e\x0c\0\0\
\x01\0\0\0\x13\0\0\0\0\x02\0\0\0\0\0\0\x18\0\0\x02\x65\x04\0\0\x02\0\0\0\x10\0\
\0\0\x13\0\0\0\x03\0\0\0\0\0\0\x02\x78\0\0\0\x19\0\0\0\x40\0\0\0\0\x02\0\0\0\0\
\0\0\x1c\0\0\0\0\x0d\0\0\x01\0\0\0\x06\0\0\0\x1c\0\0\0\x17\0\0\x02\x7d\x0c\0\0\
\x01\0\0\0\x1a\0\0\x02\xc9\x04\0\0\x01\0\0\0\x08\0\0\x02\xd2\0\0\0\x1d\0\0\0\0\
\0\0\0\0\x02\0\0\0\0\0\0\x1e\0\0\x03\x23\x04\0\0\x06\0\0\0\x38\0\0\x01\xc9\0\0\
\0\x0e\0\0\0\0\0\0\x01\xcc\0\0\0\x11\0\0\0\x20\0\0\x03\x30\0\0\0\x1f\0\0\0\xc0\
\0\0\x03\x41\0\0\0\x19\0\0\x01\0\0\0\x03\x4a\0\0\0\x21\0\0\x01\x40\0\0\x03\x54\
\0\0\0\x22\0\0\x01\x80\0\0\0\0\x02\0\0\0\0\0\0\x20\0\0\0\0\x0a\0\0\0\0\0\0\x10\
\0\0\0\0\x02\0\0\0\0\0\0\x23\0\0\0\0\x02\0\0\0\0\0\0\x24\0\0\x03\x9e\x04\0\0\
\x02\0\0\0\x08\0\0\x03\xac\0\0\0\x0e\0\0\0\0\0\0\x03\xb5\0\0\0\x0e\0\0\0\x20\0\
\0\x03\x54\x04\0\0\x03\0\0\0\x18\0\0\x03\xbf\0\0\0\x1f\0\0\0\0\0\0\x03\xc7\0\0\
\0\x25\0\0\0\x40\0\0\x03\xcd\0\0\0\x27\0\0\0\x80\0\0\0\0\x02\0\0\0\0\0\0\x26\0\
\0\0\0\x02\0\0\0\0\0\0\x28\0\0\x03\xd1\x04\0\0\x01\0\0\0\x04\0\0\x03\xdc\0\0\0\
\x0e\0\0\0\0\0\0\x04\x45\x04\0\0\x01\0\0\0\x04\0\0\x04\x4e\0\0\0\x0e\0\0\0\0\0\
\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\x20\0\0\0\x12\0\0\0\x30\0\0\x04\xc4\x0e\0\0\0\0\
\0\0\x29\0\0\0\0\0\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\x20\0\0\0\x12\0\0\0\x1a\0\0\
\x04\xd8\x0e\0\0\0\0\0\0\x2b\0\0\0\0\0\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\x20\0\0\0\
\x12\0\0\0\x20\0\0\x04\xee\x0e\0\0\0\0\0\0\x2d\0\0\0\0\0\0\0\0\x03\0\0\0\0\0\0\
\0\0\0\0\x20\0\0\0\x12\0\0\0\x11\0\0\x05\x03\x0e\0\0\0\0\0\0\x2f\0\0\0\0\0\0\0\
\0\x03\0\0\0\0\0\0\0\0\0\0\x10\0\0\0\x12\0\0\0\x04\0\0\x05\x1a\x0e\0\0\0\0\0\0\
\x31\0\0\0\x01\0\0\x05\x22\x0f\0\0\x01\0\0\0\x04\0\0\0\x36\0\0\0\0\0\0\0\x04\0\
\0\x05\x29\x0f\0\0\x04\0\0\0\x7b\0\0\0\x2a\0\0\0\0\0\0\0\x30\0\0\0\x2c\0\0\0\
\x30\0\0\0\x1a\0\0\0\x2e\0\0\0\x4a\0\0\0\x20\0\0\0\x30\0\0\0\x6a\0\0\0\x11\0\0\
\x05\x31\x0f\0\0\x01\0\0\0\x04\0\0\0\x32\0\0\0\0\0\0\0\x04\0\0\x05\x39\x0e\0\0\
\0\0\0\0\x06\0\0\0\x01\0\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x5f\x62\x70\x66\
\x5f\x6d\x61\x70\0\x6d\x65\x74\x61\0\x6d\x61\x70\0\x63\x74\x78\0\x69\x6e\x74\0\
\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\0\x69\x74\x65\x72\x2f\x62\x70\
\x66\x5f\x6d\x61\x70\0\x30\x3a\x30\0\x2f\x68\x6f\x6d\x65\x32\x2f\x69\x69\x69\
\x2f\x6c\x69\x6e\x75\x78\x2d\x6b\x65\x72\x6e\x65\x6c\x2d\x74\x6f\x6f\x6c\x63\
\x68\x61\x69\x6e\x2f\x73\x72\x63\x2f\x6c\x69\x6e\x75\x78\x2f\x6b\x65\x72\x6e\
\x65\x6c\x2f\x62\x70\x66\x2f\x70\x72\x65\x6c\x6f\x61\x64\x2f\x69\x74\x65\x72\
\x61\x74\x6f\x72\x73\x2f\x69\x74\x65\x72\x61\x74\x6f\x72\x73\x2e\x62\x70\x66\
\x2e\x63\0\x09\x73\x74\x72\x75\x63\x74\x20\x73\x65\x71\x5f\x66\x69\x6c\x65\x20\
\x2a\x73\x65\x71\x20\x3d\x20\x63\x74\x78\x2d\x3e\x6d\x65\x74\x61\x2d\x3e\x73\
\x65\x71\x3b\0\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x6d\x65\x74\x61\0\x73\x65\
\x71\0\x73\x65\x73\x73\x69\x6f\x6e\x5f\x69\x64\0\x73\x65\x71\x5f\x6e\x75\x6d\0\
\x73\x65\x71\x5f\x66\x69\x6c\x65\0\x5f\x5f\x75\x36\x34\0\x75\x6e\x73\x69\x67\
\x6e\x65\x64\x20\x6c\x6f\x6e\x67\x20\x6c\x6f\x6e\x67\0\x30\x3a\x31\0\x09\x73\
\x74\x72\x75\x63\x74\x20\x62\x70\x66\x5f\x6d\x61\x70\x20\x2a\x6d\x61\x70\x20\
\x3d\x20\x63\x74\x78\x2d\x3e\x6d\x61\x70\x3b\0\x09\x69\x66\x20\x28\x21\x6d\x61\
\x70\x29\0\x30\x3a\x32\0\x09\x5f\x5f\x75\x36\x34\x20\x73\x65\x71\x5f\x6e\x75\
\x6d\x20\x3d\x20\x63\x74\x78\x2d\x3e\x6d\x65\x74\x61\x2d\x3e\x73\x65\x71\x5f\
\x6e\x75\x6d\x3b\0\x09\x69\x66\x20\x28\x73\x65\x71\x5f\x6e\x75\x6d\x20\x3d\x3d\
\x20\x30\x29\0\x09\x09\x42\x50\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\
\x28\x73\x65\x71\x2c\x20\x22\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\
\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\
\x65\x73\x20\x20\x63\x75\x72\x5f\x65\x6e\x74\x72\x69\x65\x73\x5c\x6e\x22\x29\
\x3b\0\x62\x70\x66\x5f\x6d\x61\x70\0\x69\x64\0\x6e\x61\x6d\x65\0\x6d\x61\x78\
\x5f\x65\x6e\x74\x72\x69\x65\x73\0\x5f\x5f\x75\x33\x32\0\x75\x6e\x73\x69\x67\
\x6e\x65\x64\x20\x69\x6e\x74\0\x63\x68\x61\x72\0\x5f\x5f\x41\x52\x52\x41\x59\
\x5f\x53\x49\x5a\x45\x5f\x54\x59\x50\x45\x5f\x5f\0\x09\x42\x50\x46\x5f\x53\x45\
\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x25\x34\x75\x20\
\x25\x2d\x31\x36\x73\x20\x20\x25\x31\x30\x64\x20\x20\x20\x25\x31\x30\x6c\x6c\
\x64\x5c\x6e\x22\x2c\0\x7d\0\x5f\x5f\x73\x36\x34\0\x6c\x6f\x6e\x67\x20\x6c\x6f\
\x6e\x67\0\x62\x70\x66\x5f\x6d\x61\x70\x5f\x73\x75\x6d\x5f\x65\x6c\x65\x6d\x5f\
\x63\x6f\x75\x6e\x74\0\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x5f\x62\x70\x66\x5f\
\x70\x72\x6f\x67\0\x70\x72\x6f\x67\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\
\x72\x6f\x67\0\x69\x74\x65\x72\x2f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x09\x73\
\x74\x72\x75\x63\x74\x20\x62\x70\x66\x5f\x70\x72\x6f\x67\x20\x2a\x70\x72\x6f\
\x67\x20\x3d\x20\x63\x74\x78\x2d\x3e\x70\x72\x6f\x67\x3b\0\x09\x69\x66\x20\x28\
\x21\x70\x72\x6f\x67\x29\0\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x61\x75\x78\0\x09\
\x61\x75\x78\x20\x3d\x20\x70\x72\x6f\x67\x2d\x3e\x61\x75\x78\x3b\0\x09\x09\x42\
\x50\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\
\x22\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\x20\x20\x20\x20\
\x20\x20\x20\x20\x61\x74\x74\x61\x63\x68\x65\x64\x5c\x6e\x22\x29\x3b\0\x62\x70\
\x66\x5f\x70\x72\x6f\x67\x5f\x61\x75\x78\0\x61\x74\x74\x61\x63\x68\x5f\x66\x75\
\x6e\x63\x5f\x6e\x61\x6d\x65\0\x64\x73\x74\x5f\x70\x72\x6f\x67\0\x66\x75\x6e\
\x63\x5f\x69\x6e\x66\x6f\0\x62\x74\x66\0\x09\x42\x50\x46\x5f\x53\x45\x51\x5f\
\x50\x52\x49\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x25\x34\x75\x20\x25\x2d\
\x31\x36\x73\x20\x25\x73\x20\x25\x73\x5c\x6e\x22\x2c\x20\x61\x75\x78\x2d\x3e\
\x69\x64\x2c\0\x30\x3a\x34\0\x30\x3a\x35\0\x09\x69\x66\x20\x28\x21\x62\x74\x66\
\x29\0\x62\x70\x66\x5f\x66\x75\x6e\x63\x5f\x69\x6e\x66\x6f\0\x69\x6e\x73\x6e\
\x5f\x6f\x66\x66\0\x74\x79\x70\x65\x5f\x69\x64\0\x30\0\x73\x74\x72\x69\x6e\x67\
\x73\0\x74\x79\x70\x65\x73\0\x68\x64\x72\0\x62\x74\x66\x5f\x68\x65\x61\x64\x65\
\x72\0\x73\x74\x72\x5f\x6c\x65\x6e\0\x09\x74\x79\x70\x65\x73\x20\x3d\x20\x62\
\x74\x66\x2d\x3e\x74\x79\x70\x65\x73\x3b\0\x09\x62\x70\x66\x5f\x70\x72\x6f\x62\
\x65\x5f\x72\x65\x61\x64\x5f\x6b\x65\x72\x6e\x65\x6c\x28\x26\x74\x2c\x20\x73\
\x69\x7a\x65\x6f\x66\x28\x74\x29\x2c\x20\x74\x79\x70\x65\x73\x20\x2b\x20\x62\
\x74\x66\x5f\x69\x64\x29\x3b\0\x09\x73\x74\x72\x20\x3d\x20\x62\x74\x66\x2d\x3e\
\x73\x74\x72\x69\x6e\x67\x73\x3b\0\x62\x74\x66\x5f\x74\x79\x70\x65\0\x6e\x61\
\x6d\x65\x5f\x6f\x66\x66\0\x09\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x20\x3d\x20\x42\
\x50\x46\x5f\x43\x4f\x52\x45\x5f\x52\x45\x41\x44\x28\x74\x2c\x20\x6e\x61\x6d\
\x65\x5f\x6f\x66\x66\x29\x3b\0\x30\x3a\x32\x3a\x30\0\x09\x69\x66\x20\x28\x6e\
\x61\x6d\x65\x5f\x6f\x66\x66\x20\x3e\x3d\x20\x62\x74\x66\x2d\x3e\x68\x64\x72\
\x2e\x73\x74\x72\x5f\x6c\x65\x6e\x29\0\x09\x72\x65\x74\x75\x72\x6e\x20\x73\x74\
\x72\x20\x2b\x20\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x3b\0\x30\x3a\x33\0\x64\x75\
\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\0\x64\x75\
\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\x2e\x31\0\
\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\x5f\x66\x6d\
\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\x5f\x66\
\x6d\x74\x2e\x32\0\x4c\x49\x43\x45\x4e\x53\x45\0\x2e\x6b\x73\x79\x6d\x73\0\x2e\
\x72\x6f\x64\x61\x74\x61\0\x6c\x69\x63\x65\x6e\x73\x65\0\x64\x75\x6d\x6d\x79\
\x5f\x6b\x73\x79\x6d\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x09\xdc\0\0\
\0\0\0\0\0\0\0\0\0\0\0\0\0\x02\0\0\0\x04\0\0\0\x7b\0\0\0\x01\0\0\0\x80\0\0\0\0\
\0\0\0\0\x69\x74\x65\x72\x61\x74\x6f\x72\x2e\x72\x6f\x64\x61\x74\x61\0\0\0\0\0\
\0\0\0\0\0\0\0\0\0\0\0\x34\0\0\0\0\0\0\0\0\0\0\0\0\x20\x20\x69\x64\x20\x6e\x61\
\x6d\x65\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x6d\x61\x78\x5f\
\x65\x6e\x74\x72\x69\x65\x73\x20\x20\x63\x75\x72\x5f\x65\x6e\x74\x72\x69\x65\
\x73\x0a\0\x25\x34\x75\x20\x25\x2d\x31\x36\x73\x20\x20\x25\x31\x30\x64\x20\x20\
\x20\x25\x31\x30\x6c\x6c\x64\x0a\0\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\
\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x61\x74\x74\x61\x63\x68\x65\x64\
\x0a\0\x25\x34\x75\x20\x25\x2d\x31\x36\x73\x20\x25\x73\x20\x25\x73\x0a\0\0\0\0\
\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
\0\0\0\0\0\0\0\0\0\0\0\0\x47\x50\x4c\0\0\0\0\0\x79\x21\0\0\0\0\0\0\x79\x62\0\0\
\0\0\0\0\x79\x71\0\x08\0\0\0\0\x15\x70\0\x1a\0\0\0\0\x79\x12\0\x10\0\0\0\0\x55\
\x10\0\x08\0\0\0\0\xbf\x4a\0\0\0\0\0\0\x07\x40\0\0\xff\xff\xff\xe8\xbf\x16\0\0\
\0\0\0\0\x18\x26\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb7\x30\0\0\0\0\0\x23\xb7\x50\0\0\
\0\0\0\0\x85\0\0\0\0\0\0\x7e\x61\x17\0\0\0\0\0\0\x7b\xa1\xff\xe8\0\0\0\0\xb7\
\x10\0\0\0\0\0\x04\xbf\x27\0\0\0\0\0\0\x0f\x21\0\0\0\0\0\0\x7b\xa2\xff\xf0\0\0\
\0\0\x61\x17\0\x14\0\0\0\0\x7b\xa1\xff\xf8\0\0\0\0\xbf\x4a\0\0\0\0\0\0\x07\x40\
\0\0\xff\xff\xff\xe8\xbf\x16\0\0\0\0\0\0\x18\x26\0\0\0\0\0\0\0\0\0\0\0\0\0\x23\
\xb7\x30\0\0\0\0\0\x0e\xb7\x50\0\0\0\0\0\x18\x85\0\0\0\0\0\0\x7e\xb7\0\0\0\0\0\
\0\0\x95\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x07\0\0\0\0\0\0\0\x42\0\0\0\x9a\0\x01\x3c\
\x1e\0\0\0\x01\0\0\0\x42\0\0\0\x9a\0\x01\x3c\x24\0\0\0\x02\0\0\0\x42\0\0\x01\
\x0d\0\x01\x44\x1d\0\0\0\x03\0\0\0\x42\0\0\x01\x2e\0\x01\x4c\x06\0\0\0\x04\0\0\
\0\x42\0\0\x01\x3d\0\x01\x40\x1d\0\0\0\x05\0\0\0\x42\0\0\x01\x62\0\x01\x58\x06\
\0\0\0\x07\0\0\0\x42\0\0\x01\x75\0\x01\x5c\x03\0\0\0\x0e\0\0\0\x42\0\0\x01\xfb\
\0\x01\x64\x02\0\0\0\x1e\0\0\0\x42\0\0\x02\x49\0\x01\x6c\x01\0\0\0\0\0\0\0\x02\
\0\0\0\x3e\0\0\0\0\0\0\0\x08\0\0\0\x08\0\0\0\x3e\0\0\0\0\0\0\0\x10\0\0\0\x02\0\
\0\x01\x09\0\0\0\0\0\0\0\x20\0\0\0\x08\0\0\x01\x39\0\0\0\0\0\0\0\x70\0\0\0\x0d\
\0\0\0\x3e\0\0\0\0\0\0\0\x80\0\0\0\x0d\0\0\x01\x09\0\0\0\0\0\0\0\xa0\0\0\0\x0d\
\0\0\x01\x39\0\0\0\0\0\0\0\x1a\0\0\0\x20\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\
\x6d\x61\x70\0\0\0\0\0\0\0\0\0\0\0\x1c\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\
\x01\0\0\0\x10\0\0\0\0\0\0\0\0\0\0\0\x09\0\0\0\x01\0\0\0\0\0\0\0\x07\0\0\0\0\0\
\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x10\0\0\0\0\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\
\x62\x70\x66\x5f\x6d\x61\x70\0\0\0\0\0\0\0\0\x47\x50\x4c\0\0\0\0\0\x79\x21\0\0\
\0\0\0\0\x79\x62\0\0\0\0\0\0\x79\x11\0\x08\0\0\0\0\x15\x10\0\x3b\0\0\0\0\x79\
\x71\0\0\0\0\0\0\x79\x12\0\x10\0\0\0\0\x55\x10\0\x08\0\0\0\0\xbf\x4a\0\0\0\0\0\
\0\x07\x40\0\0\xff\xff\xff\xd0\xbf\x16\0\0\0\0\0\0\x18\x26\0\0\0\0\0\0\0\0\0\0\
\0\0\0\x31\xb7\x30\0\0\0\0\0\x20\xb7\x50\0\0\0\0\0\0\x85\0\0\0\0\0\0\x7e\x7b\
\xa6\xff\xc8\0\0\0\0\x61\x17\0\0\0\0\0\0\x7b\xa1\xff\xd0\0\0\0\0\xb7\x30\0\0\0\
\0\0\x04\xbf\x97\0\0\0\0\0\0\x0f\x93\0\0\0\0\0\0\x79\x17\0\x28\0\0\0\0\x79\x87\
\0\x30\0\0\0\0\x15\x80\0\x18\0\0\0\0\xb7\x20\0\0\0\0\0\0\x0f\x12\0\0\0\0\0\0\
\x61\x11\0\x04\0\0\0\0\x79\x38\0\x08\0\0\0\0\x67\x10\0\0\0\0\0\x03\x0f\x31\0\0\
\0\0\0\0\x79\x68\0\0\0\0\0\0\xbf\x1a\0\0\0\0\0\0\x07\x10\0\0\xff\xff\xff\xf8\
\xb7\x20\0\0\0\0\0\x08\x85\0\0\0\0\0\0\x71\xb7\x10\0\0\0\0\0\0\x79\x3a\xff\xf8\
\0\0\0\0\0\0\0\0\0\0\0\x47\x50\x4c\0\0\0\0\0\x79\x21\0\0\0\0\0\0\x79\x62\0\0\0\
\0\0\0\x79\x71\0\x08\0\0\0\0\x15\x70\0\x1d\0\0\0\0\x79\x12\0\x10\0\0\0\0\x55\
\x10\0\x08\0\0\0\0\xbf\x4a\0\0\0\0\0\0\x07\x40\0\0\xff\xff\xff\xe0\xbf\x16\0\0\
\0\0\0\0\x18\x26\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb4\x30\0\0\0\0\0\x30\xb4\x50\0\0\
\0\0\0\0\x85\0\0\0\0\0\0\x7e\x61\x17\0\0\0\0\0\0\x7b\xa1\xff\xe0\0\0\0\0\xb7\
\x10\0\0\0\0\0\x04\xbf\x27\0\0\0\0\0\0\x0f\x21\0\0\0\0\0\0\x7b\xa2\xff\xe8\0\0\
\0\0\x61\x17\0\x14\0\0\0\0\x7b\xa1\xff\xf0\0\0\0\0\xbf\x17\0\0\0\0\0\0\x85\x02\
\0\0\0\0\0\0\x7b\xa0\xff\xf8\0\0\0\0\xbf\x4a\0\0\0\0\0\0\x07\x40\0\0\xff\xff\
\xff\xe0\xbf\x16\0\0\0\0\0\0\x18\x26\0\0\0\0\0\0\0\0\0\0\0\0\0\x30\xb4\x30\0\0\
\0\0\0\x1a\xb4\x50\0\0\0\0\0\x20\x85\0\0\0\0\0\0\x7e\xb4\0\0\0\0\0\0\0\x95\0\0\
\0\0\0\0\0\0\0\0\0\0\0\0\x07\0\0\0\0\0\0\0\x42\0\0\0\x9b\0\x01\x44\x1e\0\0\0\
\x01\0\0\0\x42\0\0\0\x9b\0\x01\x44\x24\0\0\0\x02\0\0\0\x42\0\0\x01\x0e\0\x01\
\x4c\x1d\0\0\0\x03\0\0\0\x42\0\0\x01\x2f\0\x01\x54\x06\0\0\0\x04\0\0\0\x42\0\0\
\x01\x3e\0\x01\x48\x1d\0\0\0\x05\0\0\0\x42\0\0\x01\x63\0\x01\x60\x0e\0\0\0\x08\
\0\0\0\x42\0\0\x01\x76\0\x01\x64\x03\0\0\0\x0e\0\0\0\x42\0\0\x02\x09\0\x01\x6c\
\x02\0\0\0\x21\0\0\0\x42\0\0\x02\x3c\0\x01\x80\x01\0\0\0\0\0\0\0\x02\0\0\0\x3e\
\0\0\0\0\0\0\0\x08\0\0\0\x08\0\0\0\x3e\0\0\0\0\0\0\0\x10\0\0\0\x02\0\0\x01\x0a\
\0\0\0\0\0\0\0\x20\0\0\0\x08\0\0\x01\x3a\0\0\0\0\0\0\0\x70\0\0\0\x0d\0\0\0\x3e\
\0\0\0\0\0\0\0\x80\0\0\0\x0d\0\0\x01\x0a\0\0\0\0\0\0\0\xa0\0\0\0\x0d\0\0\x01\
\x3a\0\0\0\0\0\0\0\x1a\0\0\0\x23\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\
\x70\0\0\0\0\0\0\0\0\0\0\0\x1c\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\
\0\x10\0\0\0\0\0\0\0\0\0\0\0\x09\0\0\0\x01\0\0\0\0\0\0\0\x07\0\0\0\0\0\0\0\0\0\
\0\0\0\0\0\0\0\0\0\0\x10\0\0\0\0\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x62\x70\
\x66\x5f\x6d\x61\x70\0\0\0\0\0\0\0\0\x62\x70\x66\x5f\x6d\x61\x70\x5f\x73\x75\
\x6d\x5f\x65\x6c\x65\x6d\x5f\x63\x6f\x75\x6e\x74\0\0\x47\x50\x4c\0\0\0\0\0\x79\
\x21\0\0\0\0\0\0\x79\x62\0\0\0\0\0\0\x79\x11\0\x08\0\0\0\0\x15\x10\0\x3b\0\0\0\
\0\x79\x71\0\0\0\0\0\0\x79\x12\0\x10\0\0\0\0\x55\x10\0\x08\0\0\0\0\xbf\x4a\0\0\
\0\0\0\0\x07\x40\0\0\xff\xff\xff\xd0\xbf\x16\0\0\0\0\0\0\x18\x26\0\0\0\0\0\0\0\
\0\0\0\0\0\0\x4a\xb4\x30\0\0\0\0\0\x20\xb4\x50\0\0\0\0\0\0\x85\0\0\0\0\0\0\x7e\
\x7b\xa6\xff\xc8\0\0\0\0\x61\x17\0\0\0\0\0\0\x7b\xa1\xff\xd0\0\0\0\0\xb7\x30\0\
\0\0\0\0\x04\xbf\x97\0\0\0\0\0\0\x0f\x93\0\0\0\0\0\0\x79\x17\0\x28\0\0\0\0\x79\
\x87\0\x30\0\0\0\0\x15\x80\0\x18\0\0\0\0\xb7\x20\0\0\0\0\0\0\x0f\x12\0\0\0\0\0\
\0\x61\x11\0\x04\0\0\0\0\x79\x38\0\x08\0\0\0\0\x67\x10\0\0\0\0\0\x03\x0f\x31\0\
\0\0\0\0\0\x79\x68\0\0\0\0\0\0\xbf\x1a\0\0\0\0\0\0\x07\x10\0\0\xff\xff\xff\xf8\
\xb4\x20\0\0\0\0\0\x08\x85\0\0\0\0\0\0\x71\xb7\x10\0\0\0\0\0\0\x79\x3a\xff\xf8\
\0\0\0\0\x0f\x31\0\0\0\0\0\0\xbf\x1a\0\0\0\0\0\0\x07\x10\0\0\xff\xff\xff\xf4\
\xb7\x20\0\0\0\0\0\x04\x85\0\0\0\0\0\0\x71\xb7\x30\0\0\0\0\0\x04\x61\x1a\xff\
\xf4\0\0\0\0\x61\x28\0\x10\0\0\0\0\x3d\x12\0\x02\0\0\0\0\x0f\x61\0\0\0\0\0\0\
\xb4\x20\0\0\0\0\0\x04\x85\0\0\0\0\0\0\x71\xb7\x30\0\0\0\0\0\x04\x61\x1a\xff\
\xf4\0\0\0\0\x61\x28\0\x10\0\0\0\0\x3e\x12\0\x02\0\0\0\0\x0f\x61\0\0\0\0\0\0\
\xbf\x96\0\0\0\0\0\0\x7b\xa9\xff\xd8\0\0\0\0\x79\x17\0\x18\0\0\0\0\x7b\xa1\xff\
\xe0\0\0\0\0\x79\x17\0\x20\0\0\0\0\x79\x11\0\0\0\0\0\0\x0f\x13\0\0\0\0\0\0\x7b\
\xa1\xff\xe8\0\0\0\0\xbf\x4a\0\0\0\0\0\0\x07\x40\0\0\xff\xff\xff\xd0\x79\x1a\
\xff\xc8\0\0\0\0\x18\x26\0\0\0\0\0\0\0\0\0\0\0\0\0\x51\xb7\x30\0\0\0\0\0\x11\
\xb7\x50\0\0\0\0\0\x20\x85\0\0\0\0\0\0\x7e\xb7\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\
\0\0\0\0\0\0\0\x17\0\0\0\0\0\0\0\x42\0\0\0\x9a\0\x01\x80\x1e\0\0\0\x01\0\0\0\
\x42\0\0\0\x9a\0\x01\x80\x24\0\0\0\x02\0\0\0\x42\0\0\x02\x7f\0\x01\x88\x1f\0\0\
\0\x03\0\0\0\x42\0\0\x02\xa3\0\x01\x94\x06\0\0\0\x04\0\0\0\x42\0\0\x02\xbc\0\
\x01\xa0\x0e\0\0\0\x05\0\0\0\x42\0\0\x01\x3d\0\x01\x84\x1d\0\0\0\x06\0\0\0\x42\
\0\0\x01\x62\0\x01\xa4\x06\0\0\0\x08\0\0\0\x42\0\0\x02\xce\0\x01\xa8\x03\0\0\0\
\x10\0\0\0\x42\0\0\x03\x3e\0\x01\xb0\x02\0\0\0\x17\0\0\0\x42\0\0\x03\x79\0\x01\
\x04\x06\0\0\0\x1a\0\0\0\x42\0\0\x03\x3e\0\x01\xb0\x02\0\0\0\x1b\0\0\0\x42\0\0\
\x03\xca\0\x01\x10\x0f\0\0\0\x1c\0\0\0\x42\0\0\x03\xdf\0\x01\x14\x2d\0\0\0\x1e\
\0\0\0\x42\0\0\x04\x16\0\x01\x0c\x0d\0\0\0\x20\0\0\0\x42\0\0\x03\x3e\0\x01\xb0\
\x02\0\0\0\x21\0\0\0\x42\0\0\x03\xdf\0\x01\x14\x02\0\0\0\x24\0\0\0\x42\0\0\x04\
\x3d\0\x01\x18\x0d\0\0\0\x27\0\0\0\x42\0\0\x03\x3e\0\x01\xb0\x02\0\0\0\x28\0\0\
\0\x42\0\0\x04\x3d\0\x01\x18\x0d\0\0\0\x2b\0\0\0\x42\0\0\x04\x3d\0\x01\x18\x0d\
\0\0\0\x2c\0\0\0\x42\0\0\x04\x6b\0\x01\x1c\x1b\0\0\0\x2d\0\0\0\x42\0\0\x04\x6b\
\0\x01\x1c\x06\0\0\0\x2e\0\0\0\x42\0\0\x04\x8e\0\x01\x24\x0d\0\0\0\x30\0\0\0\
\x42\0\0\x03\x3e\0\x01\xb0\x02\0\0\0\x3f\0\0\0\x42\0\0\x02\x49\0\x01\xc0\x01\0\
\0\0\0\0\0\0\x14\0\0\0\x3e\0\0\0\0\0\0\0\x08\0\0\0\x08\0\0\0\x3e\0\0\0\0\0\0\0\
\x10\0\0\0\x14\0\0\x01\x09\0\0\0\0\0\0\0\x20\0\0\0\x18\0\0\0\x3e\0\0\0\0\0\0\0\
\x28\0\0\0\x08\0\0\x01\x39\0\0\0\0\0\0\0\x80\0\0\0\x1a\0\0\0\x3e\0\0\0\0\0\0\0\
\x90\0\0\0\x1a\0\0\x01\x09\0\0\0\0\0\0\0\xa8\0\0\0\x1a\0\0\x03\x71\0\0\0\0\0\0\
\0\xb0\0\0\0\x1a\0\0\x03\x75\0\0\0\0\0\0\0\xc0\0\0\0\x1f\0\0\x03\xa3\0\0\0\0\0\
\0\0\xd8\0\0\0\x20\0\0\x01\x09\0\0\0\0\0\0\0\xf0\0\0\0\x20\0\0\0\x3e\0\0\0\0\0\
\0\x01\x18\0\0\0\x24\0\0\0\x3e\0\0\0\0\0\0\x01\x50\0\0\0\x1a\0\0\x01\x09\0\0\0\
\0\0\0\x01\x60\0\0\0\x20\0\0\x04\x65\0\0\0\0\0\0\x01\x88\0\0\0\x1a\0\0\x01\x39\
\0\0\0\0\0\0\x01\x98\0\0\0\x1a\0\0\x04\xa6\0\0\0\0\0\0\x01\xa0\0\0\0\x18\0\0\0\
\x3e\0\0\0\0\0\0\0\x1a\0\0\0\x41\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\
\x6f\x67\0\0\0\0\0\0\0\0\0\0\x1c\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\x01\0\
\0\0\x10\0\0\0\0\0\0\0\0\0\0\0\x19\0\0\0\x01\0\0\0\0\0\0\0\x12\0\0\0\0\0\0\0\0\
\0\0\0\0\0\0\0\0\0\0\0\x10\0\0\0\0\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x62\x70\
\x66\x5f\x70\x72\x6f\x67\0\0\0\0\0\0\0";
opts.insns_sz = 2216;
opts.insns = (void *)"\
\xff\xc8\0\0\0\0\x18\x26\0\0\0\0\0\0\0\0\0\0\0\0\0\x6a\xb4\x30\0\0\0\0\0\x11\
\xb4\x50\0\0\0\0\0\x20\x85\0\0\0\0\0\0\x7e\xb4\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0\
\0\0\0\0\0\0\0\x1b\0\0\0\0\0\0\0\x42\0\0\0\x9b\0\x01\x94\x1e\0\0\0\x01\0\0\0\
\x42\0\0\0\x9b\0\x01\x94\x24\0\0\0\x02\0\0\0\x42\0\0\x02\x99\0\x01\x9c\x1f\0\0\
\0\x03\0\0\0\x42\0\0\x02\xbd\0\x01\xa8\x06\0\0\0\x04\0\0\0\x42\0\0\x02\xd6\0\
\x01\xb4\x0e\0\0\0\x05\0\0\0\x42\0\0\x01\x3e\0\x01\x98\x1d\0\0\0\x06\0\0\0\x42\
\0\0\x01\x63\0\x01\xb8\x0e\0\0\0\x09\0\0\0\x42\0\0\x02\xe8\0\x01\xbc\x03\0\0\0\
\x10\0\0\0\x42\0\0\x03\x58\0\x01\xc4\x02\0\0\0\x17\0\0\0\x42\0\0\x03\x93\0\x01\
\x04\x06\0\0\0\x1a\0\0\0\x42\0\0\x03\x58\0\x01\xc4\x02\0\0\0\x1b\0\0\0\x42\0\0\
\x03\xe4\0\x01\x10\x0f\0\0\0\x1c\0\0\0\x42\0\0\x03\xf9\0\x01\x14\x2d\0\0\0\x1e\
\0\0\0\x42\0\0\x04\x30\0\x01\x0c\x0d\0\0\0\x21\0\0\0\x42\0\0\x03\xf9\0\x01\x14\
\x02\0\0\0\x24\0\0\0\x42\0\0\x04\x57\0\x01\x18\x0d\0\0\0\x2b\0\0\0\x42\0\0\x04\
\x57\0\x01\x18\x0d\0\0\0\x2c\0\0\0\x42\0\0\x04\x85\0\x01\x1c\x1b\0\0\0\x2d\0\0\
\0\x42\0\0\x04\x85\0\x01\x1c\x0f\0\0\0\x2e\0\0\0\x42\0\0\x04\xa8\0\x01\x24\x0d\
\0\0\0\x30\0\0\0\x42\0\0\x03\x58\0\x01\xc4\x02\0\0\0\x3f\0\0\0\x42\0\0\x02\x3c\
\0\x01\xd4\x01\0\0\0\0\0\0\0\x18\0\0\0\x3e\0\0\0\0\0\0\0\x08\0\0\0\x08\0\0\0\
\x3e\0\0\0\0\0\0\0\x10\0\0\0\x18\0\0\x01\x0a\0\0\0\0\0\0\0\x20\0\0\0\x1c\0\0\0\
\x3e\0\0\0\0\0\0\0\x28\0\0\0\x08\0\0\x01\x3a\0\0\0\0\0\0\0\x80\0\0\0\x1e\0\0\0\
\x3e\0\0\0\0\0\0\0\x90\0\0\0\x1e\0\0\x01\x0a\0\0\0\0\0\0\0\xa8\0\0\0\x1e\0\0\
\x03\x8b\0\0\0\0\0\0\0\xb0\0\0\0\x1e\0\0\x03\x8f\0\0\0\0\0\0\0\xc0\0\0\0\x23\0\
\0\x03\xbd\0\0\0\0\0\0\0\xd8\0\0\0\x24\0\0\x01\x0a\0\0\0\0\0\0\0\xf0\0\0\0\x24\
\0\0\0\x3e\0\0\0\0\0\0\x01\x18\0\0\0\x28\0\0\0\x3e\0\0\0\0\0\0\x01\x50\0\0\0\
\x1e\0\0\x01\x0a\0\0\0\0\0\0\x01\x60\0\0\0\x24\0\0\x04\x7f\0\0\0\0\0\0\x01\x88\
\0\0\0\x1e\0\0\x01\x3a\0\0\0\0\0\0\x01\x98\0\0\0\x1e\0\0\x04\xc0\0\0\0\0\0\0\
\x01\xa0\0\0\0\x1c\0\0\0\x3e\0\0\0\0\0\0\0\x1a\0\0\0\x41\0\0\0\0\0\0\0\0\0\0\0\
\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x64\x75\x6d\x70\x5f\
\x62\x70\x66\x5f\x70\x72\x6f\x67\0\0\0\0\0\0\0\0\0\0\x1c\0\0\0\0\0\0\0\x08\0\0\
\0\0\0\0\0\0\0\0\0\x01\0\0\0\x10\0\0\0\0\0\0\0\0\0\0\0\x16\0\0\0\x01\0\0\0\0\0\
\0\0\x12\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x10\0\0\0\0\x62\x70\x66\x5f\x69\
\x74\x65\x72\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\0\0\0\0\0\0";
static const char opts_insn[] __attribute__((__aligned__(8))) = "\
\xbf\x61\0\0\0\0\0\0\xbf\x1a\0\0\0\0\0\0\x07\x10\0\0\xff\xff\xff\x78\xb7\x20\0\
\0\0\0\0\x88\xb7\x30\0\0\0\0\0\0\x85\0\0\0\0\0\0\x71\x05\0\0\x14\0\0\0\0\x61\
\x1a\xff\x78\0\0\0\0\xd5\x10\0\x01\0\0\0\0\x85\0\0\0\0\0\0\xa8\x61\x1a\xff\x7c\
@ -318,72 +321,87 @@ iterators_bpf__load(struct iterators_bpf *skel)
\0\0\0\x85\0\0\0\0\0\0\xa8\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x61\x10\0\0\0\0\
\0\0\xd5\x10\0\x02\0\0\0\0\xbf\x91\0\0\0\0\0\0\x85\0\0\0\0\0\0\xa8\xbf\x07\0\0\
\0\0\0\0\x95\0\0\0\0\0\0\0\x61\x06\0\x08\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\
\0\x0e\x68\x63\x10\0\0\0\0\0\0\x61\x06\0\x0c\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\
\0\0\0\x0e\x64\x63\x10\0\0\0\0\0\0\x79\x06\0\x10\0\0\0\0\x18\x16\0\0\0\0\0\0\0\
\0\0\0\0\0\x0e\x58\x7b\x10\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x05\0\
\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x0e\x50\x7b\x10\0\0\0\0\0\0\xb7\x10\0\0\0\0\0\
\x12\x18\x26\0\0\0\0\0\0\0\0\0\0\0\0\x0e\x50\xb7\x30\0\0\0\0\0\x1c\x85\0\0\0\0\
\0\x0e\xf8\x63\x10\0\0\0\0\0\0\x61\x06\0\x0c\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\
\0\0\0\x0e\xf4\x63\x10\0\0\0\0\0\0\x79\x06\0\x10\0\0\0\0\x18\x16\0\0\0\0\0\0\0\
\0\0\0\0\0\x0e\xe8\x7b\x10\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x05\0\
\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x0e\xe0\x7b\x10\0\0\0\0\0\0\xb7\x10\0\0\0\0\0\
\x12\x18\x26\0\0\0\0\0\0\0\0\0\0\0\0\x0e\xe0\xb7\x30\0\0\0\0\0\x1c\x85\0\0\0\0\
\0\0\xa6\xbf\x70\0\0\0\0\0\0\xc5\x70\xff\xd4\0\0\0\0\x63\xa7\xff\x78\0\0\0\0\
\x61\x0a\xff\x78\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x0e\xa0\x63\x10\0\0\0\
\x61\x0a\xff\x78\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x0f\x30\x63\x10\0\0\0\
\0\0\0\x61\x06\0\x1c\0\0\0\0\x15\0\0\x03\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\
\0\x0e\x7c\x63\x10\0\0\0\0\0\0\xb7\x10\0\0\0\0\0\0\x18\x26\0\0\0\0\0\0\0\0\0\0\
\0\0\x0e\x70\xb7\x30\0\0\0\0\0\x48\x85\0\0\0\0\0\0\xa6\xbf\x70\0\0\0\0\0\0\xc5\
\0\x0f\x0c\x63\x10\0\0\0\0\0\0\xb7\x10\0\0\0\0\0\0\x18\x26\0\0\0\0\0\0\0\0\0\0\
\0\0\x0f\0\xb7\x30\0\0\0\0\0\x48\x85\0\0\0\0\0\0\xa6\xbf\x70\0\0\0\0\0\0\xc5\
\x70\xff\xc3\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x63\x17\0\0\0\0\0\0\
\x79\x36\0\x20\0\0\0\0\x15\x30\0\x08\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\
\x0e\xb8\xb7\x20\0\0\0\0\0\x62\x61\x06\0\x04\0\0\0\0\x45\0\0\x02\0\0\0\x01\x85\
\x0f\x48\xb7\x20\0\0\0\0\0\x7b\x61\x06\0\x04\0\0\0\0\x45\0\0\x02\0\0\0\x01\x85\
\0\0\0\0\0\0\x94\x05\0\0\x01\0\0\0\0\x85\0\0\0\0\0\0\x71\x18\x26\0\0\0\0\0\0\0\
\0\0\0\0\0\0\0\x61\x02\0\0\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x0f\x28\x63\
\x10\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x0f\x20\x18\x16\0\0\0\0\0\0\0\
\0\0\0\0\0\x0f\x30\x7b\x10\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x0e\xb8\
\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x0f\x38\x7b\x10\0\0\0\0\0\0\xb7\x10\0\0\0\0\0\
\x02\x18\x26\0\0\0\0\0\0\0\0\0\0\0\0\x0f\x28\xb7\x30\0\0\0\0\0\x20\x85\0\0\0\0\
\0\0\0\0\0\0\0\x61\x02\0\0\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x0f\xd0\x63\
\x10\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x0f\xc8\x18\x16\0\0\0\0\0\0\0\
\0\0\0\0\0\x0f\xd8\x7b\x10\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x0f\x48\
\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x0f\xe0\x7b\x10\0\0\0\0\0\0\xb7\x10\0\0\0\0\0\
\x02\x18\x26\0\0\0\0\0\0\0\0\0\0\0\0\x0f\xd0\xb7\x30\0\0\0\0\0\x20\x85\0\0\0\0\
\0\0\xa6\xbf\x70\0\0\0\0\0\0\xc5\x70\xff\x9f\0\0\0\0\x18\x26\0\0\0\0\0\0\0\0\0\
\0\0\0\0\0\x61\x02\0\0\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x0f\x48\x63\x10\
\0\0\0\0\0\0\xb7\x10\0\0\0\0\0\x16\x18\x26\0\0\0\0\0\0\0\0\0\0\0\0\x0f\x48\xb7\
\0\0\0\0\0\x61\x02\0\0\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x0f\xf0\x63\x10\
\0\0\0\0\0\0\xb7\x10\0\0\0\0\0\x16\x18\x26\0\0\0\0\0\0\0\0\0\0\0\0\x0f\xf0\xb7\
\x30\0\0\0\0\0\x04\x85\0\0\0\0\0\0\xa6\xbf\x70\0\0\0\0\0\0\xc5\x70\xff\x92\0\0\
\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x0f\x50\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\
\x11\x70\x7b\x10\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x0f\x58\x18\x16\0\
\0\0\0\0\0\0\0\0\0\0\0\x11\x68\x7b\x10\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\
\0\0\x10\x58\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x11\xb0\x7b\x10\0\0\0\0\0\0\x18\
\x06\0\0\0\0\0\0\0\0\0\0\0\0\x10\x60\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x11\xc0\
\x7b\x10\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x10\xf0\x18\x16\0\0\0\0\0\
\0\0\0\0\0\0\0\x11\xe0\x7b\x10\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x11\xd8\x7b\x10\0\0\0\0\0\0\x61\x06\0\x08\0\0\
\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x11\x78\x63\x10\0\0\0\0\0\0\x61\x06\0\x0c\
\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x11\x7c\x63\x10\0\0\0\0\0\0\x79\x06\0\
\x10\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x11\x80\x7b\x10\0\0\0\0\0\0\x61\
\x0a\xff\x78\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x11\xa8\x63\x10\0\0\0\0\0\
\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x11\xf0\xb7\x20\0\0\0\0\0\x11\xb7\x30\0\0\0\
\0\0\x0c\xb7\x40\0\0\0\0\0\0\x85\0\0\0\0\0\0\xa7\xbf\x70\0\0\0\0\0\0\xc5\x70\
\xff\x5c\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x11\x60\x63\x07\0\x6c\0\0\0\0\
\x77\x70\0\0\0\0\0\x20\x63\x07\0\x70\0\0\0\0\xb7\x10\0\0\0\0\0\x05\x18\x26\0\0\
\0\0\0\0\0\0\0\0\0\0\x11\x60\xb7\x30\0\0\0\0\0\x8c\x85\0\0\0\0\0\0\xa6\xbf\x70\
\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x11\xd0\x61\x10\0\0\0\0\0\0\xd5\
\x10\0\x02\0\0\0\0\xbf\x91\0\0\0\0\0\0\x85\0\0\0\0\0\0\xa8\xc5\x70\xff\x4a\0\0\
\0\0\x63\xa7\xff\x80\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x12\x08\x18\x16\0\
\0\0\0\0\0\0\0\0\0\0\0\x16\xe0\x7b\x10\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\
\0\0\x12\x10\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x16\xd8\x7b\x10\0\0\0\0\0\0\x18\
\x06\0\0\0\0\0\0\0\0\0\0\0\0\x14\x18\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x17\x20\
\x7b\x10\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x14\x20\x18\x16\0\0\0\0\0\
\0\0\0\0\0\0\0\x17\x30\x7b\x10\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x15\
\xb0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x17\x50\x7b\x10\0\0\0\0\0\0\x18\x06\0\0\0\
\0\0\0\0\0\0\0\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x17\x48\x7b\x10\0\0\0\0\
\0\0\x61\x06\0\x08\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x16\xe8\x63\x10\0\0\
\0\0\0\0\x61\x06\0\x0c\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x16\xec\x63\x10\
\0\0\0\0\0\0\x79\x06\0\x10\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x16\xf0\x7b\
\x10\0\0\0\0\0\0\x61\x0a\xff\x78\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x17\
\x18\x63\x10\0\0\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x17\x60\xb7\x20\0\0\0\
\0\0\x12\xb7\x30\0\0\0\0\0\x0c\xb7\x40\0\0\0\0\0\0\x85\0\0\0\0\0\0\xa7\xbf\x70\
\0\0\0\0\0\0\xc5\x70\xff\x13\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x16\xd0\
\x63\x07\0\x6c\0\0\0\0\x77\x70\0\0\0\0\0\x20\x63\x07\0\x70\0\0\0\0\xb7\x10\0\0\
\0\0\0\x05\x18\x26\0\0\0\0\0\0\0\0\0\0\0\0\x16\xd0\xb7\x30\0\0\0\0\0\x8c\x85\0\
\0\0\0\0\0\xa6\xbf\x70\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x17\x40\x61\
\x10\0\0\0\0\0\0\xd5\x10\0\x02\0\0\0\0\xbf\x91\0\0\0\0\0\0\x85\0\0\0\0\0\0\xa8\
\xc5\x70\xff\x01\0\0\0\0\x63\xa7\xff\x84\0\0\0\0\x61\x1a\xff\x78\0\0\0\0\xd5\
\x10\0\x02\0\0\0\0\xbf\x91\0\0\0\0\0\0\x85\0\0\0\0\0\0\xa8\x61\x0a\xff\x80\0\0\
\0\0\x63\x60\0\x28\0\0\0\0\x61\x0a\xff\x84\0\0\0\0\x63\x60\0\x2c\0\0\0\0\x18\
\x16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x61\x01\0\0\0\0\0\0\x63\x60\0\x18\0\0\0\0\xb7\
\0\0\0\0\0\0\0\x95\0\0\0\0\0\0\0";
\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x0f\xf8\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\
\x12\x30\x7b\x10\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x10\0\x18\x16\0\0\
\0\0\0\0\0\0\0\0\0\0\x12\x28\x7b\x10\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\
\0\x11\x18\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x12\x70\x7b\x10\0\0\0\0\0\0\x18\x06\
\0\0\0\0\0\0\0\0\0\0\0\0\x11\x20\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x12\x80\x7b\
\x10\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x11\xb0\x18\x16\0\0\0\0\0\0\0\
\0\0\0\0\0\x12\xa0\x7b\x10\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x18\
\x16\0\0\0\0\0\0\0\0\0\0\0\0\x12\x98\x7b\x10\0\0\0\0\0\0\x61\x06\0\x08\0\0\0\0\
\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x12\x38\x63\x10\0\0\0\0\0\0\x61\x06\0\x0c\0\0\
\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x12\x3c\x63\x10\0\0\0\0\0\0\x79\x06\0\x10\
\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x12\x40\x7b\x10\0\0\0\0\0\0\x61\x0a\
\xff\x78\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x12\x68\x63\x10\0\0\0\0\0\0\
\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x12\xb0\xb7\x20\0\0\0\0\0\x11\xb7\x30\0\0\0\0\
\0\x0c\xb7\x40\0\0\0\0\0\0\x85\0\0\0\0\0\0\xa7\xbf\x70\0\0\0\0\0\0\xc5\x70\xff\
\x5c\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x12\x20\x63\x07\0\x6c\0\0\0\0\x77\
\x70\0\0\0\0\0\x20\x63\x07\0\x70\0\0\0\0\x18\x86\0\0\0\0\0\0\0\0\0\0\0\0\x10\
\xb8\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x12\xc8\xb7\x20\0\0\0\0\0\x17\xb7\x30\0\0\
\0\0\0\x0c\xb7\x40\0\0\0\0\0\0\x85\0\0\0\0\0\0\xa7\xbf\x70\0\0\0\0\0\0\xc5\x70\
\xff\x4d\0\0\0\0\x75\x70\0\x03\0\0\0\0\x62\x80\0\x04\0\0\0\0\x6a\x80\0\x02\0\0\
\0\0\x05\0\0\x0a\0\0\0\0\x63\x87\0\x04\0\0\0\0\xbf\x97\0\0\0\0\0\0\x77\x90\0\0\
\0\0\0\x20\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\x63\x09\0\0\0\0\0\0\x55\x90\0\
\x02\0\0\0\0\x6a\x80\0\x02\0\0\0\0\x05\0\0\x01\0\0\0\0\x6a\x80\0\x02\0\0\0\x40\
\xb7\x10\0\0\0\0\0\x05\x18\x26\0\0\0\0\0\0\0\0\0\0\0\0\x12\x20\xb7\x30\0\0\0\0\
\0\x8c\x85\0\0\0\0\0\0\xa6\xbf\x70\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\
\x01\0\x61\x10\0\0\0\0\0\0\xd5\x10\0\x02\0\0\0\0\xbf\x91\0\0\0\0\0\0\x85\0\0\0\
\0\0\0\xa8\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x12\x90\x61\x10\0\0\0\0\0\0\xd5\x10\
\0\x02\0\0\0\0\xbf\x91\0\0\0\0\0\0\x85\0\0\0\0\0\0\xa8\xc5\x70\xff\x2c\0\0\0\0\
\x63\xa7\xff\x80\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x12\xe0\x18\x16\0\0\0\
\0\0\0\0\0\0\0\0\0\x17\x88\x7b\x10\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\
\x12\xe8\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x17\x80\x7b\x10\0\0\0\0\0\0\x18\x06\0\
\0\0\0\0\0\0\0\0\0\0\0\x14\xf0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x17\xc8\x7b\x10\
\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x14\xf8\x18\x16\0\0\0\0\0\0\0\0\0\
\0\0\0\x17\xd8\x7b\x10\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x16\x58\x18\
\x16\0\0\0\0\0\0\0\0\0\0\0\0\x17\xf8\x7b\x10\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\
\0\0\0\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x17\xf0\x7b\x10\0\0\0\0\0\0\x61\
\x06\0\x08\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x17\x90\x63\x10\0\0\0\0\0\0\
\x61\x06\0\x0c\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x17\x94\x63\x10\0\0\0\0\
\0\0\x79\x06\0\x10\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x17\x98\x7b\x10\0\0\
\0\0\0\0\x61\x0a\xff\x78\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x17\xc0\x63\
\x10\0\0\0\0\0\0\x18\x16\0\0\0\0\0\0\0\0\0\0\0\0\x18\x08\xb7\x20\0\0\0\0\0\x12\
\xb7\x30\0\0\0\0\0\x0c\xb7\x40\0\0\0\0\0\0\x85\0\0\0\0\0\0\xa7\xbf\x70\0\0\0\0\
\0\0\xc5\x70\xfe\xf5\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x17\x78\x63\x07\0\
\x6c\0\0\0\0\x77\x70\0\0\0\0\0\x20\x63\x07\0\x70\0\0\0\0\xb7\x10\0\0\0\0\0\x05\
\x18\x26\0\0\0\0\0\0\0\0\0\0\0\0\x17\x78\xb7\x30\0\0\0\0\0\x8c\x85\0\0\0\0\0\0\
\xa6\xbf\x70\0\0\0\0\0\0\x18\x06\0\0\0\0\0\0\0\0\0\0\0\0\x17\xe8\x61\x10\0\0\0\
\0\0\0\xd5\x10\0\x02\0\0\0\0\xbf\x91\0\0\0\0\0\0\x85\0\0\0\0\0\0\xa8\xc5\x70\
\xfe\xe3\0\0\0\0\x63\xa7\xff\x84\0\0\0\0\x61\x1a\xff\x78\0\0\0\0\xd5\x10\0\x02\
\0\0\0\0\xbf\x91\0\0\0\0\0\0\x85\0\0\0\0\0\0\xa8\x61\x0a\xff\x80\0\0\0\0\x63\
\x60\0\x28\0\0\0\0\x61\x0a\xff\x84\0\0\0\0\x63\x60\0\x2c\0\0\0\0\x18\x16\0\0\0\
\0\0\0\0\0\0\0\0\0\0\0\x61\x01\0\0\0\0\0\0\x63\x60\0\x18\0\0\0\0\xb7\0\0\0\0\0\
\0\0\x95\0\0\0\0\0\0\0";
opts.ctx = (struct bpf_loader_ctx *)skel;
opts.data_sz = sizeof(opts_data) - 1;
opts.data = (void *)opts_data;
opts.insns_sz = sizeof(opts_insn) - 1;
opts.insns = (void *)opts_insn;
err = bpf_load_and_run(&opts);
if (err < 0)
return err;

3
kernel/bpf/prog_iter.c
View file

@ -78,8 +78,7 @@ static const struct seq_operations bpf_prog_seq_ops = {
.show = bpf_prog_seq_show,
};
BTF_ID_LIST(btf_bpf_prog_id)
BTF_ID(struct, bpf_prog)
BTF_ID_LIST_SINGLE(btf_bpf_prog_id, struct, bpf_prog)
static const struct bpf_iter_seq_info bpf_prog_seq_info = {
.seq_ops = &bpf_prog_seq_ops,

23
kernel/bpf/rqspinlock.c
View file

@ -666,6 +666,27 @@ EXPORT_SYMBOL_GPL(resilient_queued_spin_lock_slowpath);
__bpf_kfunc_start_defs();
static void bpf_prog_report_rqspinlock_violation(const char *str, void *lock, bool irqsave)
{
struct rqspinlock_held *rqh = this_cpu_ptr(&rqspinlock_held_locks);
struct bpf_stream_stage ss;
struct bpf_prog *prog;
prog = bpf_prog_find_from_stack();
if (!prog)
return;
bpf_stream_stage(ss, prog, BPF_STDERR, ({
bpf_stream_printk(ss, "ERROR: %s for bpf_res_spin_lock%s\n", str, irqsave ? "_irqsave" : "");
bpf_stream_printk(ss, "Attempted lock = 0x%px\n", lock);
bpf_stream_printk(ss, "Total held locks = %d\n", rqh->cnt);
for (int i = 0; i < min(RES_NR_HELD, rqh->cnt); i++)
bpf_stream_printk(ss, "Held lock[%2d] = 0x%px\n", i, rqh->locks[i]);
bpf_stream_dump_stack(ss);
}));
}
#define REPORT_STR(ret) ({ (ret) == -ETIMEDOUT ? "Timeout detected" : "AA or ABBA deadlock detected"; })
__bpf_kfunc int bpf_res_spin_lock(struct bpf_res_spin_lock *lock)
{
int ret;
@ -676,6 +697,7 @@ __bpf_kfunc int bpf_res_spin_lock(struct bpf_res_spin_lock *lock)
preempt_disable();
ret = res_spin_lock((rqspinlock_t *)lock);
if (unlikely(ret)) {
bpf_prog_report_rqspinlock_violation(REPORT_STR(ret), lock, false);
preempt_enable();
return ret;
}
@ -698,6 +720,7 @@ __bpf_kfunc int bpf_res_spin_lock_irqsave(struct bpf_res_spin_lock *lock, unsign
local_irq_save(flags);
ret = res_spin_lock((rqspinlock_t *)lock);
if (unlikely(ret)) {
bpf_prog_report_rqspinlock_violation(REPORT_STR(ret), lock, true);
local_irq_restore(flags);
preempt_enable();
return ret;

526
kernel/bpf/stream.c Normal file
View file

@ -0,0 +1,526 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
#include <linux/bpf.h>
#include <linux/filter.h>
#include <linux/bpf_mem_alloc.h>
#include <linux/percpu.h>
#include <linux/refcount.h>
#include <linux/gfp.h>
#include <linux/memory.h>
#include <linux/local_lock.h>
#include <linux/mutex.h>
/*
* Simple per-CPU NMI-safe bump allocation mechanism, backed by the NMI-safe
* try_alloc_pages()/free_pages_nolock() primitives. We allocate a page and
* stash it in a local per-CPU variable, and bump allocate from the page
* whenever items need to be printed to a stream. Each page holds a global
* atomic refcount in its first 4 bytes, and then records of variable length
* that describe the printed messages. Once the global refcount has dropped to
* zero, it is a signal to free the page back to the kernel's page allocator,
* given all the individual records in it have been consumed.
*
* It is possible the same page is used to serve allocations across different
* programs, which may be consumed at different times individually, hence
* maintaining a reference count per-page is critical for correct lifetime
* tracking.
*
* The bpf_stream_page code will be replaced to use kmalloc_nolock() once it
* lands.
*/
struct bpf_stream_page {
refcount_t ref;
u32 consumed;
char buf[];
};
/* Available room to add data to a refcounted page. */
#define BPF_STREAM_PAGE_SZ (PAGE_SIZE - offsetofend(struct bpf_stream_page, consumed))
static DEFINE_PER_CPU(local_trylock_t, stream_local_lock) = INIT_LOCAL_TRYLOCK(stream_local_lock);
static DEFINE_PER_CPU(struct bpf_stream_page *, stream_pcpu_page);
static bool bpf_stream_page_local_lock(unsigned long *flags)
{
return local_trylock_irqsave(&stream_local_lock, *flags);
}
static void bpf_stream_page_local_unlock(unsigned long *flags)
{
local_unlock_irqrestore(&stream_local_lock, *flags);
}
static void bpf_stream_page_free(struct bpf_stream_page *stream_page)
{
struct page *p;
if (!stream_page)
return;
p = virt_to_page(stream_page);
free_pages_nolock(p, 0);
}
static void bpf_stream_page_get(struct bpf_stream_page *stream_page)
{
refcount_inc(&stream_page->ref);
}
static void bpf_stream_page_put(struct bpf_stream_page *stream_page)
{
if (refcount_dec_and_test(&stream_page->ref))
bpf_stream_page_free(stream_page);
}
static void bpf_stream_page_init(struct bpf_stream_page *stream_page)
{
refcount_set(&stream_page->ref, 1);
stream_page->consumed = 0;
}
static struct bpf_stream_page *bpf_stream_page_replace(void)
{
struct bpf_stream_page *stream_page, *old_stream_page;
struct page *page;
page = alloc_pages_nolock(NUMA_NO_NODE, 0);
if (!page)
return NULL;
stream_page = page_address(page);
bpf_stream_page_init(stream_page);
old_stream_page = this_cpu_read(stream_pcpu_page);
if (old_stream_page)
bpf_stream_page_put(old_stream_page);
this_cpu_write(stream_pcpu_page, stream_page);
return stream_page;
}
static int bpf_stream_page_check_room(struct bpf_stream_page *stream_page, int len)
{
int min = offsetof(struct bpf_stream_elem, str[0]);
int consumed = stream_page->consumed;
int total = BPF_STREAM_PAGE_SZ;
int rem = max(0, total - consumed - min);
/* Let's give room of at least 8 bytes. */
WARN_ON_ONCE(rem % 8 != 0);
rem = rem < 8 ? 0 : rem;
return min(len, rem);
}
static void bpf_stream_elem_init(struct bpf_stream_elem *elem, int len)
{
init_llist_node(&elem->node);
elem->total_len = len;
elem->consumed_len = 0;
}
static struct bpf_stream_page *bpf_stream_page_from_elem(struct bpf_stream_elem *elem)
{
unsigned long addr = (unsigned long)elem;
return (struct bpf_stream_page *)PAGE_ALIGN_DOWN(addr);
}
static struct bpf_stream_elem *bpf_stream_page_push_elem(struct bpf_stream_page *stream_page, int len)
{
u32 consumed = stream_page->consumed;
stream_page->consumed += round_up(offsetof(struct bpf_stream_elem, str[len]), 8);
return (struct bpf_stream_elem *)&stream_page->buf[consumed];
}
static struct bpf_stream_elem *bpf_stream_page_reserve_elem(int len)
{
struct bpf_stream_elem *elem = NULL;
struct bpf_stream_page *page;
int room = 0;
page = this_cpu_read(stream_pcpu_page);
if (!page)
page = bpf_stream_page_replace();
if (!page)
return NULL;
room = bpf_stream_page_check_room(page, len);
if (room != len)
page = bpf_stream_page_replace();
if (!page)
return NULL;
bpf_stream_page_get(page);
room = bpf_stream_page_check_room(page, len);
WARN_ON_ONCE(room != len);
elem = bpf_stream_page_push_elem(page, room);
bpf_stream_elem_init(elem, room);
return elem;
}
static struct bpf_stream_elem *bpf_stream_elem_alloc(int len)
{
const int max_len = ARRAY_SIZE((struct bpf_bprintf_buffers){}.buf);
struct bpf_stream_elem *elem;
unsigned long flags;
BUILD_BUG_ON(max_len > BPF_STREAM_PAGE_SZ);
/*
* Length denotes the amount of data to be written as part of stream element,
* thus includes '\0' byte. We're capped by how much bpf_bprintf_buffers can
* accomodate, therefore deny allocations that won't fit into them.
*/
if (len < 0 || len > max_len)
return NULL;
if (!bpf_stream_page_local_lock(&flags))
return NULL;
elem = bpf_stream_page_reserve_elem(len);
bpf_stream_page_local_unlock(&flags);
return elem;
}
static int __bpf_stream_push_str(struct llist_head *log, const char *str, int len)
{
struct bpf_stream_elem *elem = NULL;
/*
* Allocate a bpf_prog_stream_elem and push it to the bpf_prog_stream
* log, elements will be popped at once and reversed to print the log.
*/
elem = bpf_stream_elem_alloc(len);
if (!elem)
return -ENOMEM;
memcpy(elem->str, str, len);
llist_add(&elem->node, log);
return 0;
}
static int bpf_stream_consume_capacity(struct bpf_stream *stream, int len)
{
if (atomic_read(&stream->capacity) >= BPF_STREAM_MAX_CAPACITY)
return -ENOSPC;
if (atomic_add_return(len, &stream->capacity) >= BPF_STREAM_MAX_CAPACITY) {
atomic_sub(len, &stream->capacity);
return -ENOSPC;
}
return 0;
}
static void bpf_stream_release_capacity(struct bpf_stream *stream, struct bpf_stream_elem *elem)
{
int len = elem->total_len;
atomic_sub(len, &stream->capacity);
}
static int bpf_stream_push_str(struct bpf_stream *stream, const char *str, int len)
{
int ret = bpf_stream_consume_capacity(stream, len);
return ret ?: __bpf_stream_push_str(&stream->log, str, len);
}
static struct bpf_stream *bpf_stream_get(enum bpf_stream_id stream_id, struct bpf_prog_aux *aux)
{
if (stream_id != BPF_STDOUT && stream_id != BPF_STDERR)
return NULL;
return &aux->stream[stream_id - 1];
}
static void bpf_stream_free_elem(struct bpf_stream_elem *elem)
{
struct bpf_stream_page *p;
p = bpf_stream_page_from_elem(elem);
bpf_stream_page_put(p);
}
static void bpf_stream_free_list(struct llist_node *list)
{
struct bpf_stream_elem *elem, *tmp;
llist_for_each_entry_safe(elem, tmp, list, node)
bpf_stream_free_elem(elem);
}
static struct llist_node *bpf_stream_backlog_peek(struct bpf_stream *stream)
{
return stream->backlog_head;
}
static struct llist_node *bpf_stream_backlog_pop(struct bpf_stream *stream)
{
struct llist_node *node;
node = stream->backlog_head;
if (stream->backlog_head == stream->backlog_tail)
stream->backlog_head = stream->backlog_tail = NULL;
else
stream->backlog_head = node->next;
return node;
}
static void bpf_stream_backlog_fill(struct bpf_stream *stream)
{
struct llist_node *head, *tail;
if (llist_empty(&stream->log))
return;
tail = llist_del_all(&stream->log);
if (!tail)
return;
head = llist_reverse_order(tail);
if (!stream->backlog_head) {
stream->backlog_head = head;
stream->backlog_tail = tail;
} else {
stream->backlog_tail->next = head;
stream->backlog_tail = tail;
}
return;
}
static bool bpf_stream_consume_elem(struct bpf_stream_elem *elem, int *len)
{
int rem = elem->total_len - elem->consumed_len;
int used = min(rem, *len);
elem->consumed_len += used;
*len -= used;
return elem->consumed_len == elem->total_len;
}
static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len)
{
int rem_len = len, cons_len, ret = 0;
struct bpf_stream_elem *elem = NULL;
struct llist_node *node;
mutex_lock(&stream->lock);
while (rem_len) {
int pos = len - rem_len;
bool cont;
node = bpf_stream_backlog_peek(stream);
if (!node) {
bpf_stream_backlog_fill(stream);
node = bpf_stream_backlog_peek(stream);
}
if (!node)
break;
elem = container_of(node, typeof(*elem), node);
cons_len = elem->consumed_len;
cont = bpf_stream_consume_elem(elem, &rem_len) == false;
ret = copy_to_user(buf + pos, elem->str + cons_len,
elem->consumed_len - cons_len);
/* Restore in case of error. */
if (ret) {
ret = -EFAULT;
elem->consumed_len = cons_len;
break;
}
if (cont)
continue;
bpf_stream_backlog_pop(stream);
bpf_stream_release_capacity(stream, elem);
bpf_stream_free_elem(elem);
}
mutex_unlock(&stream->lock);
return ret ? ret : len - rem_len;
}
int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len)
{
struct bpf_stream *stream;
stream = bpf_stream_get(stream_id, prog->aux);
if (!stream)
return -ENOENT;
return bpf_stream_read(stream, buf, len);
}
__bpf_kfunc_start_defs();
/*
* Avoid using enum bpf_stream_id so that kfunc users don't have to pull in the
* enum in headers.
*/
__bpf_kfunc int bpf_stream_vprintk(int stream_id, const char *fmt__str, const void *args, u32 len__sz, void *aux__prog)
{
struct bpf_bprintf_data data = {
.get_bin_args = true,
.get_buf = true,
};
struct bpf_prog_aux *aux = aux__prog;
u32 fmt_size = strlen(fmt__str) + 1;
struct bpf_stream *stream;
u32 data_len = len__sz;
int ret, num_args;
stream = bpf_stream_get(stream_id, aux);
if (!stream)
return -ENOENT;
if (data_len & 7 || data_len > MAX_BPRINTF_VARARGS * 8 ||
(data_len && !args))
return -EINVAL;
num_args = data_len / 8;
ret = bpf_bprintf_prepare(fmt__str, fmt_size, args, num_args, &data);
if (ret < 0)
return ret;
ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt__str, data.bin_args);
/* Exclude NULL byte during push. */
ret = bpf_stream_push_str(stream, data.buf, ret);
bpf_bprintf_cleanup(&data);
return ret;
}
__bpf_kfunc_end_defs();
/* Added kfunc to common_btf_ids */
void bpf_prog_stream_init(struct bpf_prog *prog)
{
int i;
for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) {
atomic_set(&prog->aux->stream[i].capacity, 0);
init_llist_head(&prog->aux->stream[i].log);
mutex_init(&prog->aux->stream[i].lock);
prog->aux->stream[i].backlog_head = NULL;
prog->aux->stream[i].backlog_tail = NULL;
}
}
void bpf_prog_stream_free(struct bpf_prog *prog)
{
struct llist_node *list;
int i;
for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) {
list = llist_del_all(&prog->aux->stream[i].log);
bpf_stream_free_list(list);
bpf_stream_free_list(prog->aux->stream[i].backlog_head);
}
}
void bpf_stream_stage_init(struct bpf_stream_stage *ss)
{
init_llist_head(&ss->log);
ss->len = 0;
}
void bpf_stream_stage_free(struct bpf_stream_stage *ss)
{
struct llist_node *node;
node = llist_del_all(&ss->log);
bpf_stream_free_list(node);
}
int bpf_stream_stage_printk(struct bpf_stream_stage *ss, const char *fmt, ...)
{
struct bpf_bprintf_buffers *buf;
va_list args;
int ret;
if (bpf_try_get_buffers(&buf))
return -EBUSY;
va_start(args, fmt);
ret = vsnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args);
va_end(args);
ss->len += ret;
/* Exclude NULL byte during push. */
ret = __bpf_stream_push_str(&ss->log, buf->buf, ret);
bpf_put_buffers();
return ret;
}
int bpf_stream_stage_commit(struct bpf_stream_stage *ss, struct bpf_prog *prog,
enum bpf_stream_id stream_id)
{
struct llist_node *list, *head, *tail;
struct bpf_stream *stream;
int ret;
stream = bpf_stream_get(stream_id, prog->aux);
if (!stream)
return -EINVAL;
ret = bpf_stream_consume_capacity(stream, ss->len);
if (ret)
return ret;
list = llist_del_all(&ss->log);
head = tail = list;
if (!list)
return 0;
while (llist_next(list)) {
tail = llist_next(list);
list = tail;
}
llist_add_batch(head, tail, &stream->log);
return 0;
}
struct dump_stack_ctx {
struct bpf_stream_stage *ss;
int err;
};
static bool dump_stack_cb(void *cookie, u64 ip, u64 sp, u64 bp)
{
struct dump_stack_ctx *ctxp = cookie;
const char *file = "", *line = "";
struct bpf_prog *prog;
int num, ret;
rcu_read_lock();
prog = bpf_prog_ksym_find(ip);
rcu_read_unlock();
if (prog) {
ret = bpf_prog_get_file_line(prog, ip, &file, &line, &num);
if (ret < 0)
goto end;
ctxp->err = bpf_stream_stage_printk(ctxp->ss, "%pS\n %s @ %s:%d\n",
(void *)(long)ip, line, file, num);
return !ctxp->err;
}
end:
ctxp->err = bpf_stream_stage_printk(ctxp->ss, "%pS\n", (void *)(long)ip);
return !ctxp->err;
}
int bpf_stream_stage_dump_stack(struct bpf_stream_stage *ss)
{
struct dump_stack_ctx ctx = { .ss = ss };
int ret;
ret = bpf_stream_stage_printk(ss, "CPU: %d UID: %d PID: %d Comm: %s\n",
raw_smp_processor_id(), __kuid_val(current_real_cred()->euid),
current->pid, current->comm);
if (ret)
return ret;
ret = bpf_stream_stage_printk(ss, "Call trace:\n");
if (ret)
return ret;
arch_bpf_stack_walk(dump_stack_cb, &ctx);
if (ctx.err)
return ctx.err;
return bpf_stream_stage_printk(ss, "\n");
}

289
kernel/bpf/syscall.c
View file

@ -3069,7 +3069,7 @@ static int bpf_obj_get(const union bpf_attr *attr)
*/
void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type,
const struct bpf_link_ops *ops, struct bpf_prog *prog,
bool sleepable)
enum bpf_attach_type attach_type, bool sleepable)
{
WARN_ON(ops->dealloc && ops->dealloc_deferred);
atomic64_set(&link->refcnt, 1);
@ -3078,12 +3078,14 @@ void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type,
link->id = 0;
link->ops = ops;
link->prog = prog;
link->attach_type = attach_type;
}
void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
const struct bpf_link_ops *ops, struct bpf_prog *prog)
const struct bpf_link_ops *ops, struct bpf_prog *prog,
enum bpf_attach_type attach_type)
{
bpf_link_init_sleepable(link, type, ops, prog, false);
bpf_link_init_sleepable(link, type, ops, prog, attach_type, false);
}
static void bpf_link_free_id(int id)
@ -3228,7 +3230,14 @@ static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp)
char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
if (type < ARRAY_SIZE(bpf_link_type_strs) && bpf_link_type_strs[type]) {
seq_printf(m, "link_type:\t%s\n", bpf_link_type_strs[type]);
if (link->type == BPF_LINK_TYPE_KPROBE_MULTI)
seq_printf(m, "link_type:\t%s\n", link->flags == BPF_F_KPROBE_MULTI_RETURN ?
"kretprobe_multi" : "kprobe_multi");
else if (link->type == BPF_LINK_TYPE_UPROBE_MULTI)
seq_printf(m, "link_type:\t%s\n", link->flags == BPF_F_UPROBE_MULTI_RETURN ?
"uretprobe_multi" : "uprobe_multi");
else
seq_printf(m, "link_type:\t%s\n", bpf_link_type_strs[type]);
} else {
WARN_ONCE(1, "missing BPF_LINK_TYPE(...) for link type %u\n", type);
seq_printf(m, "link_type:\t<%u>\n", type);
@ -3403,10 +3412,12 @@ static void bpf_tracing_link_show_fdinfo(const struct bpf_link *link,
seq_printf(seq,
"attach_type:\t%d\n"
"target_obj_id:\t%u\n"
"target_btf_id:\t%u\n",
tr_link->attach_type,
"target_btf_id:\t%u\n"
"cookie:\t%llu\n",
link->attach_type,
target_obj_id,
target_btf_id);
target_btf_id,
tr_link->link.cookie);
}
static int bpf_tracing_link_fill_link_info(const struct bpf_link *link,
@ -3415,7 +3426,8 @@ static int bpf_tracing_link_fill_link_info(const struct bpf_link *link,
struct bpf_tracing_link *tr_link =
container_of(link, struct bpf_tracing_link, link.link);
info->tracing.attach_type = tr_link->attach_type;
info->tracing.attach_type = link->attach_type;
info->tracing.cookie = tr_link->link.cookie;
bpf_trampoline_unpack_key(tr_link->trampoline->key,
&info->tracing.target_obj_id,
&info->tracing.target_btf_id);
@ -3433,7 +3445,8 @@ static const struct bpf_link_ops bpf_tracing_link_lops = {
static int bpf_tracing_prog_attach(struct bpf_prog *prog,
int tgt_prog_fd,
u32 btf_id,
u64 bpf_cookie)
u64 bpf_cookie,
enum bpf_attach_type attach_type)
{
struct bpf_link_primer link_primer;
struct bpf_prog *tgt_prog = NULL;
@ -3501,8 +3514,8 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
goto out_put_prog;
}
bpf_link_init(&link->link.link, BPF_LINK_TYPE_TRACING,
&bpf_tracing_link_lops, prog);
link->attach_type = prog->expected_attach_type;
&bpf_tracing_link_lops, prog, attach_type);
link->link.cookie = bpf_cookie;
mutex_lock(&prog->aux->dst_mutex);
@ -3651,8 +3664,10 @@ static void bpf_raw_tp_link_show_fdinfo(const struct bpf_link *link,
container_of(link, struct bpf_raw_tp_link, link);
seq_printf(seq,
"tp_name:\t%s\n",
raw_tp_link->btp->tp->name);
"tp_name:\t%s\n"
"cookie:\t%llu\n",
raw_tp_link->btp->tp->name,
raw_tp_link->cookie);
}
static int bpf_copy_to_user(char __user *ubuf, const char *buf, u32 ulen,
@ -3688,6 +3703,7 @@ static int bpf_raw_tp_link_fill_link_info(const struct bpf_link *link,
return -EINVAL;
info->raw_tracepoint.tp_name_len = tp_len + 1;
info->raw_tracepoint.cookie = raw_tp_link->cookie;
if (!ubuf)
return 0;
@ -3794,6 +3810,32 @@ static int bpf_perf_link_fill_kprobe(const struct perf_event *event,
info->perf_event.kprobe.cookie = event->bpf_cookie;
return 0;
}
static void bpf_perf_link_fdinfo_kprobe(const struct perf_event *event,
struct seq_file *seq)
{
const char *name;
int err;
u32 prog_id, type;
u64 offset, addr;
unsigned long missed;
err = bpf_get_perf_event_info(event, &prog_id, &type, &name,
&offset, &addr, &missed);
if (err)
return;
seq_printf(seq,
"name:\t%s\n"
"offset:\t%#llx\n"
"missed:\t%lu\n"
"addr:\t%#llx\n"
"event_type:\t%s\n"
"cookie:\t%llu\n",
name, offset, missed, addr,
type == BPF_FD_TYPE_KRETPROBE ? "kretprobe" : "kprobe",
event->bpf_cookie);
}
#endif
#ifdef CONFIG_UPROBE_EVENTS
@ -3822,6 +3864,31 @@ static int bpf_perf_link_fill_uprobe(const struct perf_event *event,
info->perf_event.uprobe.ref_ctr_offset = ref_ctr_offset;
return 0;
}
static void bpf_perf_link_fdinfo_uprobe(const struct perf_event *event,
struct seq_file *seq)
{
const char *name;
int err;
u32 prog_id, type;
u64 offset, ref_ctr_offset;
unsigned long missed;
err = bpf_get_perf_event_info(event, &prog_id, &type, &name,
&offset, &ref_ctr_offset, &missed);
if (err)
return;
seq_printf(seq,
"name:\t%s\n"
"offset:\t%#llx\n"
"ref_ctr_offset:\t%#llx\n"
"event_type:\t%s\n"
"cookie:\t%llu\n",
name, offset, ref_ctr_offset,
type == BPF_FD_TYPE_URETPROBE ? "uretprobe" : "uprobe",
event->bpf_cookie);
}
#endif
static int bpf_perf_link_fill_probe(const struct perf_event *event,
@ -3890,10 +3957,79 @@ static int bpf_perf_link_fill_link_info(const struct bpf_link *link,
}
}
static void bpf_perf_event_link_show_fdinfo(const struct perf_event *event,
struct seq_file *seq)
{
seq_printf(seq,
"type:\t%u\n"
"config:\t%llu\n"
"event_type:\t%s\n"
"cookie:\t%llu\n",
event->attr.type, event->attr.config,
"event", event->bpf_cookie);
}
static void bpf_tracepoint_link_show_fdinfo(const struct perf_event *event,
struct seq_file *seq)
{
int err;
const char *name;
u32 prog_id;
err = bpf_get_perf_event_info(event, &prog_id, NULL, &name, NULL,
NULL, NULL);
if (err)
return;
seq_printf(seq,
"tp_name:\t%s\n"
"event_type:\t%s\n"
"cookie:\t%llu\n",
name, "tracepoint", event->bpf_cookie);
}
static void bpf_probe_link_show_fdinfo(const struct perf_event *event,
struct seq_file *seq)
{
#ifdef CONFIG_KPROBE_EVENTS
if (event->tp_event->flags & TRACE_EVENT_FL_KPROBE)
return bpf_perf_link_fdinfo_kprobe(event, seq);
#endif
#ifdef CONFIG_UPROBE_EVENTS
if (event->tp_event->flags & TRACE_EVENT_FL_UPROBE)
return bpf_perf_link_fdinfo_uprobe(event, seq);
#endif
}
static void bpf_perf_link_show_fdinfo(const struct bpf_link *link,
struct seq_file *seq)
{
struct bpf_perf_link *perf_link;
const struct perf_event *event;
perf_link = container_of(link, struct bpf_perf_link, link);
event = perf_get_event(perf_link->perf_file);
if (IS_ERR(event))
return;
switch (event->prog->type) {
case BPF_PROG_TYPE_PERF_EVENT:
return bpf_perf_event_link_show_fdinfo(event, seq);
case BPF_PROG_TYPE_TRACEPOINT:
return bpf_tracepoint_link_show_fdinfo(event, seq);
case BPF_PROG_TYPE_KPROBE:
return bpf_probe_link_show_fdinfo(event, seq);
default:
return;
}
}
static const struct bpf_link_ops bpf_perf_link_lops = {
.release = bpf_perf_link_release,
.dealloc = bpf_perf_link_dealloc,
.fill_link_info = bpf_perf_link_fill_link_info,
.show_fdinfo = bpf_perf_link_show_fdinfo,
};
static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
@ -3916,7 +4052,8 @@ static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *pro
err = -ENOMEM;
goto out_put_file;
}
bpf_link_init(&link->link, BPF_LINK_TYPE_PERF_EVENT, &bpf_perf_link_lops, prog);
bpf_link_init(&link->link, BPF_LINK_TYPE_PERF_EVENT, &bpf_perf_link_lops, prog,
attr->link_create.attach_type);
link->perf_file = perf_file;
err = bpf_link_prime(&link->link, &link_primer);
@ -3948,7 +4085,8 @@ static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *pro
#endif /* CONFIG_PERF_EVENTS */
static int bpf_raw_tp_link_attach(struct bpf_prog *prog,
const char __user *user_tp_name, u64 cookie)
const char __user *user_tp_name, u64 cookie,
enum bpf_attach_type attach_type)
{
struct bpf_link_primer link_primer;
struct bpf_raw_tp_link *link;
@ -3971,7 +4109,7 @@ static int bpf_raw_tp_link_attach(struct bpf_prog *prog,
tp_name = prog->aux->attach_func_name;
break;
}
return bpf_tracing_prog_attach(prog, 0, 0, 0);
return bpf_tracing_prog_attach(prog, 0, 0, 0, attach_type);
case BPF_PROG_TYPE_RAW_TRACEPOINT:
case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
if (strncpy_from_user(buf, user_tp_name, sizeof(buf) - 1) < 0)
@ -3993,7 +4131,7 @@ static int bpf_raw_tp_link_attach(struct bpf_prog *prog,
goto out_put_btp;
}
bpf_link_init_sleepable(&link->link, BPF_LINK_TYPE_RAW_TRACEPOINT,
&bpf_raw_tp_link_lops, prog,
&bpf_raw_tp_link_lops, prog, attach_type,
tracepoint_is_faultable(btp->tp));
link->btp = btp;
link->cookie = cookie;
@ -4035,7 +4173,7 @@ static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
tp_name = u64_to_user_ptr(attr->raw_tracepoint.name);
cookie = attr->raw_tracepoint.cookie;
fd = bpf_raw_tp_link_attach(prog, tp_name, cookie);
fd = bpf_raw_tp_link_attach(prog, tp_name, cookie, prog->expected_attach_type);
if (fd < 0)
bpf_prog_put(prog);
return fd;
@ -4185,6 +4323,25 @@ static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog,
}
}
static bool is_cgroup_prog_type(enum bpf_prog_type ptype, enum bpf_attach_type atype,
bool check_atype)
{
switch (ptype) {
case BPF_PROG_TYPE_CGROUP_DEVICE:
case BPF_PROG_TYPE_CGROUP_SKB:
case BPF_PROG_TYPE_CGROUP_SOCK:
case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
case BPF_PROG_TYPE_CGROUP_SOCKOPT:
case BPF_PROG_TYPE_CGROUP_SYSCTL:
case BPF_PROG_TYPE_SOCK_OPS:
return true;
case BPF_PROG_TYPE_LSM:
return check_atype ? atype == BPF_LSM_CGROUP : true;
default:
return false;
}
}
#define BPF_PROG_ATTACH_LAST_FIELD expected_revision
#define BPF_F_ATTACH_MASK_BASE \
@ -4215,6 +4372,9 @@ static int bpf_prog_attach(const union bpf_attr *attr)
if (bpf_mprog_supported(ptype)) {
if (attr->attach_flags & ~BPF_F_ATTACH_MASK_MPROG)
return -EINVAL;
} else if (is_cgroup_prog_type(ptype, 0, false)) {
if (attr->attach_flags & ~(BPF_F_ATTACH_MASK_BASE | BPF_F_ATTACH_MASK_MPROG))
return -EINVAL;
} else {
if (attr->attach_flags & ~BPF_F_ATTACH_MASK_BASE)
return -EINVAL;
@ -4232,6 +4392,11 @@ static int bpf_prog_attach(const union bpf_attr *attr)
return -EINVAL;
}
if (is_cgroup_prog_type(ptype, prog->expected_attach_type, true)) {
ret = cgroup_bpf_prog_attach(attr, ptype, prog);
goto out;
}
switch (ptype) {
case BPF_PROG_TYPE_SK_SKB:
case BPF_PROG_TYPE_SK_MSG:
@ -4243,20 +4408,6 @@ static int bpf_prog_attach(const union bpf_attr *attr)
case BPF_PROG_TYPE_FLOW_DISSECTOR:
ret = netns_bpf_prog_attach(attr, prog);
break;
case BPF_PROG_TYPE_CGROUP_DEVICE:
case BPF_PROG_TYPE_CGROUP_SKB:
case BPF_PROG_TYPE_CGROUP_SOCK:
case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
case BPF_PROG_TYPE_CGROUP_SOCKOPT:
case BPF_PROG_TYPE_CGROUP_SYSCTL:
case BPF_PROG_TYPE_SOCK_OPS:
case BPF_PROG_TYPE_LSM:
if (ptype == BPF_PROG_TYPE_LSM &&
prog->expected_attach_type != BPF_LSM_CGROUP)
ret = -EINVAL;
else
ret = cgroup_bpf_prog_attach(attr, ptype, prog);
break;
case BPF_PROG_TYPE_SCHED_CLS:
if (attr->attach_type == BPF_TCX_INGRESS ||
attr->attach_type == BPF_TCX_EGRESS)
@ -4267,7 +4418,7 @@ static int bpf_prog_attach(const union bpf_attr *attr)
default:
ret = -EINVAL;
}
out:
if (ret)
bpf_prog_put(prog);
return ret;
@ -4295,6 +4446,9 @@ static int bpf_prog_detach(const union bpf_attr *attr)
if (IS_ERR(prog))
return PTR_ERR(prog);
}
} else if (is_cgroup_prog_type(ptype, 0, false)) {
if (attr->attach_flags || attr->relative_fd)
return -EINVAL;
} else if (attr->attach_flags ||
attr->relative_fd ||
attr->expected_revision) {
@ -5085,6 +5239,21 @@ static int bpf_link_get_info_by_fd(struct file *file,
}
static int token_get_info_by_fd(struct file *file,
struct bpf_token *token,
const union bpf_attr *attr,
union bpf_attr __user *uattr)
{
struct bpf_token_info __user *uinfo = u64_to_user_ptr(attr->info.info);
u32 info_len = attr->info.info_len;
int err;
err = bpf_check_uarg_tail_zero(USER_BPFPTR(uinfo), sizeof(*uinfo), info_len);
if (err)
return err;
return bpf_token_get_info_by_fd(token, attr, uattr);
}
#define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info
static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
@ -5108,6 +5277,9 @@ static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
else if (fd_file(f)->f_op == &bpf_link_fops || fd_file(f)->f_op == &bpf_link_fops_poll)
return bpf_link_get_info_by_fd(fd_file(f), fd_file(f)->private_data,
attr, uattr);
else if (fd_file(f)->f_op == &bpf_token_fops)
return token_get_info_by_fd(fd_file(f), fd_file(f)->private_data,
attr, uattr);
return -EINVAL;
}
@ -5195,21 +5367,10 @@ static int bpf_task_fd_query_copy(const union bpf_attr *attr,
if (put_user(zero, ubuf))
return -EFAULT;
} else if (input_len >= len + 1) {
/* ubuf can hold the string with NULL terminator */
if (copy_to_user(ubuf, buf, len + 1))
return -EFAULT;
} else {
/* ubuf cannot hold the string with NULL terminator,
* do a partial copy with NULL terminator.
*/
char zero = '\0';
err = -ENOSPC;
if (copy_to_user(ubuf, buf, input_len - 1))
return -EFAULT;
if (put_user(zero, ubuf + input_len - 1))
return -EFAULT;
err = bpf_copy_to_user(ubuf, buf, input_len, len);
if (err == -EFAULT)
return err;
}
}
@ -5387,7 +5548,8 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr)
ret = bpf_tracing_prog_attach(prog,
attr->link_create.target_fd,
attr->link_create.target_btf_id,
attr->link_create.tracing.cookie);
attr->link_create.tracing.cookie,
attr->link_create.attach_type);
break;
case BPF_PROG_TYPE_LSM:
case BPF_PROG_TYPE_TRACING:
@ -5396,7 +5558,8 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr)
goto out;
}
if (prog->expected_attach_type == BPF_TRACE_RAW_TP)
ret = bpf_raw_tp_link_attach(prog, NULL, attr->link_create.tracing.cookie);
ret = bpf_raw_tp_link_attach(prog, NULL, attr->link_create.tracing.cookie,
attr->link_create.attach_type);
else if (prog->expected_attach_type == BPF_TRACE_ITER)
ret = bpf_iter_link_attach(attr, uattr, prog);
else if (prog->expected_attach_type == BPF_LSM_CGROUP)
@ -5405,7 +5568,8 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr)
ret = bpf_tracing_prog_attach(prog,
attr->link_create.target_fd,
attr->link_create.target_btf_id,
attr->link_create.tracing.cookie);
attr->link_create.tracing.cookie,
attr->link_create.attach_type);
break;
case BPF_PROG_TYPE_FLOW_DISSECTOR:
case BPF_PROG_TYPE_SK_LOOKUP:
@ -5794,6 +5958,28 @@ static int token_create(union bpf_attr *attr)
return bpf_token_create(attr);
}
#define BPF_PROG_STREAM_READ_BY_FD_LAST_FIELD prog_stream_read.prog_fd
static int prog_stream_read(union bpf_attr *attr)
{
char __user *buf = u64_to_user_ptr(attr->prog_stream_read.stream_buf);
u32 len = attr->prog_stream_read.stream_buf_len;
struct bpf_prog *prog;
int ret;
if (CHECK_ATTR(BPF_PROG_STREAM_READ_BY_FD))
return -EINVAL;
prog = bpf_prog_get(attr->prog_stream_read.prog_fd);
if (IS_ERR(prog))
return PTR_ERR(prog);
ret = bpf_prog_stream_read(prog, attr->prog_stream_read.stream_id, buf, len);
bpf_prog_put(prog);
return ret;
}
static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size)
{
union bpf_attr attr;
@ -5930,6 +6116,9 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size)
case BPF_TOKEN_CREATE:
err = token_create(&attr);
break;
case BPF_PROG_STREAM_READ_BY_FD:
err = prog_stream_read(&attr);
break;
default:
err = -EINVAL;
break;

16
kernel/bpf/tcx.c
View file

@ -142,7 +142,7 @@ static int tcx_link_prog_attach(struct bpf_link *link, u32 flags, u32 id_or_fd,
u64 revision)
{
struct tcx_link *tcx = tcx_link(link);
bool created, ingress = tcx->location == BPF_TCX_INGRESS;
bool created, ingress = link->attach_type == BPF_TCX_INGRESS;
struct bpf_mprog_entry *entry, *entry_new;
struct net_device *dev = tcx->dev;
int ret;
@ -169,7 +169,7 @@ static int tcx_link_prog_attach(struct bpf_link *link, u32 flags, u32 id_or_fd,
static void tcx_link_release(struct bpf_link *link)
{
struct tcx_link *tcx = tcx_link(link);
bool ingress = tcx->location == BPF_TCX_INGRESS;
bool ingress = link->attach_type == BPF_TCX_INGRESS;
struct bpf_mprog_entry *entry, *entry_new;
struct net_device *dev;
int ret = 0;
@ -204,7 +204,7 @@ static int tcx_link_update(struct bpf_link *link, struct bpf_prog *nprog,
struct bpf_prog *oprog)
{
struct tcx_link *tcx = tcx_link(link);
bool ingress = tcx->location == BPF_TCX_INGRESS;
bool ingress = link->attach_type == BPF_TCX_INGRESS;
struct bpf_mprog_entry *entry, *entry_new;
struct net_device *dev;
int ret = 0;
@ -260,8 +260,8 @@ static void tcx_link_fdinfo(const struct bpf_link *link, struct seq_file *seq)
seq_printf(seq, "ifindex:\t%u\n", ifindex);
seq_printf(seq, "attach_type:\t%u (%s)\n",
tcx->location,
tcx->location == BPF_TCX_INGRESS ? "ingress" : "egress");
link->attach_type,
link->attach_type == BPF_TCX_INGRESS ? "ingress" : "egress");
}
static int tcx_link_fill_info(const struct bpf_link *link,
@ -276,7 +276,7 @@ static int tcx_link_fill_info(const struct bpf_link *link,
rtnl_unlock();
info->tcx.ifindex = ifindex;
info->tcx.attach_type = tcx->location;
info->tcx.attach_type = link->attach_type;
return 0;
}
@ -301,8 +301,8 @@ static int tcx_link_init(struct tcx_link *tcx,
struct net_device *dev,
struct bpf_prog *prog)
{
bpf_link_init(&tcx->link, BPF_LINK_TYPE_TCX, &tcx_link_lops, prog);
tcx->location = attr->link_create.attach_type;
bpf_link_init(&tcx->link, BPF_LINK_TYPE_TCX, &tcx_link_lops, prog,
attr->link_create.attach_type);
tcx->dev = dev;
return bpf_link_prime(&tcx->link, link_primer);
}

5
kernel/bpf/tnum.c
View file

@ -83,6 +83,11 @@ struct tnum tnum_sub(struct tnum a, struct tnum b)
return TNUM(dv & ~mu, mu);
}
struct tnum tnum_neg(struct tnum a)
{
return tnum_sub(TNUM(0, 0), a);
}
struct tnum tnum_and(struct tnum a, struct tnum b)
{
u64 alpha, beta, v;

25
kernel/bpf/token.c
View file

@ -103,7 +103,7 @@ static void bpf_token_show_fdinfo(struct seq_file *m, struct file *filp)
static const struct inode_operations bpf_token_iops = { };
static const struct file_operations bpf_token_fops = {
const struct file_operations bpf_token_fops = {
.release = bpf_token_release,
.show_fdinfo = bpf_token_show_fdinfo,
};
@ -210,6 +210,29 @@ out_file:
return err;
}
int bpf_token_get_info_by_fd(struct bpf_token *token,
const union bpf_attr *attr,
union bpf_attr __user *uattr)
{
struct bpf_token_info __user *uinfo = u64_to_user_ptr(attr->info.info);
struct bpf_token_info info;
u32 info_len = attr->info.info_len;
info_len = min_t(u32, info_len, sizeof(info));
memset(&info, 0, sizeof(info));
info.allowed_cmds = token->allowed_cmds;
info.allowed_maps = token->allowed_maps;
info.allowed_progs = token->allowed_progs;
info.allowed_attachs = token->allowed_attachs;
if (copy_to_user(uinfo, &info, info_len) ||
put_user(info_len, &uattr->info.info_len))
return -EFAULT;
return 0;
}
struct bpf_token *bpf_token_get_from_fd(u32 ufd)
{
CLASS(fd, f)(ufd);

49
kernel/bpf/trampoline.c
View file

@ -674,7 +674,8 @@ static const struct bpf_link_ops bpf_shim_tramp_link_lops = {
static struct bpf_shim_tramp_link *cgroup_shim_alloc(const struct bpf_prog *prog,
bpf_func_t bpf_func,
int cgroup_atype)
int cgroup_atype,
enum bpf_attach_type attach_type)
{
struct bpf_shim_tramp_link *shim_link = NULL;
struct bpf_prog *p;
@ -701,7 +702,7 @@ static struct bpf_shim_tramp_link *cgroup_shim_alloc(const struct bpf_prog *prog
p->expected_attach_type = BPF_LSM_MAC;
bpf_prog_inc(p);
bpf_link_init(&shim_link->link.link, BPF_LINK_TYPE_UNSPEC,
&bpf_shim_tramp_link_lops, p);
&bpf_shim_tramp_link_lops, p, attach_type);
bpf_cgroup_atype_get(p->aux->attach_btf_id, cgroup_atype);
return shim_link;
@ -726,7 +727,8 @@ static struct bpf_shim_tramp_link *cgroup_shim_find(struct bpf_trampoline *tr,
}
int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
int cgroup_atype)
int cgroup_atype,
enum bpf_attach_type attach_type)
{
struct bpf_shim_tramp_link *shim_link = NULL;
struct bpf_attach_target_info tgt_info = {};
@ -763,7 +765,7 @@ int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
/* Allocate and install new shim. */
shim_link = cgroup_shim_alloc(prog, bpf_func, cgroup_atype);
shim_link = cgroup_shim_alloc(prog, bpf_func, cgroup_atype, attach_type);
if (!shim_link) {
err = -ENOMEM;
goto err;
@ -911,27 +913,32 @@ static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tram
return bpf_prog_start_time();
}
static void notrace update_prog_stats(struct bpf_prog *prog,
u64 start)
static void notrace __update_prog_stats(struct bpf_prog *prog, u64 start)
{
struct bpf_prog_stats *stats;
unsigned long flags;
u64 duration;
if (static_branch_unlikely(&bpf_stats_enabled_key) &&
/* static_key could be enabled in __bpf_prog_enter*
* and disabled in __bpf_prog_exit*.
* And vice versa.
* Hence check that 'start' is valid.
*/
start > NO_START_TIME) {
u64 duration = sched_clock() - start;
unsigned long flags;
/*
* static_key could be enabled in __bpf_prog_enter* and disabled in
* __bpf_prog_exit*. And vice versa. Check that 'start' is valid.
*/
if (start <= NO_START_TIME)
return;
stats = this_cpu_ptr(prog->stats);
flags = u64_stats_update_begin_irqsave(&stats->syncp);
u64_stats_inc(&stats->cnt);
u64_stats_add(&stats->nsecs, duration);
u64_stats_update_end_irqrestore(&stats->syncp, flags);
}
duration = sched_clock() - start;
stats = this_cpu_ptr(prog->stats);
flags = u64_stats_update_begin_irqsave(&stats->syncp);
u64_stats_inc(&stats->cnt);
u64_stats_add(&stats->nsecs, duration);
u64_stats_update_end_irqrestore(&stats->syncp, flags);
}
static __always_inline void notrace update_prog_stats(struct bpf_prog *prog,
u64 start)
{
if (static_branch_unlikely(&bpf_stats_enabled_key))
__update_prog_stats(prog, start);
}
static void notrace __bpf_prog_exit_recur(struct bpf_prog *prog, u64 start,

2357
kernel/bpf/verifier.c
View file

File diff suppressed because it is too large Load diff

5
kernel/cgroup/cgroup.c
View file

@ -2074,6 +2074,11 @@ static void init_cgroup_housekeeping(struct cgroup *cgrp)
for_each_subsys(ss, ssid)
INIT_LIST_HEAD(&cgrp->e_csets[ssid]);
#ifdef CONFIG_CGROUP_BPF
for (int i = 0; i < ARRAY_SIZE(cgrp->bpf.revisions); i++)
cgrp->bpf.revisions[i] = 1;
#endif
init_waitqueue_head(&cgrp->offline_waitq);
INIT_WORK(&cgrp->release_agent_work, cgroup1_release_agent);
}

3
kernel/kallsyms.c
View file

@ -829,8 +829,7 @@ static struct bpf_iter_reg ksym_iter_reg_info = {
.seq_info = &ksym_iter_seq_info,
};
BTF_ID_LIST(btf_ksym_iter_id)
BTF_ID(struct, kallsym_iter)
BTF_ID_LIST_SINGLE(btf_ksym_iter_id, struct, kallsym_iter)
static int __init bpf_ksym_iter_register(void)
{

90
kernel/trace/bpf_trace.c
View file

@ -781,8 +781,7 @@ BPF_CALL_1(bpf_task_pt_regs, struct task_struct *, task)
return (unsigned long) task_pt_regs(task);
}
BTF_ID_LIST(bpf_task_pt_regs_ids)
BTF_ID(struct, pt_regs)
BTF_ID_LIST_SINGLE(bpf_task_pt_regs_ids, struct, pt_regs)
const struct bpf_func_proto bpf_task_pt_regs_proto = {
.func = bpf_task_pt_regs,
@ -1270,7 +1269,7 @@ __bpf_kfunc_start_defs();
* Return: a bpf_key pointer with a valid key pointer if the key is found, a
* NULL pointer otherwise.
*/
__bpf_kfunc struct bpf_key *bpf_lookup_user_key(u32 serial, u64 flags)
__bpf_kfunc struct bpf_key *bpf_lookup_user_key(s32 serial, u64 flags)
{
key_ref_t key_ref;
struct bpf_key *bkey;
@ -2466,7 +2465,6 @@ struct bpf_kprobe_multi_link {
u32 cnt;
u32 mods_cnt;
struct module **mods;
u32 flags;
};
struct bpf_kprobe_multi_run_ctx {
@ -2586,7 +2584,7 @@ static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link,
kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);
info->kprobe_multi.count = kmulti_link->cnt;
info->kprobe_multi.flags = kmulti_link->flags;
info->kprobe_multi.flags = kmulti_link->link.flags;
info->kprobe_multi.missed = kmulti_link->fp.nmissed;
if (!uaddrs)
@ -2620,10 +2618,37 @@ static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link,
return err;
}
#ifdef CONFIG_PROC_FS
static void bpf_kprobe_multi_show_fdinfo(const struct bpf_link *link,
struct seq_file *seq)
{
struct bpf_kprobe_multi_link *kmulti_link;
kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);
seq_printf(seq,
"kprobe_cnt:\t%u\n"
"missed:\t%lu\n",
kmulti_link->cnt,
kmulti_link->fp.nmissed);
seq_printf(seq, "%s\t %s\n", "cookie", "func");
for (int i = 0; i < kmulti_link->cnt; i++) {
seq_printf(seq,
"%llu\t %pS\n",
kmulti_link->cookies[i],
(void *)kmulti_link->addrs[i]);
}
}
#endif
static const struct bpf_link_ops bpf_kprobe_multi_link_lops = {
.release = bpf_kprobe_multi_link_release,
.dealloc_deferred = bpf_kprobe_multi_link_dealloc,
.fill_link_info = bpf_kprobe_multi_link_fill_link_info,
#ifdef CONFIG_PROC_FS
.show_fdinfo = bpf_kprobe_multi_show_fdinfo,
#endif
};
static void bpf_kprobe_multi_cookie_swap(void *a, void *b, int size, const void *priv)
@ -2960,7 +2985,7 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr
}
bpf_link_init(&link->link, BPF_LINK_TYPE_KPROBE_MULTI,
&bpf_kprobe_multi_link_lops, prog);
&bpf_kprobe_multi_link_lops, prog, attr->link_create.attach_type);
err = bpf_link_prime(&link->link, &link_primer);
if (err)
@ -2976,7 +3001,7 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr
link->addrs = addrs;
link->cookies = cookies;
link->cnt = cnt;
link->flags = flags;
link->link.flags = flags;
if (cookies) {
/*
@ -3045,7 +3070,6 @@ struct bpf_uprobe_multi_link {
struct path path;
struct bpf_link link;
u32 cnt;
u32 flags;
struct bpf_uprobe *uprobes;
struct task_struct *task;
};
@ -3109,7 +3133,7 @@ static int bpf_uprobe_multi_link_fill_link_info(const struct bpf_link *link,
umulti_link = container_of(link, struct bpf_uprobe_multi_link, link);
info->uprobe_multi.count = umulti_link->cnt;
info->uprobe_multi.flags = umulti_link->flags;
info->uprobe_multi.flags = umulti_link->link.flags;
info->uprobe_multi.pid = umulti_link->task ?
task_pid_nr_ns(umulti_link->task, task_active_pid_ns(current)) : 0;
@ -3154,10 +3178,54 @@ static int bpf_uprobe_multi_link_fill_link_info(const struct bpf_link *link,
return err;
}
#ifdef CONFIG_PROC_FS
static void bpf_uprobe_multi_show_fdinfo(const struct bpf_link *link,
struct seq_file *seq)
{
struct bpf_uprobe_multi_link *umulti_link;
char *p, *buf;
pid_t pid;
umulti_link = container_of(link, struct bpf_uprobe_multi_link, link);
buf = kmalloc(PATH_MAX, GFP_KERNEL);
if (!buf)
return;
p = d_path(&umulti_link->path, buf, PATH_MAX);
if (IS_ERR(p)) {
kfree(buf);
return;
}
pid = umulti_link->task ?
task_pid_nr_ns(umulti_link->task, task_active_pid_ns(current)) : 0;
seq_printf(seq,
"uprobe_cnt:\t%u\n"
"pid:\t%u\n"
"path:\t%s\n",
umulti_link->cnt, pid, p);
seq_printf(seq, "%s\t %s\t %s\n", "cookie", "offset", "ref_ctr_offset");
for (int i = 0; i < umulti_link->cnt; i++) {
seq_printf(seq,
"%llu\t %#llx\t %#lx\n",
umulti_link->uprobes[i].cookie,
umulti_link->uprobes[i].offset,
umulti_link->uprobes[i].ref_ctr_offset);
}
kfree(buf);
}
#endif
static const struct bpf_link_ops bpf_uprobe_multi_link_lops = {
.release = bpf_uprobe_multi_link_release,
.dealloc_deferred = bpf_uprobe_multi_link_dealloc,
.fill_link_info = bpf_uprobe_multi_link_fill_link_info,
#ifdef CONFIG_PROC_FS
.show_fdinfo = bpf_uprobe_multi_show_fdinfo,
#endif
};
static int uprobe_prog_run(struct bpf_uprobe *uprobe,
@ -3369,10 +3437,10 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr
link->uprobes = uprobes;
link->path = path;
link->task = task;
link->flags = flags;
link->link.flags = flags;
bpf_link_init(&link->link, BPF_LINK_TYPE_UPROBE_MULTI,
&bpf_uprobe_multi_link_lops, prog);
&bpf_uprobe_multi_link_lops, prog, attr->link_create.attach_type);
for (i = 0; i < cnt; i++) {
uprobes[i].uprobe = uprobe_register(d_real_inode(link->path.dentry),

191
kernel/usermode_driver.c
View file

@ -1,191 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* umd - User mode driver support
*/
#include <linux/shmem_fs.h>
#include <linux/pipe_fs_i.h>
#include <linux/mount.h>
#include <linux/fs_struct.h>
#include <linux/task_work.h>
#include <linux/usermode_driver.h>
static struct vfsmount *blob_to_mnt(const void *data, size_t len, const char *name)
{
struct file_system_type *type;
struct vfsmount *mnt;
struct file *file;
ssize_t written;
loff_t pos = 0;
type = get_fs_type("tmpfs");
if (!type)
return ERR_PTR(-ENODEV);
mnt = kern_mount(type);
put_filesystem(type);
if (IS_ERR(mnt))
return mnt;
file = file_open_root_mnt(mnt, name, O_CREAT | O_WRONLY, 0700);
if (IS_ERR(file)) {
kern_unmount(mnt);
return ERR_CAST(file);
}
written = kernel_write(file, data, len, &pos);
if (written != len) {
int err = written;
if (err >= 0)
err = -ENOMEM;
filp_close(file, NULL);
kern_unmount(mnt);
return ERR_PTR(err);
}
fput(file);
/* Flush delayed fput so exec can open the file read-only */
flush_delayed_fput();
task_work_run();
return mnt;
}
/**
* umd_load_blob - Remember a blob of bytes for fork_usermode_driver
* @info: information about usermode driver
* @data: a blob of bytes that can be executed as a file
* @len: The lentgh of the blob
*
*/
int umd_load_blob(struct umd_info *info, const void *data, size_t len)
{
struct vfsmount *mnt;
if (WARN_ON_ONCE(info->wd.dentry || info->wd.mnt))
return -EBUSY;
mnt = blob_to_mnt(data, len, info->driver_name);
if (IS_ERR(mnt))
return PTR_ERR(mnt);
info->wd.mnt = mnt;
info->wd.dentry = mnt->mnt_root;
return 0;
}
EXPORT_SYMBOL_GPL(umd_load_blob);
/**
* umd_unload_blob - Disassociate @info from a previously loaded blob
* @info: information about usermode driver
*
*/
int umd_unload_blob(struct umd_info *info)
{
if (WARN_ON_ONCE(!info->wd.mnt ||
!info->wd.dentry ||
info->wd.mnt->mnt_root != info->wd.dentry))
return -EINVAL;
kern_unmount(info->wd.mnt);
info->wd.mnt = NULL;
info->wd.dentry = NULL;
return 0;
}
EXPORT_SYMBOL_GPL(umd_unload_blob);
static int umd_setup(struct subprocess_info *info, struct cred *new)
{
struct umd_info *umd_info = info->data;
struct file *from_umh[2];
struct file *to_umh[2];
int err;
/* create pipe to send data to umh */
err = create_pipe_files(to_umh, 0);
if (err)
return err;
err = replace_fd(0, to_umh[0], 0);
fput(to_umh[0]);
if (err < 0) {
fput(to_umh[1]);
return err;
}
/* create pipe to receive data from umh */
err = create_pipe_files(from_umh, 0);
if (err) {
fput(to_umh[1]);
replace_fd(0, NULL, 0);
return err;
}
err = replace_fd(1, from_umh[1], 0);
fput(from_umh[1]);
if (err < 0) {
fput(to_umh[1]);
replace_fd(0, NULL, 0);
fput(from_umh[0]);
return err;
}
set_fs_pwd(current->fs, &umd_info->wd);
umd_info->pipe_to_umh = to_umh[1];
umd_info->pipe_from_umh = from_umh[0];
umd_info->tgid = get_pid(task_tgid(current));
return 0;
}
static void umd_cleanup(struct subprocess_info *info)
{
struct umd_info *umd_info = info->data;
/* cleanup if umh_setup() was successful but exec failed */
if (info->retval)
umd_cleanup_helper(umd_info);
}
/**
* umd_cleanup_helper - release the resources which were allocated in umd_setup
* @info: information about usermode driver
*/
void umd_cleanup_helper(struct umd_info *info)
{
fput(info->pipe_to_umh);
fput(info->pipe_from_umh);
put_pid(info->tgid);
info->tgid = NULL;
}
EXPORT_SYMBOL_GPL(umd_cleanup_helper);
/**
* fork_usermode_driver - fork a usermode driver
* @info: information about usermode driver (shouldn't be NULL)
*
* Returns either negative error or zero which indicates success in
* executing a usermode driver. In such case 'struct umd_info *info'
* is populated with two pipes and a tgid of the process. The caller is
* responsible for health check of the user process, killing it via
* tgid, and closing the pipes when user process is no longer needed.
*/
int fork_usermode_driver(struct umd_info *info)
{
struct subprocess_info *sub_info;
const char *argv[] = { info->driver_name, NULL };
int err;
if (WARN_ON_ONCE(info->tgid))
return -EBUSY;
err = -ENOMEM;
sub_info = call_usermodehelper_setup(info->driver_name,
(char **)argv, NULL, GFP_KERNEL,
umd_setup, umd_cleanup, info);
if (!sub_info)
goto out;
err = call_usermodehelper_exec(sub_info, UMH_WAIT_EXEC);
out:
return err;
}
EXPORT_SYMBOL_GPL(fork_usermode_driver);

3
net/bpf/bpf_dummy_struct_ops.c
View file

@ -171,7 +171,8 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
}
/* prog doesn't take the ownership of the reference from caller */
bpf_prog_inc(prog);
bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS, &bpf_struct_ops_link_lops, prog);
bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS, &bpf_struct_ops_link_lops, prog,
prog->expected_attach_type);
op_idx = prog->expected_attach_type;
err = bpf_struct_ops_prepare_trampoline(tlinks, link,

2
net/bpf/test_run.c
View file

@ -1255,7 +1255,7 @@ int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
headroom -= ctx->data;
}
max_data_sz = 4096 - headroom - tailroom;
max_data_sz = PAGE_SIZE - headroom - tailroom;
if (size > max_data_sz) {
/* disallow live data mode for jumbo frames */
if (do_live)

3
net/core/dev.c
View file

@ -10453,7 +10453,8 @@ int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
goto unlock;
}
bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog);
bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog,
attr->link_create.attach_type);
link->dev = dev;
link->flags = attr->link_create.flags;

13
net/core/sock_map.c
View file

@ -1709,7 +1709,6 @@ EXPORT_SYMBOL_GPL(sock_map_close);
struct sockmap_link {
struct bpf_link link;
struct bpf_map *map;
enum bpf_attach_type attach_type;
};
static void sock_map_link_release(struct bpf_link *link)
@ -1721,7 +1720,7 @@ static void sock_map_link_release(struct bpf_link *link)
goto out;
WARN_ON_ONCE(sock_map_prog_update(sockmap_link->map, NULL, link->prog, link,
sockmap_link->attach_type));
link->attach_type));
bpf_map_put_with_uref(sockmap_link->map);
sockmap_link->map = NULL;
@ -1772,7 +1771,7 @@ static int sock_map_link_update_prog(struct bpf_link *link,
}
ret = sock_map_prog_link_lookup(sockmap_link->map, &pprog, &plink,
sockmap_link->attach_type);
link->attach_type);
if (ret)
goto out;
@ -1817,7 +1816,7 @@ static int sock_map_link_fill_info(const struct bpf_link *link,
u32 map_id = sock_map_link_get_map_id(sockmap_link);
info->sockmap.map_id = map_id;
info->sockmap.attach_type = sockmap_link->attach_type;
info->sockmap.attach_type = link->attach_type;
return 0;
}
@ -1828,7 +1827,7 @@ static void sock_map_link_show_fdinfo(const struct bpf_link *link,
u32 map_id = sock_map_link_get_map_id(sockmap_link);
seq_printf(seq, "map_id:\t%u\n", map_id);
seq_printf(seq, "attach_type:\t%u\n", sockmap_link->attach_type);
seq_printf(seq, "attach_type:\t%u\n", link->attach_type);
}
static const struct bpf_link_ops sock_map_link_ops = {
@ -1866,9 +1865,9 @@ int sock_map_link_create(const union bpf_attr *attr, struct bpf_prog *prog)
}
attach_type = attr->link_create.attach_type;
bpf_link_init(&sockmap_link->link, BPF_LINK_TYPE_SOCKMAP, &sock_map_link_ops, prog);
bpf_link_init(&sockmap_link->link, BPF_LINK_TYPE_SOCKMAP, &sock_map_link_ops, prog,
attach_type);
sockmap_link->map = map;
sockmap_link->attach_type = attach_type;
ret = bpf_link_prime(&sockmap_link->link, &link_primer);
if (ret) {

3
net/ipv6/route.c
View file

@ -6821,8 +6821,7 @@ void __init ip6_route_init_special_entries(void)
#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
DEFINE_BPF_ITER_FUNC(ipv6_route, struct bpf_iter_meta *meta, struct fib6_info *rt)
BTF_ID_LIST(btf_fib6_info_id)
BTF_ID(struct, fib6_info)
BTF_ID_LIST_SINGLE(btf_fib6_info_id, struct, fib6_info)
static const struct bpf_iter_seq_info ipv6_route_seq_info = {
.seq_ops = &ipv6_route_seq_ops,

3
net/netfilter/nf_bpf_link.c
View file

@ -225,7 +225,8 @@ int bpf_nf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
if (!link)
return -ENOMEM;
bpf_link_init(&link->link, BPF_LINK_TYPE_NETFILTER, &bpf_nf_link_lops, prog);
bpf_link_init(&link->link, BPF_LINK_TYPE_NETFILTER, &bpf_nf_link_lops, prog,
attr->link_create.attach_type);
link->hook_ops.hook = nf_hook_run_bpf;
link->hook_ops.hook_ops_type = NF_HOOK_OP_BPF;

3
net/netlink/af_netlink.c
View file

@ -2887,8 +2887,7 @@ static const struct rhashtable_params netlink_rhashtable_params = {
};
#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
BTF_ID_LIST(btf_netlink_sock_id)
BTF_ID(struct, netlink_sock)
BTF_ID_LIST_SINGLE(btf_netlink_sock_id, struct, netlink_sock)
static const struct bpf_iter_seq_info netlink_seq_info = {
.seq_ops = &netlink_seq_ops,

9
net/sched/bpf_qdisc.c
View file

@ -130,8 +130,7 @@ static int bpf_qdisc_btf_struct_access(struct bpf_verifier_log *log,
return 0;
}
BTF_ID_LIST(bpf_qdisc_init_prologue_ids)
BTF_ID(func, bpf_qdisc_init_prologue)
BTF_ID_LIST_SINGLE(bpf_qdisc_init_prologue_ids, func, bpf_qdisc_init_prologue)
static int bpf_qdisc_gen_prologue(struct bpf_insn *insn_buf, bool direct_write,
const struct bpf_prog *prog)
@ -161,8 +160,7 @@ static int bpf_qdisc_gen_prologue(struct bpf_insn *insn_buf, bool direct_write,
return insn - insn_buf;
}
BTF_ID_LIST(bpf_qdisc_reset_destroy_epilogue_ids)
BTF_ID(func, bpf_qdisc_reset_destroy_epilogue)
BTF_ID_LIST_SINGLE(bpf_qdisc_reset_destroy_epilogue_ids, func, bpf_qdisc_reset_destroy_epilogue)
static int bpf_qdisc_gen_epilogue(struct bpf_insn *insn_buf, const struct bpf_prog *prog,
s16 ctx_stack_off)
@ -451,8 +449,7 @@ static struct bpf_struct_ops bpf_Qdisc_ops = {
.owner = THIS_MODULE,
};
BTF_ID_LIST(bpf_sk_buff_dtor_ids)
BTF_ID(func, bpf_kfree_skb)
BTF_ID_LIST_SINGLE(bpf_sk_buff_dtor_ids, func, bpf_kfree_skb)
static int __init bpf_qdisc_kfunc_init(void)
{

2
tools/bpf/bpf_jit_disasm.c
View file

@ -45,6 +45,8 @@ static void get_exec_path(char *tpath, size_t size)
assert(path);
len = readlink(path, tpath, size);
if (len < 0)
len = 0;
tpath[len] = 0;
free(path);

7
tools/bpf/bpftool/Documentation/bpftool-prog.rst
View file

@ -35,6 +35,7 @@ PROG COMMANDS
| **bpftool** **prog attach** *PROG* *ATTACH_TYPE* [*MAP*]
| **bpftool** **prog detach** *PROG* *ATTACH_TYPE* [*MAP*]
| **bpftool** **prog tracelog**
| **bpftool** **prog tracelog** [ { **stdout** | **stderr** } *PROG* ]
| **bpftool** **prog run** *PROG* **data_in** *FILE* [**data_out** *FILE* [**data_size_out** *L*]] [**ctx_in** *FILE* [**ctx_out** *FILE* [**ctx_size_out** *M*]]] [**repeat** *N*]
| **bpftool** **prog profile** *PROG* [**duration** *DURATION*] *METRICs*
| **bpftool** **prog help**
@ -179,6 +180,12 @@ bpftool prog tracelog
purposes. For streaming data from BPF programs to user space, one can use
perf events (see also **bpftool-map**\ (8)).
bpftool prog tracelog { stdout | stderr } *PROG*
Dump the BPF stream of the program. BPF programs can write to these streams
at runtime with the **bpf_stream_vprintk**\ () kfunc. The kernel may write
error messages to the standard error stream. This facility should be used
only for debugging purposes.
bpftool prog run *PROG* data_in *FILE* [data_out *FILE* [data_size_out *L*]] [ctx_in *FILE* [ctx_out *FILE* [ctx_size_out *M*]]] [repeat *N*]
Run BPF program *PROG* in the kernel testing infrastructure for BPF,
meaning that the program works on the data and context provided by the

16
tools/bpf/bpftool/bash-completion/bpftool
View file

@ -518,7 +518,21 @@ _bpftool()
esac
;;
tracelog)
return 0
case $prev in
$command)
COMPREPLY+=( $( compgen -W "stdout stderr" -- \
"$cur" ) )
return 0
;;
stdout|stderr)
COMPREPLY=( $( compgen -W "$PROG_TYPE" -- \
"$cur" ) )
return 0
;;
*)
return 0
;;
esac
;;
profile)
case $cword in

8
tools/bpf/bpftool/btf.c
View file

@ -905,7 +905,8 @@ static int do_dump(int argc, char **argv)
return -1;
}
fd = map_parse_fd_and_info(&argc, &argv, &info, &len);
fd = map_parse_fd_and_info(&argc, &argv, &info, &len,
BPF_F_RDONLY);
if (fd < 0)
return -1;
@ -1118,10 +1119,13 @@ build_btf_type_table(struct hashmap *tab, enum bpf_obj_type type,
[BPF_OBJ_PROG] = "prog",
[BPF_OBJ_MAP] = "map",
};
LIBBPF_OPTS(bpf_get_fd_by_id_opts, opts_ro);
__u32 btf_id, id = 0;
int err;
int fd;
opts_ro.open_flags = BPF_F_RDONLY;
while (true) {
switch (type) {
case BPF_OBJ_PROG:
@ -1151,7 +1155,7 @@ build_btf_type_table(struct hashmap *tab, enum bpf_obj_type type,
fd = bpf_prog_get_fd_by_id(id);
break;
case BPF_OBJ_MAP:
fd = bpf_map_get_fd_by_id(id);
fd = bpf_map_get_fd_by_id_opts(id, &opts_ro);
break;
default:
err = -1;

59
tools/bpf/bpftool/common.c
View file

@ -4,6 +4,7 @@
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include <assert.h>
#include <ctype.h>
#include <errno.h>
#include <fcntl.h>
@ -193,7 +194,8 @@ int mount_tracefs(const char *target)
return err;
}
int open_obj_pinned(const char *path, bool quiet)
int open_obj_pinned(const char *path, bool quiet,
const struct bpf_obj_get_opts *opts)
{
char *pname;
int fd = -1;
@ -205,7 +207,7 @@ int open_obj_pinned(const char *path, bool quiet)
goto out_ret;
}
fd = bpf_obj_get(pname);
fd = bpf_obj_get_opts(pname, opts);
if (fd < 0) {
if (!quiet)
p_err("bpf obj get (%s): %s", pname,
@ -221,12 +223,13 @@ out_ret:
return fd;
}
int open_obj_pinned_any(const char *path, enum bpf_obj_type exp_type)
int open_obj_pinned_any(const char *path, enum bpf_obj_type exp_type,
const struct bpf_obj_get_opts *opts)
{
enum bpf_obj_type type;
int fd;
fd = open_obj_pinned(path, false);
fd = open_obj_pinned(path, false, opts);
if (fd < 0)
return -1;
@ -555,7 +558,7 @@ static int do_build_table_cb(const char *fpath, const struct stat *sb,
if (typeflag != FTW_F)
goto out_ret;
fd = open_obj_pinned(fpath, true);
fd = open_obj_pinned(fpath, true, NULL);
if (fd < 0)
goto out_ret;
@ -928,7 +931,7 @@ int prog_parse_fds(int *argc, char ***argv, int **fds)
path = **argv;
NEXT_ARGP();
(*fds)[0] = open_obj_pinned_any(path, BPF_OBJ_PROG);
(*fds)[0] = open_obj_pinned_any(path, BPF_OBJ_PROG, NULL);
if ((*fds)[0] < 0)
return -1;
return 1;
@ -965,7 +968,8 @@ exit_free:
return fd;
}
static int map_fd_by_name(char *name, int **fds)
static int map_fd_by_name(char *name, int **fds,
const struct bpf_get_fd_by_id_opts *opts)
{
unsigned int id = 0;
int fd, nb_fds = 0;
@ -973,6 +977,7 @@ static int map_fd_by_name(char *name, int **fds)
int err;
while (true) {
LIBBPF_OPTS(bpf_get_fd_by_id_opts, opts_ro);
struct bpf_map_info info = {};
__u32 len = sizeof(info);
@ -985,7 +990,9 @@ static int map_fd_by_name(char *name, int **fds)
return nb_fds;
}
fd = bpf_map_get_fd_by_id(id);
/* Request a read-only fd to query the map info */
opts_ro.open_flags = BPF_F_RDONLY;
fd = bpf_map_get_fd_by_id_opts(id, &opts_ro);
if (fd < 0) {
p_err("can't get map by id (%u): %s",
id, strerror(errno));
@ -1004,6 +1011,19 @@ static int map_fd_by_name(char *name, int **fds)
continue;
}
/* Get an fd with the requested options, if they differ
* from the read-only options used to get the fd above.
*/
if (memcmp(opts, &opts_ro, sizeof(opts_ro))) {
close(fd);
fd = bpf_map_get_fd_by_id_opts(id, opts);
if (fd < 0) {
p_err("can't get map by id (%u): %s", id,
strerror(errno));
goto err_close_fds;
}
}
if (nb_fds > 0) {
tmp = realloc(*fds, (nb_fds + 1) * sizeof(int));
if (!tmp) {
@ -1023,8 +1043,13 @@ err_close_fds:
return -1;
}
int map_parse_fds(int *argc, char ***argv, int **fds)
int map_parse_fds(int *argc, char ***argv, int **fds, __u32 open_flags)
{
LIBBPF_OPTS(bpf_get_fd_by_id_opts, opts);
assert((open_flags & ~BPF_F_RDONLY) == 0);
opts.open_flags = open_flags;
if (is_prefix(**argv, "id")) {
unsigned int id;
char *endptr;
@ -1038,7 +1063,7 @@ int map_parse_fds(int *argc, char ***argv, int **fds)
}
NEXT_ARGP();
(*fds)[0] = bpf_map_get_fd_by_id(id);
(*fds)[0] = bpf_map_get_fd_by_id_opts(id, &opts);
if ((*fds)[0] < 0) {
p_err("get map by id (%u): %s", id, strerror(errno));
return -1;
@ -1056,16 +1081,18 @@ int map_parse_fds(int *argc, char ***argv, int **fds)
}
NEXT_ARGP();
return map_fd_by_name(name, fds);
return map_fd_by_name(name, fds, &opts);
} else if (is_prefix(**argv, "pinned")) {
char *path;
LIBBPF_OPTS(bpf_obj_get_opts, get_opts);
get_opts.file_flags = open_flags;
NEXT_ARGP();
path = **argv;
NEXT_ARGP();
(*fds)[0] = open_obj_pinned_any(path, BPF_OBJ_MAP);
(*fds)[0] = open_obj_pinned_any(path, BPF_OBJ_MAP, &get_opts);
if ((*fds)[0] < 0)
return -1;
return 1;
@ -1075,7 +1102,7 @@ int map_parse_fds(int *argc, char ***argv, int **fds)
return -1;
}
int map_parse_fd(int *argc, char ***argv)
int map_parse_fd(int *argc, char ***argv, __u32 open_flags)
{
int *fds = NULL;
int nb_fds, fd;
@ -1085,7 +1112,7 @@ int map_parse_fd(int *argc, char ***argv)
p_err("mem alloc failed");
return -1;
}
nb_fds = map_parse_fds(argc, argv, &fds);
nb_fds = map_parse_fds(argc, argv, &fds, open_flags);
if (nb_fds != 1) {
if (nb_fds > 1) {
p_err("several maps match this handle");
@ -1103,12 +1130,12 @@ exit_free:
}
int map_parse_fd_and_info(int *argc, char ***argv, struct bpf_map_info *info,
__u32 *info_len)
__u32 *info_len, __u32 open_flags)
{
int err;
int fd;
fd = map_parse_fd(argc, argv);
fd = map_parse_fd(argc, argv, open_flags);
if (fd < 0)
return -1;

2
tools/bpf/bpftool/iter.c
View file

@ -37,7 +37,7 @@ static int do_pin(int argc, char **argv)
return -1;
}
map_fd = map_parse_fd(&argc, &argv);
map_fd = map_parse_fd(&argc, &argv, BPF_F_RDONLY);
if (map_fd < 0)
return -1;

8
tools/bpf/bpftool/link.c
View file

@ -117,7 +117,7 @@ static int link_parse_fd(int *argc, char ***argv)
path = **argv;
NEXT_ARGP();
return open_obj_pinned_any(path, BPF_OBJ_LINK);
return open_obj_pinned_any(path, BPF_OBJ_LINK, NULL);
}
p_err("expected 'id' or 'pinned', got: '%s'?", **argv);
@ -485,6 +485,7 @@ static int show_link_close_json(int fd, struct bpf_link_info *info)
case BPF_LINK_TYPE_RAW_TRACEPOINT:
jsonw_string_field(json_wtr, "tp_name",
u64_to_ptr(info->raw_tracepoint.tp_name));
jsonw_uint_field(json_wtr, "cookie", info->raw_tracepoint.cookie);
break;
case BPF_LINK_TYPE_TRACING:
err = get_prog_info(info->prog_id, &prog_info);
@ -502,6 +503,7 @@ static int show_link_close_json(int fd, struct bpf_link_info *info)
json_wtr);
jsonw_uint_field(json_wtr, "target_obj_id", info->tracing.target_obj_id);
jsonw_uint_field(json_wtr, "target_btf_id", info->tracing.target_btf_id);
jsonw_uint_field(json_wtr, "cookie", info->tracing.cookie);
break;
case BPF_LINK_TYPE_CGROUP:
jsonw_lluint_field(json_wtr, "cgroup_id",
@ -879,6 +881,8 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info)
case BPF_LINK_TYPE_RAW_TRACEPOINT:
printf("\n\ttp '%s' ",
(const char *)u64_to_ptr(info->raw_tracepoint.tp_name));
if (info->raw_tracepoint.cookie)
printf("cookie %llu ", info->raw_tracepoint.cookie);
break;
case BPF_LINK_TYPE_TRACING:
err = get_prog_info(info->prog_id, &prog_info);
@ -897,6 +901,8 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info)
printf("\n\ttarget_obj_id %u target_btf_id %u ",
info->tracing.target_obj_id,
info->tracing.target_btf_id);
if (info->tracing.cookie)
printf("\n\tcookie %llu ", info->tracing.cookie);
break;
case BPF_LINK_TYPE_CGROUP:
printf("\n\tcgroup_id %zu ", (size_t)info->cgroup.cgroup_id);

6
tools/bpf/bpftool/main.c
View file

@ -534,9 +534,9 @@ int main(int argc, char **argv)
usage();
if (version_requested)
return do_version(argc, argv);
ret = cmd_select(commands, argc, argv, do_help);
ret = do_version(argc, argv);
else
ret = cmd_select(commands, argc, argv, do_help);
if (json_output)
jsonw_destroy(&json_wtr);

13
tools/bpf/bpftool/main.h
View file

@ -15,6 +15,7 @@
#include <bpf/hashmap.h>
#include <bpf/libbpf.h>
#include <bpf/bpf.h>
#include "json_writer.h"
@ -140,8 +141,10 @@ void get_prog_full_name(const struct bpf_prog_info *prog_info, int prog_fd,
int get_fd_type(int fd);
const char *get_fd_type_name(enum bpf_obj_type type);
char *get_fdinfo(int fd, const char *key);
int open_obj_pinned(const char *path, bool quiet);
int open_obj_pinned_any(const char *path, enum bpf_obj_type exp_type);
int open_obj_pinned(const char *path, bool quiet,
const struct bpf_obj_get_opts *opts);
int open_obj_pinned_any(const char *path, enum bpf_obj_type exp_type,
const struct bpf_obj_get_opts *opts);
int mount_bpffs_for_file(const char *file_name);
int create_and_mount_bpffs_dir(const char *dir_name);
int do_pin_any(int argc, char **argv, int (*get_fd_by_id)(int *, char ***));
@ -167,10 +170,10 @@ int do_iter(int argc, char **argv) __weak;
int parse_u32_arg(int *argc, char ***argv, __u32 *val, const char *what);
int prog_parse_fd(int *argc, char ***argv);
int prog_parse_fds(int *argc, char ***argv, int **fds);
int map_parse_fd(int *argc, char ***argv);
int map_parse_fds(int *argc, char ***argv, int **fds);
int map_parse_fd(int *argc, char ***argv, __u32 open_flags);
int map_parse_fds(int *argc, char ***argv, int **fds, __u32 open_flags);
int map_parse_fd_and_info(int *argc, char ***argv, struct bpf_map_info *info,
__u32 *info_len);
__u32 *info_len, __u32 open_flags);
struct bpf_prog_linfo;
#if defined(HAVE_LLVM_SUPPORT) || defined(HAVE_LIBBFD_SUPPORT)

56
tools/bpf/bpftool/map.c
View file

@ -337,9 +337,9 @@ static void fill_per_cpu_value(struct bpf_map_info *info, void *value)
memcpy(value + i * step, value, info->value_size);
}
static int parse_elem(char **argv, struct bpf_map_info *info,
void *key, void *value, __u32 key_size, __u32 value_size,
__u32 *flags, __u32 **value_fd)
static int parse_elem(char **argv, struct bpf_map_info *info, void *key,
void *value, __u32 key_size, __u32 value_size,
__u32 *flags, __u32 **value_fd, __u32 open_flags)
{
if (!*argv) {
if (!key && !value)
@ -362,7 +362,7 @@ static int parse_elem(char **argv, struct bpf_map_info *info,
return -1;
return parse_elem(argv, info, NULL, value, key_size, value_size,
flags, value_fd);
flags, value_fd, open_flags);
} else if (is_prefix(*argv, "value")) {
int fd;
@ -388,7 +388,7 @@ static int parse_elem(char **argv, struct bpf_map_info *info,
return -1;
}
fd = map_parse_fd(&argc, &argv);
fd = map_parse_fd(&argc, &argv, open_flags);
if (fd < 0)
return -1;
@ -424,7 +424,7 @@ static int parse_elem(char **argv, struct bpf_map_info *info,
}
return parse_elem(argv, info, key, NULL, key_size, value_size,
flags, NULL);
flags, NULL, open_flags);
} else if (is_prefix(*argv, "any") || is_prefix(*argv, "noexist") ||
is_prefix(*argv, "exist")) {
if (!flags) {
@ -440,7 +440,7 @@ static int parse_elem(char **argv, struct bpf_map_info *info,
*flags = BPF_EXIST;
return parse_elem(argv + 1, info, key, value, key_size,
value_size, NULL, value_fd);
value_size, NULL, value_fd, open_flags);
}
p_err("expected key or value, got: %s", *argv);
@ -639,7 +639,7 @@ static int do_show_subset(int argc, char **argv)
p_err("mem alloc failed");
return -1;
}
nb_fds = map_parse_fds(&argc, &argv, &fds);
nb_fds = map_parse_fds(&argc, &argv, &fds, BPF_F_RDONLY);
if (nb_fds < 1)
goto exit_free;
@ -672,12 +672,15 @@ exit_free:
static int do_show(int argc, char **argv)
{
LIBBPF_OPTS(bpf_get_fd_by_id_opts, opts);
struct bpf_map_info info = {};
__u32 len = sizeof(info);
__u32 id = 0;
int err;
int fd;
opts.open_flags = BPF_F_RDONLY;
if (show_pinned) {
map_table = hashmap__new(hash_fn_for_key_as_id,
equal_fn_for_key_as_id, NULL);
@ -707,7 +710,7 @@ static int do_show(int argc, char **argv)
break;
}
fd = bpf_map_get_fd_by_id(id);
fd = bpf_map_get_fd_by_id_opts(id, &opts);
if (fd < 0) {
if (errno == ENOENT)
continue;
@ -909,7 +912,7 @@ static int do_dump(int argc, char **argv)
p_err("mem alloc failed");
return -1;
}
nb_fds = map_parse_fds(&argc, &argv, &fds);
nb_fds = map_parse_fds(&argc, &argv, &fds, BPF_F_RDONLY);
if (nb_fds < 1)
goto exit_free;
@ -997,7 +1000,7 @@ static int do_update(int argc, char **argv)
if (argc < 2)
usage();
fd = map_parse_fd_and_info(&argc, &argv, &info, &len);
fd = map_parse_fd_and_info(&argc, &argv, &info, &len, 0);
if (fd < 0)
return -1;
@ -1006,7 +1009,7 @@ static int do_update(int argc, char **argv)
goto exit_free;
err = parse_elem(argv, &info, key, value, info.key_size,
info.value_size, &flags, &value_fd);
info.value_size, &flags, &value_fd, 0);
if (err)
goto exit_free;
@ -1076,7 +1079,7 @@ static int do_lookup(int argc, char **argv)
if (argc < 2)
usage();
fd = map_parse_fd_and_info(&argc, &argv, &info, &len);
fd = map_parse_fd_and_info(&argc, &argv, &info, &len, BPF_F_RDONLY);
if (fd < 0)
return -1;
@ -1084,7 +1087,8 @@ static int do_lookup(int argc, char **argv)
if (err)
goto exit_free;
err = parse_elem(argv, &info, key, NULL, info.key_size, 0, NULL, NULL);
err = parse_elem(argv, &info, key, NULL, info.key_size, 0, NULL, NULL,
BPF_F_RDONLY);
if (err)
goto exit_free;
@ -1127,7 +1131,7 @@ static int do_getnext(int argc, char **argv)
if (argc < 2)
usage();
fd = map_parse_fd_and_info(&argc, &argv, &info, &len);
fd = map_parse_fd_and_info(&argc, &argv, &info, &len, BPF_F_RDONLY);
if (fd < 0)
return -1;
@ -1140,8 +1144,8 @@ static int do_getnext(int argc, char **argv)
}
if (argc) {
err = parse_elem(argv, &info, key, NULL, info.key_size, 0,
NULL, NULL);
err = parse_elem(argv, &info, key, NULL, info.key_size, 0, NULL,
NULL, BPF_F_RDONLY);
if (err)
goto exit_free;
} else {
@ -1198,7 +1202,7 @@ static int do_delete(int argc, char **argv)
if (argc < 2)
usage();
fd = map_parse_fd_and_info(&argc, &argv, &info, &len);
fd = map_parse_fd_and_info(&argc, &argv, &info, &len, 0);
if (fd < 0)
return -1;
@ -1209,7 +1213,8 @@ static int do_delete(int argc, char **argv)
goto exit_free;
}
err = parse_elem(argv, &info, key, NULL, info.key_size, 0, NULL, NULL);
err = parse_elem(argv, &info, key, NULL, info.key_size, 0, NULL, NULL,
0);
if (err)
goto exit_free;
@ -1226,11 +1231,16 @@ exit_free:
return err;
}
static int map_parse_read_only_fd(int *argc, char ***argv)
{
return map_parse_fd(argc, argv, BPF_F_RDONLY);
}
static int do_pin(int argc, char **argv)
{
int err;
err = do_pin_any(argc, argv, map_parse_fd);
err = do_pin_any(argc, argv, map_parse_read_only_fd);
if (!err && json_output)
jsonw_null(json_wtr);
return err;
@ -1319,7 +1329,7 @@ offload_dev:
if (!REQ_ARGS(2))
usage();
inner_map_fd = map_parse_fd_and_info(&argc, &argv,
&info, &len);
&info, &len, BPF_F_RDONLY);
if (inner_map_fd < 0)
return -1;
attr.inner_map_fd = inner_map_fd;
@ -1368,7 +1378,7 @@ static int do_pop_dequeue(int argc, char **argv)
if (argc < 2)
usage();
fd = map_parse_fd_and_info(&argc, &argv, &info, &len);
fd = map_parse_fd_and_info(&argc, &argv, &info, &len, 0);
if (fd < 0)
return -1;
@ -1407,7 +1417,7 @@ static int do_freeze(int argc, char **argv)
if (!REQ_ARGS(2))
return -1;
fd = map_parse_fd(&argc, &argv);
fd = map_parse_fd(&argc, &argv, 0);
if (fd < 0)
return -1;

3
tools/bpf/bpftool/map_perf_ring.c
View file

@ -128,7 +128,8 @@ int do_event_pipe(int argc, char **argv)
int err, map_fd;
map_info_len = sizeof(map_info);
map_fd = map_parse_fd_and_info(&argc, &argv, &map_info, &map_info_len);
map_fd = map_parse_fd_and_info(&argc, &argv, &map_info, &map_info_len,
0);
if (map_fd < 0)
return -1;

15
tools/bpf/bpftool/net.c
View file

@ -366,17 +366,18 @@ static int dump_link_nlmsg(void *cookie, void *msg, struct nlattr **tb)
{
struct bpf_netdev_t *netinfo = cookie;
struct ifinfomsg *ifinfo = msg;
struct ip_devname_ifindex *tmp;
if (netinfo->filter_idx > 0 && netinfo->filter_idx != ifinfo->ifi_index)
return 0;
if (netinfo->used_len == netinfo->array_len) {
netinfo->devices = realloc(netinfo->devices,
(netinfo->array_len + 16) *
sizeof(struct ip_devname_ifindex));
if (!netinfo->devices)
tmp = realloc(netinfo->devices,
(netinfo->array_len + 16) * sizeof(struct ip_devname_ifindex));
if (!tmp)
return -ENOMEM;
netinfo->devices = tmp;
netinfo->array_len += 16;
}
netinfo->devices[netinfo->used_len].ifindex = ifinfo->ifi_index;
@ -395,6 +396,7 @@ static int dump_class_qdisc_nlmsg(void *cookie, void *msg, struct nlattr **tb)
{
struct bpf_tcinfo_t *tcinfo = cookie;
struct tcmsg *info = msg;
struct tc_kind_handle *tmp;
if (tcinfo->is_qdisc) {
/* skip clsact qdisc */
@ -406,11 +408,12 @@ static int dump_class_qdisc_nlmsg(void *cookie, void *msg, struct nlattr **tb)
}
if (tcinfo->used_len == tcinfo->array_len) {
tcinfo->handle_array = realloc(tcinfo->handle_array,
tmp = realloc(tcinfo->handle_array,
(tcinfo->array_len + 16) * sizeof(struct tc_kind_handle));
if (!tcinfo->handle_array)
if (!tmp)
return -ENOMEM;
tcinfo->handle_array = tmp;
tcinfo->array_len += 16;
}
tcinfo->handle_array[tcinfo->used_len].handle = info->tcm_handle;

53
tools/bpf/bpftool/prog.c
View file

@ -1062,7 +1062,7 @@ static int parse_attach_detach_args(int argc, char **argv, int *progfd,
if (!REQ_ARGS(2))
return -EINVAL;
*mapfd = map_parse_fd(&argc, &argv);
*mapfd = map_parse_fd(&argc, &argv, 0);
if (*mapfd < 0)
return *mapfd;
@ -1113,6 +1113,52 @@ static int do_detach(int argc, char **argv)
return 0;
}
enum prog_tracelog_mode {
TRACE_STDOUT,
TRACE_STDERR,
};
static int
prog_tracelog_stream(int prog_fd, enum prog_tracelog_mode mode)
{
FILE *file = mode == TRACE_STDOUT ? stdout : stderr;
int stream_id = mode == TRACE_STDOUT ? 1 : 2;
char buf[512];
int ret;
ret = 0;
do {
ret = bpf_prog_stream_read(prog_fd, stream_id, buf, sizeof(buf), NULL);
if (ret > 0)
fwrite(buf, sizeof(buf[0]), ret, file);
} while (ret > 0);
fflush(file);
return ret ? -1 : 0;
}
static int do_tracelog_any(int argc, char **argv)
{
enum prog_tracelog_mode mode;
int fd;
if (argc == 0)
return do_tracelog(argc, argv);
if (!is_prefix(*argv, "stdout") && !is_prefix(*argv, "stderr"))
usage();
mode = is_prefix(*argv, "stdout") ? TRACE_STDOUT : TRACE_STDERR;
NEXT_ARG();
if (!REQ_ARGS(2))
return -1;
fd = prog_parse_fd(&argc, &argv);
if (fd < 0)
return -1;
return prog_tracelog_stream(fd, mode);
}
static int check_single_stdin(char *file_data_in, char *file_ctx_in)
{
if (file_data_in && file_ctx_in &&
@ -1608,7 +1654,7 @@ static int load_with_options(int argc, char **argv, bool first_prog_only)
}
NEXT_ARG();
fd = map_parse_fd(&argc, &argv);
fd = map_parse_fd(&argc, &argv, 0);
if (fd < 0)
goto err_free_reuse_maps;
@ -2493,6 +2539,7 @@ static int do_help(int argc, char **argv)
" [repeat N]\n"
" %1$s %2$s profile PROG [duration DURATION] METRICs\n"
" %1$s %2$s tracelog\n"
" %1$s %2$s tracelog { stdout | stderr } PROG\n"
" %1$s %2$s help\n"
"\n"
" " HELP_SPEC_MAP "\n"
@ -2532,7 +2579,7 @@ static const struct cmd cmds[] = {
{ "loadall", do_loadall },
{ "attach", do_attach },
{ "detach", do_detach },
{ "tracelog", do_tracelog },
{ "tracelog", do_tracelog_any },
{ "run", do_run },
{ "profile", do_profile },
{ 0 }

45
tools/include/uapi/linux/bpf.h
View file

@ -450,6 +450,7 @@ union bpf_iter_link_info {
* * **struct bpf_map_info**
* * **struct bpf_btf_info**
* * **struct bpf_link_info**
* * **struct bpf_token_info**
*
* Return
* Returns zero on success. On error, -1 is returned and *errno*
@ -906,6 +907,17 @@ union bpf_iter_link_info {
* A new file descriptor (a nonnegative integer), or -1 if an
* error occurred (in which case, *errno* is set appropriately).
*
* BPF_PROG_STREAM_READ_BY_FD
* Description
* Read data of a program's BPF stream. The program is identified
* by *prog_fd*, and the stream is identified by the *stream_id*.
* The data is copied to a buffer pointed to by *stream_buf*, and
* filled less than or equal to *stream_buf_len* bytes.
*
* Return
* Number of bytes read from the stream on success, or -1 if an
* error occurred (in which case, *errno* is set appropriately).
*
* NOTES
* eBPF objects (maps and programs) can be shared between processes.
*
@ -961,6 +973,7 @@ enum bpf_cmd {
BPF_LINK_DETACH,
BPF_PROG_BIND_MAP,
BPF_TOKEN_CREATE,
BPF_PROG_STREAM_READ_BY_FD,
__MAX_BPF_CMD,
};
@ -1463,6 +1476,11 @@ struct bpf_stack_build_id {
#define BPF_OBJ_NAME_LEN 16U
enum {
BPF_STREAM_STDOUT = 1,
BPF_STREAM_STDERR = 2,
};
union bpf_attr {
struct { /* anonymous struct used by BPF_MAP_CREATE command */
__u32 map_type; /* one of enum bpf_map_type */
@ -1794,6 +1812,13 @@ union bpf_attr {
};
__u64 expected_revision;
} netkit;
struct {
union {
__u32 relative_fd;
__u32 relative_id;
};
__u64 expected_revision;
} cgroup;
};
} link_create;
@ -1842,6 +1867,13 @@ union bpf_attr {
__u32 bpffs_fd;
} token_create;
struct {
__aligned_u64 stream_buf;
__u32 stream_buf_len;
__u32 stream_id;
__u32 prog_fd;
} prog_stream_read;
} __attribute__((aligned(8)));
/* The description below is an attempt at providing documentation to eBPF
@ -2403,7 +2435,7 @@ union bpf_attr {
* into it. An example is available in file
* *samples/bpf/trace_output_user.c* in the Linux kernel source
* tree (the eBPF program counterpart is in
* *samples/bpf/trace_output_kern.c*).
* *samples/bpf/trace_output.bpf.c*).
*
* **bpf_perf_event_output**\ () achieves better performance
* than **bpf_trace_printk**\ () for sharing data with user
@ -6653,11 +6685,15 @@ struct bpf_link_info {
struct {
__aligned_u64 tp_name; /* in/out: tp_name buffer ptr */
__u32 tp_name_len; /* in/out: tp_name buffer len */
__u32 :32;
__u64 cookie;
} raw_tracepoint;
struct {
__u32 attach_type;
__u32 target_obj_id; /* prog_id for PROG_EXT, otherwise btf object id */
__u32 target_btf_id; /* BTF type id inside the object */
__u32 :32;
__u64 cookie;
} tracing;
struct {
__u64 cgroup_id;
@ -6768,6 +6804,13 @@ struct bpf_link_info {
};
} __attribute__((aligned(8)));
struct bpf_token_info {
__u64 allowed_cmds;
__u64 allowed_maps;
__u64 allowed_progs;
__u64 allowed_attachs;
} __attribute__((aligned(8)));
/* User bpf_sock_addr struct to access socket fields and sockaddr struct passed
* by user and intended to be used by socket (e.g. to bind to, depends on
* attach type).

64
tools/lib/bpf/bpf.c
View file

@ -837,6 +837,50 @@ int bpf_link_create(int prog_fd, int target_fd,
if (!OPTS_ZEROED(opts, netkit))
return libbpf_err(-EINVAL);
break;
case BPF_CGROUP_INET_INGRESS:
case BPF_CGROUP_INET_EGRESS:
case BPF_CGROUP_INET_SOCK_CREATE:
case BPF_CGROUP_INET_SOCK_RELEASE:
case BPF_CGROUP_INET4_BIND:
case BPF_CGROUP_INET6_BIND:
case BPF_CGROUP_INET4_POST_BIND:
case BPF_CGROUP_INET6_POST_BIND:
case BPF_CGROUP_INET4_CONNECT:
case BPF_CGROUP_INET6_CONNECT:
case BPF_CGROUP_UNIX_CONNECT:
case BPF_CGROUP_INET4_GETPEERNAME:
case BPF_CGROUP_INET6_GETPEERNAME:
case BPF_CGROUP_UNIX_GETPEERNAME:
case BPF_CGROUP_INET4_GETSOCKNAME:
case BPF_CGROUP_INET6_GETSOCKNAME:
case BPF_CGROUP_UNIX_GETSOCKNAME:
case BPF_CGROUP_UDP4_SENDMSG:
case BPF_CGROUP_UDP6_SENDMSG:
case BPF_CGROUP_UNIX_SENDMSG:
case BPF_CGROUP_UDP4_RECVMSG:
case BPF_CGROUP_UDP6_RECVMSG:
case BPF_CGROUP_UNIX_RECVMSG:
case BPF_CGROUP_SOCK_OPS:
case BPF_CGROUP_DEVICE:
case BPF_CGROUP_SYSCTL:
case BPF_CGROUP_GETSOCKOPT:
case BPF_CGROUP_SETSOCKOPT:
case BPF_LSM_CGROUP:
relative_fd = OPTS_GET(opts, cgroup.relative_fd, 0);
relative_id = OPTS_GET(opts, cgroup.relative_id, 0);
if (relative_fd && relative_id)
return libbpf_err(-EINVAL);
if (relative_id) {
attr.link_create.cgroup.relative_id = relative_id;
attr.link_create.flags |= BPF_F_ID;
} else {
attr.link_create.cgroup.relative_fd = relative_fd;
}
attr.link_create.cgroup.expected_revision =
OPTS_GET(opts, cgroup.expected_revision, 0);
if (!OPTS_ZEROED(opts, cgroup))
return libbpf_err(-EINVAL);
break;
default:
if (!OPTS_ZEROED(opts, flags))
return libbpf_err(-EINVAL);
@ -1331,3 +1375,23 @@ int bpf_token_create(int bpffs_fd, struct bpf_token_create_opts *opts)
fd = sys_bpf_fd(BPF_TOKEN_CREATE, &attr, attr_sz);
return libbpf_err_errno(fd);
}
int bpf_prog_stream_read(int prog_fd, __u32 stream_id, void *buf, __u32 buf_len,
struct bpf_prog_stream_read_opts *opts)
{
const size_t attr_sz = offsetofend(union bpf_attr, prog_stream_read);
union bpf_attr attr;
int err;
if (!OPTS_VALID(opts, bpf_prog_stream_read_opts))
return libbpf_err(-EINVAL);
memset(&attr, 0, attr_sz);
attr.prog_stream_read.stream_buf = ptr_to_u64(buf);
attr.prog_stream_read.stream_buf_len = buf_len;
attr.prog_stream_read.stream_id = stream_id;
attr.prog_stream_read.prog_fd = prog_fd;
err = sys_bpf(BPF_PROG_STREAM_READ_BY_FD, &attr, attr_sz);
return libbpf_err_errno(err);
}

26
tools/lib/bpf/bpf.h
View file

@ -438,6 +438,11 @@ struct bpf_link_create_opts {
__u32 relative_id;
__u64 expected_revision;
} netkit;
struct {
__u32 relative_fd;
__u32 relative_id;
__u64 expected_revision;
} cgroup;
};
size_t :0;
};
@ -704,6 +709,27 @@ struct bpf_token_create_opts {
LIBBPF_API int bpf_token_create(int bpffs_fd,
struct bpf_token_create_opts *opts);
struct bpf_prog_stream_read_opts {
size_t sz;
size_t :0;
};
#define bpf_prog_stream_read_opts__last_field sz
/**
* @brief **bpf_prog_stream_read** reads data from the BPF stream of a given BPF
* program.
*
* @param prog_fd FD for the BPF program whose BPF stream is to be read.
* @param stream_id ID of the BPF stream to be read.
* @param buf Buffer to read data into from the BPF stream.
* @param buf_len Maximum number of bytes to read from the BPF stream.
* @param opts optional options, can be NULL
*
* @return The number of bytes read, on success; negative error code, otherwise
* (errno is also set to the error code)
*/
LIBBPF_API int bpf_prog_stream_read(int prog_fd, __u32 stream_id, void *buf, __u32 buf_len,
struct bpf_prog_stream_read_opts *opts);
#ifdef __cplusplus
} /* extern "C" */
#endif

17
tools/lib/bpf/bpf_helpers.h
View file

@ -215,6 +215,7 @@ enum libbpf_tristate {
#define __arg_nonnull __attribute((btf_decl_tag("arg:nonnull")))
#define __arg_nullable __attribute((btf_decl_tag("arg:nullable")))
#define __arg_trusted __attribute((btf_decl_tag("arg:trusted")))
#define __arg_untrusted __attribute((btf_decl_tag("arg:untrusted")))
#define __arg_arena __attribute((btf_decl_tag("arg:arena")))
#ifndef ___bpf_concat
@ -314,6 +315,22 @@ enum libbpf_tristate {
___param, sizeof(___param)); \
})
extern int bpf_stream_vprintk(int stream_id, const char *fmt__str, const void *args,
__u32 len__sz, void *aux__prog) __weak __ksym;
#define bpf_stream_printk(stream_id, fmt, args...) \
({ \
static const char ___fmt[] = fmt; \
unsigned long long ___param[___bpf_narg(args)]; \
\
_Pragma("GCC diagnostic push") \
_Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \
___bpf_fill(___param, args); \
_Pragma("GCC diagnostic pop") \
\
bpf_stream_vprintk(stream_id, ___fmt, ___param, sizeof(___param), NULL);\
})
/* Use __bpf_printk when bpf_printk call has 3 or fewer fmt args
* Otherwise use __bpf_vprintk
*/

3
tools/lib/bpf/btf.h
View file

@ -326,9 +326,10 @@ struct btf_dump_type_data_opts {
bool compact; /* no newlines/indentation */
bool skip_names; /* skip member/type names */
bool emit_zeroes; /* show 0-valued fields */
bool emit_strings; /* print char arrays as strings */
size_t :0;
};
#define btf_dump_type_data_opts__last_field emit_zeroes
#define btf_dump_type_data_opts__last_field emit_strings
LIBBPF_API int
btf_dump__dump_type_data(struct btf_dump *d, __u32 id,

55
tools/lib/bpf/btf_dump.c
View file

@ -68,6 +68,7 @@ struct btf_dump_data {
bool compact;
bool skip_names;
bool emit_zeroes;
bool emit_strings;
__u8 indent_lvl; /* base indent level */
char indent_str[BTF_DATA_INDENT_STR_LEN];
/* below are used during iteration */
@ -2031,6 +2032,52 @@ static int btf_dump_var_data(struct btf_dump *d,
return btf_dump_dump_type_data(d, NULL, t, type_id, data, 0, 0);
}
static int btf_dump_string_data(struct btf_dump *d,
const struct btf_type *t,
__u32 id,
const void *data)
{
const struct btf_array *array = btf_array(t);
const char *chars = data;
__u32 i;
/* Make sure it is a NUL-terminated string. */
for (i = 0; i < array->nelems; i++) {
if ((void *)(chars + i) >= d->typed_dump->data_end)
return -E2BIG;
if (chars[i] == '\0')
break;
}
if (i == array->nelems) {
/* The caller will print this as a regular array. */
return -EINVAL;
}
btf_dump_data_pfx(d);
btf_dump_printf(d, "\"");
for (i = 0; i < array->nelems; i++) {
char c = chars[i];
if (c == '\0') {
/*
* When printing character arrays as strings, NUL bytes
* are always treated as string terminators; they are
* never printed.
*/
break;
}
if (isprint(c))
btf_dump_printf(d, "%c", c);
else
btf_dump_printf(d, "\\x%02x", (__u8)c);
}
btf_dump_printf(d, "\"");
return 0;
}
static int btf_dump_array_data(struct btf_dump *d,
const struct btf_type *t,
__u32 id,
@ -2058,8 +2105,13 @@ static int btf_dump_array_data(struct btf_dump *d,
* char arrays, so if size is 1 and element is
* printable as a char, we'll do that.
*/
if (elem_size == 1)
if (elem_size == 1) {
if (d->typed_dump->emit_strings &&
btf_dump_string_data(d, t, id, data) == 0) {
return 0;
}
d->typed_dump->is_array_char = true;
}
}
/* note that we increment depth before calling btf_dump_print() below;
@ -2547,6 +2599,7 @@ int btf_dump__dump_type_data(struct btf_dump *d, __u32 id,
d->typed_dump->compact = OPTS_GET(opts, compact, false);
d->typed_dump->skip_names = OPTS_GET(opts, skip_names, false);
d->typed_dump->emit_zeroes = OPTS_GET(opts, emit_zeroes, false);
d->typed_dump->emit_strings = OPTS_GET(opts, emit_strings, false);
ret = btf_dump_dump_type_data(d, NULL, t, id, data, 0, 0);

35
tools/lib/bpf/libbpf.c
View file

@ -4582,6 +4582,11 @@ static int bpf_program__record_reloc(struct bpf_program *prog,
/* arena data relocation */
if (shdr_idx == obj->efile.arena_data_shndx) {
if (obj->arena_map_idx < 0) {
pr_warn("prog '%s': bad arena data relocation at insn %u, no arena maps defined\n",
prog->name, insn_idx);
return -LIBBPF_ERRNO__RELOC;
}
reloc_desc->type = RELO_DATA;
reloc_desc->insn_idx = insn_idx;
reloc_desc->map_idx = obj->arena_map_idx;
@ -9216,7 +9221,7 @@ int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts)
return libbpf_err(-EFAULT);
if (!OPTS_VALID(opts, gen_loader_opts))
return libbpf_err(-EINVAL);
gen = calloc(sizeof(*gen), 1);
gen = calloc(1, sizeof(*gen));
if (!gen)
return libbpf_err(-ENOMEM);
gen->opts = opts;
@ -12847,6 +12852,34 @@ struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifi
return bpf_program_attach_fd(prog, ifindex, "xdp", NULL);
}
struct bpf_link *
bpf_program__attach_cgroup_opts(const struct bpf_program *prog, int cgroup_fd,
const struct bpf_cgroup_opts *opts)
{
LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
__u32 relative_id;
int relative_fd;
if (!OPTS_VALID(opts, bpf_cgroup_opts))
return libbpf_err_ptr(-EINVAL);
relative_id = OPTS_GET(opts, relative_id, 0);
relative_fd = OPTS_GET(opts, relative_fd, 0);
if (relative_fd && relative_id) {
pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n",
prog->name);
return libbpf_err_ptr(-EINVAL);
}
link_create_opts.cgroup.expected_revision = OPTS_GET(opts, expected_revision, 0);
link_create_opts.cgroup.relative_fd = relative_fd;
link_create_opts.cgroup.relative_id = relative_id;
link_create_opts.flags = OPTS_GET(opts, flags, 0);
return bpf_program_attach_fd(prog, cgroup_fd, "cgroup", &link_create_opts);
}
struct bpf_link *
bpf_program__attach_tcx(const struct bpf_program *prog, int ifindex,
const struct bpf_tcx_opts *opts)

15
tools/lib/bpf/libbpf.h
View file

@ -877,6 +877,21 @@ LIBBPF_API struct bpf_link *
bpf_program__attach_netkit(const struct bpf_program *prog, int ifindex,
const struct bpf_netkit_opts *opts);
struct bpf_cgroup_opts {
/* size of this struct, for forward/backward compatibility */
size_t sz;
__u32 flags;
__u32 relative_fd;
__u32 relative_id;
__u64 expected_revision;
size_t :0;
};
#define bpf_cgroup_opts__last_field expected_revision
LIBBPF_API struct bpf_link *
bpf_program__attach_cgroup_opts(const struct bpf_program *prog, int cgroup_fd,
const struct bpf_cgroup_opts *opts);
struct bpf_map;
LIBBPF_API struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map);

5
tools/lib/bpf/libbpf.map
View file

@ -437,6 +437,8 @@ LIBBPF_1.6.0 {
bpf_linker__add_fd;
bpf_linker__new_fd;
bpf_object__prepare;
bpf_prog_stream_read;
bpf_program__attach_cgroup_opts;
bpf_program__func_info;
bpf_program__func_info_cnt;
bpf_program__line_info;
@ -444,3 +446,6 @@ LIBBPF_1.6.0 {
btf__add_decl_attr;
btf__add_type_attr;
} LIBBPF_1.5.0;
LIBBPF_1.7.0 {
} LIBBPF_1.6.0;

2
tools/lib/bpf/libbpf_version.h
View file

@ -4,6 +4,6 @@
#define __LIBBPF_VERSION_H
#define LIBBPF_MAJOR_VERSION 1
#define LIBBPF_MINOR_VERSION 6
#define LIBBPF_MINOR_VERSION 7
#endif /* __LIBBPF_VERSION_H */

10
tools/lib/bpf/usdt.c
View file

@ -59,7 +59,7 @@
*
* STAP_PROBE3(my_usdt_provider, my_usdt_probe_name, 123, x, &y);
*
* USDT is identified by it's <provider-name>:<probe-name> pair of names. Each
* USDT is identified by its <provider-name>:<probe-name> pair of names. Each
* individual USDT has a fixed number of arguments (3 in the above example)
* and specifies values of each argument as if it was a function call.
*
@ -81,7 +81,7 @@
* NOP instruction that kernel can replace with an interrupt instruction to
* trigger instrumentation code (BPF program for all that we care about).
*
* Semaphore above is and optional feature. It records an address of a 2-byte
* Semaphore above is an optional feature. It records an address of a 2-byte
* refcount variable (normally in '.probes' ELF section) used for signaling if
* there is anything that is attached to USDT. This is useful for user
* applications if, for example, they need to prepare some arguments that are
@ -121,7 +121,7 @@
* a uprobe BPF program (which for kernel, at least currently, is just a kprobe
* program, so BPF_PROG_TYPE_KPROBE program type). With the only difference
* that uprobe is usually attached at the function entry, while USDT will
* normally will be somewhere inside the function. But it should always be
* normally be somewhere inside the function. But it should always be
* pointing to NOP instruction, which makes such uprobes the fastest uprobe
* kind.
*
@ -151,7 +151,7 @@
* libbpf sets to spec ID during attach time, or, if kernel is too old to
* support BPF cookie, through IP-to-spec-ID map that libbpf maintains in such
* case. The latter means that some modes of operation can't be supported
* without BPF cookie. Such mode is attaching to shared library "generically",
* without BPF cookie. Such a mode is attaching to shared library "generically",
* without specifying target process. In such case, it's impossible to
* calculate absolute IP addresses for IP-to-spec-ID map, and thus such mode
* is not supported without BPF cookie support.
@ -185,7 +185,7 @@
* as even if USDT spec string is the same, USDT cookie value can be
* different. It was deemed excessive to try to deduplicate across independent
* USDT attachments by taking into account USDT spec string *and* USDT cookie
* value, which would complicated spec ID accounting significantly for little
* value, which would complicate spec ID accounting significantly for little
* gain.
*/

1
tools/testing/selftests/bpf/DENYLIST
View file

@ -1,6 +1,5 @@
# TEMPORARY
# Alphabetical order
dynptr/test_probe_read_user_str_dynptr # disabled until https://patchwork.kernel.org/project/linux-mm/patch/20250422131449.57177-1-mykyta.yatsenko5@gmail.com/ makes it into the bpf-next
get_stack_raw_tp # spams with kernel warnings until next bpf -> bpf-next merge
stacktrace_build_id
stacktrace_build_id_nmi

1
tools/testing/selftests/bpf/DENYLIST.aarch64
View file

@ -1 +0,0 @@
tracing_struct/struct_many_args # struct_many_args:FAIL:tracing_struct_many_args__attach unexpected error: -524

6
tools/testing/selftests/bpf/Makefile
View file

@ -109,6 +109,7 @@ TEST_PROGS := test_kmod.sh \
test_xdping.sh \
test_bpftool_build.sh \
test_bpftool.sh \
test_bpftool_map.sh \
test_bpftool_metadata.sh \
test_doc_build.sh \
test_xsk.sh \
@ -840,6 +841,11 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \
$(call msg,BINARY,,$@)
$(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@
# This works around GCC warning about snprintf truncating strings like:
#
# char a[PATH_MAX], b[PATH_MAX];
# snprintf(a, "%s/foo", b); // triggers -Wformat-truncation
$(OUTPUT)/veristat.o: CFLAGS += -Wno-format-truncation
$(OUTPUT)/veristat.o: $(BPFOBJ)
$(OUTPUT)/veristat: $(OUTPUT)/veristat.o
$(call msg,BINARY,,$@)

3
tools/testing/selftests/bpf/bpf_arena_common.h
View file

@ -46,8 +46,11 @@
void __arena* bpf_arena_alloc_pages(void *map, void __arena *addr, __u32 page_cnt,
int node_id, __u64 flags) __ksym __weak;
int bpf_arena_reserve_pages(void *map, void __arena *addr, __u32 page_cnt) __ksym __weak;
void bpf_arena_free_pages(void *map, void __arena *ptr, __u32 page_cnt) __ksym __weak;
#define arena_base(map) ((void __arena *)((struct bpf_arena *)(map))->user_vm_start)
#else /* when compiled as user space code */
#define __arena

2
tools/testing/selftests/bpf/bpf_atomic.h
View file

@ -61,7 +61,7 @@ extern bool CONFIG_X86_64 __kconfig __weak;
#define smp_mb() \
({ \
unsigned long __val; \
volatile unsigned long __val; \
__sync_fetch_and_add(&__val, 0); \
})

2
tools/testing/selftests/bpf/bpf_kfuncs.h
View file

@ -69,7 +69,7 @@ extern int bpf_get_file_xattr(struct file *file, const char *name,
struct bpf_dynptr *value_ptr) __ksym;
extern int bpf_get_fsverity_digest(struct file *file, struct bpf_dynptr *digest_ptr) __ksym;
extern struct bpf_key *bpf_lookup_user_key(__u32 serial, __u64 flags) __ksym;
extern struct bpf_key *bpf_lookup_user_key(__s32 serial, __u64 flags) __ksym;
extern struct bpf_key *bpf_lookup_system_key(__u64 id) __ksym;
extern void bpf_key_put(struct bpf_key *key) __ksym;
extern int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_ptr,

21
tools/testing/selftests/bpf/cgroup_helpers.c
View file

@ -4,6 +4,7 @@
#include <sys/mount.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/xattr.h>
#include <linux/limits.h>
#include <stdio.h>
#include <stdlib.h>
@ -318,6 +319,26 @@ int join_parent_cgroup(const char *relative_path)
return join_cgroup_from_top(cgroup_path);
}
/**
* set_cgroup_xattr() - Set xattr on a cgroup dir
* @relative_path: The cgroup path, relative to the workdir, to set xattr
* @name: xattr name
* @value: xattr value
*
* This function set xattr on cgroup dir.
*
* On success, it returns 0, otherwise on failure it returns -1.
*/
int set_cgroup_xattr(const char *relative_path,
const char *name,
const char *value)
{
char cgroup_path[PATH_MAX + 1];
format_cgroup_path(cgroup_path, relative_path);
return setxattr(cgroup_path, name, value, strlen(value) + 1, 0);
}
/**
* __cleanup_cgroup_environment() - Delete temporary cgroups
*

4
tools/testing/selftests/bpf/cgroup_helpers.h
View file

@ -26,6 +26,10 @@ int join_cgroup(const char *relative_path);
int join_root_cgroup(void);
int join_parent_cgroup(const char *relative_path);
int set_cgroup_xattr(const char *relative_path,
const char *name,
const char *value);
int setup_cgroup_environment(void);
void cleanup_cgroup_environment(void);

1
tools/testing/selftests/bpf/config
View file

@ -108,6 +108,7 @@ CONFIG_IP_NF_IPTABLES=y
CONFIG_IP6_NF_IPTABLES=y
CONFIG_IP6_NF_FILTER=y
CONFIG_NF_NAT=y
CONFIG_PACKET=y
CONFIG_RC_CORE=y
CONFIG_SECURITY=y
CONFIG_SECURITYFS=y

93
tools/testing/selftests/bpf/config.ppc64el Normal file
View file

@ -0,0 +1,93 @@
CONFIG_ALTIVEC=y
CONFIG_AUDIT=y
CONFIG_BLK_CGROUP=y
CONFIG_BLK_DEV_INITRD=y
CONFIG_BLK_DEV_RAM=y
CONFIG_BONDING=y
CONFIG_BPF_JIT_ALWAYS_ON=y
CONFIG_BPF_PRELOAD_UMD=y
CONFIG_BPF_PRELOAD=y
CONFIG_CGROUP_CPUACCT=y
CONFIG_CGROUP_DEVICE=y
CONFIG_CGROUP_FREEZER=y
CONFIG_CGROUP_HUGETLB=y
CONFIG_CGROUP_NET_CLASSID=y
CONFIG_CGROUP_PERF=y
CONFIG_CGROUP_PIDS=y
CONFIG_CGROUP_SCHED=y
CONFIG_CGROUPS=y
CONFIG_CMDLINE_BOOL=y
CONFIG_CMDLINE="console=hvc0 wg.success=hvc1 panic_on_warn=1"
CONFIG_CPU_LITTLE_ENDIAN=y
CONFIG_CPUSETS=y
CONFIG_DEBUG_ATOMIC_SLEEP=y
CONFIG_DEBUG_FS=y
CONFIG_DETECT_HUNG_TASK=y
CONFIG_DEVTMPFS_MOUNT=y
CONFIG_DEVTMPFS=y
CONFIG_EXPERT=y
CONFIG_EXT4_FS_POSIX_ACL=y
CONFIG_EXT4_FS_SECURITY=y
CONFIG_EXT4_FS=y
CONFIG_FRAME_POINTER=y
CONFIG_FRAME_WARN=1280
CONFIG_HARDLOCKUP_DETECTOR=y
CONFIG_HIGH_RES_TIMERS=y
CONFIG_HUGETLBFS=y
CONFIG_HVC_CONSOLE=y
CONFIG_INET=y
CONFIG_IP_ADVANCED_ROUTER=y
CONFIG_IP_MULTICAST=y
CONFIG_IP_MULTIPLE_TABLES=y
CONFIG_IPV6_SEG6_LWTUNNEL=y
CONFIG_JUMP_LABEL=y
CONFIG_KALLSYMS_ALL=y
CONFIG_KPROBES=y
CONFIG_MEMCG=y
CONFIG_NAMESPACES=y
CONFIG_NET_ACT_BPF=y
CONFIG_NETDEVICES=y
CONFIG_NETFILTER_XT_MATCH_BPF=y
CONFIG_NET_L3_MASTER_DEV=y
CONFIG_NET_VRF=y
CONFIG_NET=y
CONFIG_NO_HZ_IDLE=y
CONFIG_NONPORTABLE=y
CONFIG_NR_CPUS=256
CONFIG_PACKET=y
CONFIG_PANIC_ON_OOPS=y
CONFIG_PARTITION_ADVANCED=y
CONFIG_PCI_HOST_GENERIC=y
CONFIG_PCI=y
CONFIG_POSIX_MQUEUE=y
CONFIG_PPC64=y
CONFIG_PPC_OF_BOOT_TRAMPOLINE=y
CONFIG_PPC_PSERIES=y
CONFIG_PPC_RADIX_MMU=y
CONFIG_PRINTK_TIME=y
CONFIG_PROC_KCORE=y
CONFIG_PROFILING=y
CONFIG_RCU_CPU_STALL_TIMEOUT=60
CONFIG_RT_GROUP_SCHED=y
CONFIG_SECTION_MISMATCH_WARN_ONLY=y
CONFIG_SECURITY_NETWORK=y
CONFIG_SERIAL_8250_CONSOLE=y
CONFIG_SERIAL_8250=y
CONFIG_SERIAL_OF_PLATFORM=y
CONFIG_SMP=y
CONFIG_SOC_VIRT=y
CONFIG_SYSVIPC=y
CONFIG_TCP_CONG_ADVANCED=y
CONFIG_THREAD_SHIFT=14
CONFIG_TLS=y
CONFIG_TMPFS_POSIX_ACL=y
CONFIG_TMPFS=y
CONFIG_TUN=y
CONFIG_UNIX=y
CONFIG_UPROBES=y
CONFIG_USER_NS=y
CONFIG_VETH=y
CONFIG_VLAN_8021Q=y
CONFIG_VSOCKETS_LOOPBACK=y
CONFIG_VSX=y
CONFIG_XFRM_USER=y

2
tools/testing/selftests/bpf/prog_tests/bloom_filter_map.c
View file

@ -13,7 +13,7 @@
static void test_fail_cases(void)
{
LIBBPF_OPTS(bpf_map_create_opts, opts);
__u32 value;
__u32 value = 0;
int fd, err;
/* Invalid key size */

50
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c
View file

@ -489,10 +489,28 @@ cleanup:
bpf_link__destroy(link);
}
static int verify_tracing_link_info(int fd, u64 cookie)
{
struct bpf_link_info info;
int err;
u32 len = sizeof(info);
err = bpf_link_get_info_by_fd(fd, &info, &len);
if (!ASSERT_OK(err, "get_link_info"))
return -1;
if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_TRACING, "link_type"))
return -1;
ASSERT_EQ(info.tracing.cookie, cookie, "tracing_cookie");
return 0;
}
static void tracing_subtest(struct test_bpf_cookie *skel)
{
__u64 cookie;
int prog_fd;
int prog_fd, err;
int fentry_fd = -1, fexit_fd = -1, fmod_ret_fd = -1;
LIBBPF_OPTS(bpf_test_run_opts, opts);
LIBBPF_OPTS(bpf_link_create_opts, link_opts);
@ -507,6 +525,10 @@ static void tracing_subtest(struct test_bpf_cookie *skel)
if (!ASSERT_GE(fentry_fd, 0, "fentry.link_create"))
goto cleanup;
err = verify_tracing_link_info(fentry_fd, cookie);
if (!ASSERT_OK(err, "verify_tracing_link_info"))
goto cleanup;
cookie = 0x20000000000000L;
prog_fd = bpf_program__fd(skel->progs.fexit_test1);
link_opts.tracing.cookie = cookie;
@ -635,10 +657,29 @@ cleanup:
bpf_link__destroy(link);
}
static int verify_raw_tp_link_info(int fd, u64 cookie)
{
struct bpf_link_info info;
int err;
u32 len = sizeof(info);
memset(&info, 0, sizeof(info));
err = bpf_link_get_info_by_fd(fd, &info, &len);
if (!ASSERT_OK(err, "get_link_info"))
return -1;
if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_RAW_TRACEPOINT, "link_type"))
return -1;
ASSERT_EQ(info.raw_tracepoint.cookie, cookie, "raw_tp_cookie");
return 0;
}
static void raw_tp_subtest(struct test_bpf_cookie *skel)
{
__u64 cookie;
int prog_fd, link_fd = -1;
int err, prog_fd, link_fd = -1;
struct bpf_link *link = NULL;
LIBBPF_OPTS(bpf_raw_tp_opts, raw_tp_opts);
LIBBPF_OPTS(bpf_raw_tracepoint_opts, opts);
@ -656,6 +697,11 @@ static void raw_tp_subtest(struct test_bpf_cookie *skel)
goto cleanup;
usleep(1); /* trigger */
err = verify_raw_tp_link_info(link_fd, cookie);
if (!ASSERT_OK(err, "verify_raw_tp_link_info"))
goto cleanup;
close(link_fd); /* detach */
link_fd = -1;

2
tools/testing/selftests/bpf/prog_tests/bpf_iter.c
View file

@ -323,7 +323,7 @@ static void test_task_pidfd(void)
static void test_task_sleepable(void)
{
struct bpf_iter_tasks *skel;
int pid, status, err, data_pipe[2], finish_pipe[2], c;
int pid, status, err, data_pipe[2], finish_pipe[2], c = 0;
char *test_data = NULL;
char *test_data_long = NULL;
char *data[2];

2
tools/testing/selftests/bpf/prog_tests/bpf_mod_race.c
View file

@ -78,7 +78,7 @@ static int test_setup_uffd(void *fault_addr)
}
uffd_register.range.start = (unsigned long)fault_addr;
uffd_register.range.len = 4096;
uffd_register.range.len = getpagesize();
uffd_register.mode = UFFDIO_REGISTER_MODE_MISSING;
if (ioctl(uffd, UFFDIO_REGISTER, &uffd_register)) {
close(uffd);

118
tools/testing/selftests/bpf/prog_tests/btf_dump.c
View file

@ -879,6 +879,122 @@ static void test_btf_dump_var_data(struct btf *btf, struct btf_dump *d,
"static int bpf_cgrp_storage_busy = (int)2", 2);
}
struct btf_dump_string_ctx {
struct btf *btf;
struct btf_dump *d;
char *str;
struct btf_dump_type_data_opts *opts;
int array_id;
};
static int btf_dump_one_string(struct btf_dump_string_ctx *ctx,
char *ptr, size_t ptr_sz,
const char *expected_val)
{
size_t type_sz;
int ret;
ctx->str[0] = '\0';
type_sz = btf__resolve_size(ctx->btf, ctx->array_id);
ret = btf_dump__dump_type_data(ctx->d, ctx->array_id, ptr, ptr_sz, ctx->opts);
if (type_sz <= ptr_sz) {
if (!ASSERT_EQ(ret, type_sz, "failed/unexpected type_sz"))
return -EINVAL;
}
if (!ASSERT_STREQ(ctx->str, expected_val, "ensure expected/actual match"))
return -EFAULT;
return 0;
}
static void btf_dump_strings(struct btf_dump_string_ctx *ctx)
{
struct btf_dump_type_data_opts *opts = ctx->opts;
opts->emit_strings = true;
opts->compact = true;
opts->emit_zeroes = false;
opts->skip_names = false;
btf_dump_one_string(ctx, "foo", 4, "(char[4])\"foo\"");
opts->skip_names = true;
btf_dump_one_string(ctx, "foo", 4, "\"foo\"");
/* This should have no effect. */
opts->emit_zeroes = false;
btf_dump_one_string(ctx, "foo", 4, "\"foo\"");
/* This should have no effect. */
opts->compact = false;
btf_dump_one_string(ctx, "foo", 4, "\"foo\"");
/* Non-printable characters come out as hex. */
btf_dump_one_string(ctx, "fo\xff", 4, "\"fo\\xff\"");
btf_dump_one_string(ctx, "fo\x7", 4, "\"fo\\x07\"");
/*
* Strings that are too long for the specified type ("char[4]")
* should fall back to the current behavior.
*/
opts->compact = true;
btf_dump_one_string(ctx, "abcde", 6, "['a','b','c','d',]");
/*
* Strings that are too short for the specified type ("char[4]")
* should work normally.
*/
btf_dump_one_string(ctx, "ab", 3, "\"ab\"");
/* Non-NUL-terminated arrays don't get printed as strings. */
char food[4] = { 'f', 'o', 'o', 'd' };
char bye[3] = { 'b', 'y', 'e' };
btf_dump_one_string(ctx, food, 4, "['f','o','o','d',]");
btf_dump_one_string(ctx, bye, 3, "['b','y','e',]");
/* The embedded NUL should terminate the string. */
char embed[4] = { 'f', 'o', '\0', 'd' };
btf_dump_one_string(ctx, embed, 4, "\"fo\"");
}
static void test_btf_dump_string_data(void)
{
struct test_ctx t = {};
char str[STRSIZE];
struct btf_dump *d;
DECLARE_LIBBPF_OPTS(btf_dump_type_data_opts, opts);
struct btf_dump_string_ctx ctx;
int char_id, int_id, array_id;
if (test_ctx__init(&t))
return;
d = btf_dump__new(t.btf, btf_dump_snprintf, str, NULL);
if (!ASSERT_OK_PTR(d, "could not create BTF dump"))
return;
/* Generate BTF for a four-element char array. */
char_id = btf__add_int(t.btf, "char", 1, BTF_INT_CHAR);
ASSERT_EQ(char_id, 1, "char_id");
int_id = btf__add_int(t.btf, "int", 4, BTF_INT_SIGNED);
ASSERT_EQ(int_id, 2, "int_id");
array_id = btf__add_array(t.btf, int_id, char_id, 4);
ASSERT_EQ(array_id, 3, "array_id");
ctx.btf = t.btf;
ctx.d = d;
ctx.str = str;
ctx.opts = &opts;
ctx.array_id = array_id;
btf_dump_strings(&ctx);
btf_dump__free(d);
test_ctx__free(&t);
}
static void test_btf_datasec(struct btf *btf, struct btf_dump *d, char *str,
const char *name, const char *expected_val,
void *data, size_t data_sz)
@ -970,6 +1086,8 @@ void test_btf_dump() {
test_btf_dump_struct_data(btf, d, str);
if (test__start_subtest("btf_dump: var_data"))
test_btf_dump_var_data(btf, d, str);
if (test__start_subtest("btf_dump: string_data"))
test_btf_dump_string_data();
btf_dump__free(d);
btf__free(btf);

617
tools/testing/selftests/bpf/prog_tests/cgroup_mprog_opts.c Normal file
View file

@ -0,0 +1,617 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
#include <test_progs.h>
#include "cgroup_helpers.h"
#include "cgroup_mprog.skel.h"
static void assert_mprog_count(int cg, int atype, int expected)
{
__u32 count = 0, attach_flags = 0;
int err;
err = bpf_prog_query(cg, atype, 0, &attach_flags,
NULL, &count);
ASSERT_EQ(count, expected, "count");
ASSERT_EQ(err, 0, "prog_query");
}
static void test_prog_attach_detach(int atype)
{
LIBBPF_OPTS(bpf_prog_attach_opts, opta);
LIBBPF_OPTS(bpf_prog_detach_opts, optd);
LIBBPF_OPTS(bpf_prog_query_opts, optq);
__u32 fd1, fd2, fd3, fd4, id1, id2, id3, id4;
struct cgroup_mprog *skel;
__u32 prog_ids[10];
int cg, err;
cg = test__join_cgroup("/prog_attach_detach");
if (!ASSERT_GE(cg, 0, "join_cgroup /prog_attach_detach"))
return;
skel = cgroup_mprog__open_and_load();
if (!ASSERT_OK_PTR(skel, "skel_load"))
goto cleanup;
fd1 = bpf_program__fd(skel->progs.getsockopt_1);
fd2 = bpf_program__fd(skel->progs.getsockopt_2);
fd3 = bpf_program__fd(skel->progs.getsockopt_3);
fd4 = bpf_program__fd(skel->progs.getsockopt_4);
id1 = id_from_prog_fd(fd1);
id2 = id_from_prog_fd(fd2);
id3 = id_from_prog_fd(fd3);
id4 = id_from_prog_fd(fd4);
assert_mprog_count(cg, atype, 0);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_ALLOW_MULTI | BPF_F_BEFORE | BPF_F_AFTER,
.expected_revision = 1,
);
/* ordering: [fd1] */
err = bpf_prog_attach_opts(fd1, cg, atype, &opta);
if (!ASSERT_EQ(err, 0, "prog_attach"))
goto cleanup;
assert_mprog_count(cg, atype, 1);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_ALLOW_MULTI | BPF_F_BEFORE,
.expected_revision = 2,
);
/* ordering: [fd2, fd1] */
err = bpf_prog_attach_opts(fd2, cg, atype, &opta);
if (!ASSERT_EQ(err, 0, "prog_attach"))
goto cleanup1;
assert_mprog_count(cg, atype, 2);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_ALLOW_MULTI | BPF_F_AFTER,
.relative_fd = fd2,
.expected_revision = 3,
);
/* ordering: [fd2, fd3, fd1] */
err = bpf_prog_attach_opts(fd3, cg, atype, &opta);
if (!ASSERT_EQ(err, 0, "prog_attach"))
goto cleanup2;
assert_mprog_count(cg, atype, 3);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_ALLOW_MULTI,
.expected_revision = 4,
);
/* ordering: [fd2, fd3, fd1, fd4] */
err = bpf_prog_attach_opts(fd4, cg, atype, &opta);
if (!ASSERT_EQ(err, 0, "prog_attach"))
goto cleanup3;
assert_mprog_count(cg, atype, 4);
/* retrieve optq.prog_cnt */
err = bpf_prog_query_opts(cg, atype, &optq);
if (!ASSERT_OK(err, "prog_query"))
goto cleanup4;
/* optq.prog_cnt will be used in below query */
memset(prog_ids, 0, sizeof(prog_ids));
optq.prog_ids = prog_ids;
err = bpf_prog_query_opts(cg, atype, &optq);
if (!ASSERT_OK(err, "prog_query"))
goto cleanup4;
ASSERT_EQ(optq.count, 4, "count");
ASSERT_EQ(optq.revision, 5, "revision");
ASSERT_EQ(optq.prog_ids[0], id2, "prog_ids[0]");
ASSERT_EQ(optq.prog_ids[1], id3, "prog_ids[1]");
ASSERT_EQ(optq.prog_ids[2], id1, "prog_ids[2]");
ASSERT_EQ(optq.prog_ids[3], id4, "prog_ids[3]");
ASSERT_EQ(optq.prog_ids[4], 0, "prog_ids[4]");
ASSERT_EQ(optq.link_ids, NULL, "link_ids");
cleanup4:
optd.expected_revision = 5;
err = bpf_prog_detach_opts(fd4, cg, atype, &optd);
ASSERT_OK(err, "prog_detach");
assert_mprog_count(cg, atype, 3);
cleanup3:
LIBBPF_OPTS_RESET(optd);
err = bpf_prog_detach_opts(fd3, cg, atype, &optd);
ASSERT_OK(err, "prog_detach");
assert_mprog_count(cg, atype, 2);
/* Check revision after two detach operations */
err = bpf_prog_query_opts(cg, atype, &optq);
ASSERT_OK(err, "prog_query");
ASSERT_EQ(optq.revision, 7, "revision");
cleanup2:
err = bpf_prog_detach_opts(fd2, cg, atype, &optd);
ASSERT_OK(err, "prog_detach");
assert_mprog_count(cg, atype, 1);
cleanup1:
err = bpf_prog_detach_opts(fd1, cg, atype, &optd);
ASSERT_OK(err, "prog_detach");
assert_mprog_count(cg, atype, 0);
cleanup:
cgroup_mprog__destroy(skel);
close(cg);
}
static void test_link_attach_detach(int atype)
{
LIBBPF_OPTS(bpf_cgroup_opts, opta);
LIBBPF_OPTS(bpf_cgroup_opts, optd);
LIBBPF_OPTS(bpf_prog_query_opts, optq);
struct bpf_link *link1, *link2, *link3, *link4;
__u32 fd1, fd2, fd3, fd4, id1, id2, id3, id4;
struct cgroup_mprog *skel;
__u32 prog_ids[10];
int cg, err;
cg = test__join_cgroup("/link_attach_detach");
if (!ASSERT_GE(cg, 0, "join_cgroup /link_attach_detach"))
return;
skel = cgroup_mprog__open_and_load();
if (!ASSERT_OK_PTR(skel, "skel_load"))
goto cleanup;
fd1 = bpf_program__fd(skel->progs.getsockopt_1);
fd2 = bpf_program__fd(skel->progs.getsockopt_2);
fd3 = bpf_program__fd(skel->progs.getsockopt_3);
fd4 = bpf_program__fd(skel->progs.getsockopt_4);
id1 = id_from_prog_fd(fd1);
id2 = id_from_prog_fd(fd2);
id3 = id_from_prog_fd(fd3);
id4 = id_from_prog_fd(fd4);
assert_mprog_count(cg, atype, 0);
LIBBPF_OPTS_RESET(opta,
.expected_revision = 1,
);
/* ordering: [fd1] */
link1 = bpf_program__attach_cgroup_opts(skel->progs.getsockopt_1, cg, &opta);
if (!ASSERT_OK_PTR(link1, "link_attach"))
goto cleanup;
assert_mprog_count(cg, atype, 1);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_BEFORE | BPF_F_LINK,
.relative_id = id_from_link_fd(bpf_link__fd(link1)),
.expected_revision = 2,
);
/* ordering: [fd2, fd1] */
link2 = bpf_program__attach_cgroup_opts(skel->progs.getsockopt_2, cg, &opta);
if (!ASSERT_OK_PTR(link2, "link_attach"))
goto cleanup1;
assert_mprog_count(cg, atype, 2);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_AFTER | BPF_F_LINK,
.relative_fd = bpf_link__fd(link2),
.expected_revision = 3,
);
/* ordering: [fd2, fd3, fd1] */
link3 = bpf_program__attach_cgroup_opts(skel->progs.getsockopt_3, cg, &opta);
if (!ASSERT_OK_PTR(link3, "link_attach"))
goto cleanup2;
assert_mprog_count(cg, atype, 3);
LIBBPF_OPTS_RESET(opta,
.expected_revision = 4,
);
/* ordering: [fd2, fd3, fd1, fd4] */
link4 = bpf_program__attach_cgroup_opts(skel->progs.getsockopt_4, cg, &opta);
if (!ASSERT_OK_PTR(link4, "link_attach"))
goto cleanup3;
assert_mprog_count(cg, atype, 4);
/* retrieve optq.prog_cnt */
err = bpf_prog_query_opts(cg, atype, &optq);
if (!ASSERT_OK(err, "prog_query"))
goto cleanup4;
/* optq.prog_cnt will be used in below query */
memset(prog_ids, 0, sizeof(prog_ids));
optq.prog_ids = prog_ids;
err = bpf_prog_query_opts(cg, atype, &optq);
if (!ASSERT_OK(err, "prog_query"))
goto cleanup4;
ASSERT_EQ(optq.count, 4, "count");
ASSERT_EQ(optq.revision, 5, "revision");
ASSERT_EQ(optq.prog_ids[0], id2, "prog_ids[0]");
ASSERT_EQ(optq.prog_ids[1], id3, "prog_ids[1]");
ASSERT_EQ(optq.prog_ids[2], id1, "prog_ids[2]");
ASSERT_EQ(optq.prog_ids[3], id4, "prog_ids[3]");
ASSERT_EQ(optq.prog_ids[4], 0, "prog_ids[4]");
ASSERT_EQ(optq.link_ids, NULL, "link_ids");
cleanup4:
bpf_link__destroy(link4);
assert_mprog_count(cg, atype, 3);
cleanup3:
bpf_link__destroy(link3);
assert_mprog_count(cg, atype, 2);
/* Check revision after two detach operations */
err = bpf_prog_query_opts(cg, atype, &optq);
ASSERT_OK(err, "prog_query");
ASSERT_EQ(optq.revision, 7, "revision");
cleanup2:
bpf_link__destroy(link2);
assert_mprog_count(cg, atype, 1);
cleanup1:
bpf_link__destroy(link1);
assert_mprog_count(cg, atype, 0);
cleanup:
cgroup_mprog__destroy(skel);
close(cg);
}
static void test_preorder_prog_attach_detach(int atype)
{
LIBBPF_OPTS(bpf_prog_attach_opts, opta);
LIBBPF_OPTS(bpf_prog_detach_opts, optd);
__u32 fd1, fd2, fd3, fd4;
struct cgroup_mprog *skel;
int cg, err;
cg = test__join_cgroup("/preorder_prog_attach_detach");
if (!ASSERT_GE(cg, 0, "join_cgroup /preorder_prog_attach_detach"))
return;
skel = cgroup_mprog__open_and_load();
if (!ASSERT_OK_PTR(skel, "skel_load"))
goto cleanup;
fd1 = bpf_program__fd(skel->progs.getsockopt_1);
fd2 = bpf_program__fd(skel->progs.getsockopt_2);
fd3 = bpf_program__fd(skel->progs.getsockopt_3);
fd4 = bpf_program__fd(skel->progs.getsockopt_4);
assert_mprog_count(cg, atype, 0);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_ALLOW_MULTI,
.expected_revision = 1,
);
/* ordering: [fd1] */
err = bpf_prog_attach_opts(fd1, cg, atype, &opta);
if (!ASSERT_EQ(err, 0, "prog_attach"))
goto cleanup;
assert_mprog_count(cg, atype, 1);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_ALLOW_MULTI | BPF_F_PREORDER,
.expected_revision = 2,
);
/* ordering: [fd1, fd2] */
err = bpf_prog_attach_opts(fd2, cg, atype, &opta);
if (!ASSERT_EQ(err, 0, "prog_attach"))
goto cleanup1;
assert_mprog_count(cg, atype, 2);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_ALLOW_MULTI | BPF_F_AFTER,
.relative_fd = fd2,
.expected_revision = 3,
);
err = bpf_prog_attach_opts(fd3, cg, atype, &opta);
if (!ASSERT_EQ(err, -EINVAL, "prog_attach"))
goto cleanup2;
assert_mprog_count(cg, atype, 2);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_ALLOW_MULTI | BPF_F_AFTER | BPF_F_PREORDER,
.relative_fd = fd2,
.expected_revision = 3,
);
/* ordering: [fd1, fd2, fd3] */
err = bpf_prog_attach_opts(fd3, cg, atype, &opta);
if (!ASSERT_EQ(err, 0, "prog_attach"))
goto cleanup2;
assert_mprog_count(cg, atype, 3);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_ALLOW_MULTI,
.expected_revision = 4,
);
/* ordering: [fd2, fd3, fd1, fd4] */
err = bpf_prog_attach_opts(fd4, cg, atype, &opta);
if (!ASSERT_EQ(err, 0, "prog_attach"))
goto cleanup3;
assert_mprog_count(cg, atype, 4);
err = bpf_prog_detach_opts(fd4, cg, atype, &optd);
ASSERT_OK(err, "prog_detach");
assert_mprog_count(cg, atype, 3);
cleanup3:
err = bpf_prog_detach_opts(fd3, cg, atype, &optd);
ASSERT_OK(err, "prog_detach");
assert_mprog_count(cg, atype, 2);
cleanup2:
err = bpf_prog_detach_opts(fd2, cg, atype, &optd);
ASSERT_OK(err, "prog_detach");
assert_mprog_count(cg, atype, 1);
cleanup1:
err = bpf_prog_detach_opts(fd1, cg, atype, &optd);
ASSERT_OK(err, "prog_detach");
assert_mprog_count(cg, atype, 0);
cleanup:
cgroup_mprog__destroy(skel);
close(cg);
}
static void test_preorder_link_attach_detach(int atype)
{
LIBBPF_OPTS(bpf_cgroup_opts, opta);
struct bpf_link *link1, *link2, *link3, *link4;
struct cgroup_mprog *skel;
__u32 fd2;
int cg;
cg = test__join_cgroup("/preorder_link_attach_detach");
if (!ASSERT_GE(cg, 0, "join_cgroup /preorder_link_attach_detach"))
return;
skel = cgroup_mprog__open_and_load();
if (!ASSERT_OK_PTR(skel, "skel_load"))
goto cleanup;
fd2 = bpf_program__fd(skel->progs.getsockopt_2);
assert_mprog_count(cg, atype, 0);
LIBBPF_OPTS_RESET(opta,
.expected_revision = 1,
);
/* ordering: [fd1] */
link1 = bpf_program__attach_cgroup_opts(skel->progs.getsockopt_1, cg, &opta);
if (!ASSERT_OK_PTR(link1, "link_attach"))
goto cleanup;
assert_mprog_count(cg, atype, 1);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_PREORDER,
.expected_revision = 2,
);
/* ordering: [fd1, fd2] */
link2 = bpf_program__attach_cgroup_opts(skel->progs.getsockopt_2, cg, &opta);
if (!ASSERT_OK_PTR(link2, "link_attach"))
goto cleanup1;
assert_mprog_count(cg, atype, 2);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_AFTER,
.relative_fd = fd2,
.expected_revision = 3,
);
link3 = bpf_program__attach_cgroup_opts(skel->progs.getsockopt_3, cg, &opta);
if (!ASSERT_ERR_PTR(link3, "link_attach"))
goto cleanup2;
assert_mprog_count(cg, atype, 2);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_AFTER | BPF_F_PREORDER | BPF_F_LINK,
.relative_fd = bpf_link__fd(link2),
.expected_revision = 3,
);
/* ordering: [fd1, fd2, fd3] */
link3 = bpf_program__attach_cgroup_opts(skel->progs.getsockopt_3, cg, &opta);
if (!ASSERT_OK_PTR(link3, "link_attach"))
goto cleanup2;
assert_mprog_count(cg, atype, 3);
LIBBPF_OPTS_RESET(opta,
.expected_revision = 4,
);
/* ordering: [fd2, fd3, fd1, fd4] */
link4 = bpf_program__attach_cgroup_opts(skel->progs.getsockopt_4, cg, &opta);
if (!ASSERT_OK_PTR(link4, "prog_attach"))
goto cleanup3;
assert_mprog_count(cg, atype, 4);
bpf_link__destroy(link4);
assert_mprog_count(cg, atype, 3);
cleanup3:
bpf_link__destroy(link3);
assert_mprog_count(cg, atype, 2);
cleanup2:
bpf_link__destroy(link2);
assert_mprog_count(cg, atype, 1);
cleanup1:
bpf_link__destroy(link1);
assert_mprog_count(cg, atype, 0);
cleanup:
cgroup_mprog__destroy(skel);
close(cg);
}
static void test_invalid_attach_detach(int atype)
{
LIBBPF_OPTS(bpf_prog_attach_opts, opta);
__u32 fd1, fd2, id2;
struct cgroup_mprog *skel;
int cg, err;
cg = test__join_cgroup("/invalid_attach_detach");
if (!ASSERT_GE(cg, 0, "join_cgroup /invalid_attach_detach"))
return;
skel = cgroup_mprog__open_and_load();
if (!ASSERT_OK_PTR(skel, "skel_load"))
goto cleanup;
fd1 = bpf_program__fd(skel->progs.getsockopt_1);
fd2 = bpf_program__fd(skel->progs.getsockopt_2);
id2 = id_from_prog_fd(fd2);
assert_mprog_count(cg, atype, 0);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_ALLOW_MULTI | BPF_F_BEFORE | BPF_F_AFTER,
.relative_id = id2,
);
err = bpf_prog_attach_opts(fd1, cg, atype, &opta);
ASSERT_EQ(err, -EINVAL, "prog_attach");
assert_mprog_count(cg, atype, 0);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_ALLOW_MULTI | BPF_F_BEFORE | BPF_F_ID,
);
err = bpf_prog_attach_opts(fd1, cg, atype, &opta);
ASSERT_EQ(err, -ENOENT, "prog_attach");
assert_mprog_count(cg, atype, 0);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_ALLOW_MULTI | BPF_F_AFTER | BPF_F_ID,
);
err = bpf_prog_attach_opts(fd1, cg, atype, &opta);
ASSERT_EQ(err, -ENOENT, "prog_attach");
assert_mprog_count(cg, atype, 0);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_ALLOW_MULTI | BPF_F_BEFORE | BPF_F_AFTER,
.relative_id = id2,
);
err = bpf_prog_attach_opts(fd1, cg, atype, &opta);
ASSERT_EQ(err, -EINVAL, "prog_attach");
assert_mprog_count(cg, atype, 0);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_ALLOW_MULTI | BPF_F_LINK,
.relative_id = id2,
);
err = bpf_prog_attach_opts(fd1, cg, atype, &opta);
ASSERT_EQ(err, -EINVAL, "prog_attach");
assert_mprog_count(cg, atype, 0);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_ALLOW_MULTI,
.relative_id = id2,
);
err = bpf_prog_attach_opts(fd1, cg, atype, &opta);
ASSERT_EQ(err, -EINVAL, "prog_attach");
assert_mprog_count(cg, atype, 0);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_ALLOW_MULTI | BPF_F_BEFORE,
.relative_fd = fd1,
);
err = bpf_prog_attach_opts(fd1, cg, atype, &opta);
ASSERT_EQ(err, -ENOENT, "prog_attach");
assert_mprog_count(cg, atype, 0);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_ALLOW_MULTI | BPF_F_AFTER,
.relative_fd = fd1,
);
err = bpf_prog_attach_opts(fd1, cg, atype, &opta);
ASSERT_EQ(err, -ENOENT, "prog_attach");
assert_mprog_count(cg, atype, 0);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_ALLOW_MULTI,
);
err = bpf_prog_attach_opts(fd1, cg, atype, &opta);
if (!ASSERT_EQ(err, 0, "prog_attach"))
goto cleanup;
assert_mprog_count(cg, atype, 1);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_ALLOW_MULTI | BPF_F_AFTER,
);
err = bpf_prog_attach_opts(fd1, cg, atype, &opta);
ASSERT_EQ(err, -EINVAL, "prog_attach");
assert_mprog_count(cg, atype, 1);
LIBBPF_OPTS_RESET(opta,
.flags = BPF_F_ALLOW_MULTI | BPF_F_REPLACE | BPF_F_AFTER,
.replace_prog_fd = fd1,
);
err = bpf_prog_attach_opts(fd1, cg, atype, &opta);
ASSERT_EQ(err, -EINVAL, "prog_attach");
assert_mprog_count(cg, atype, 1);
cleanup:
cgroup_mprog__destroy(skel);
close(cg);
}
void test_cgroup_mprog_opts(void)
{
if (test__start_subtest("prog_attach_detach"))
test_prog_attach_detach(BPF_CGROUP_GETSOCKOPT);
if (test__start_subtest("link_attach_detach"))
test_link_attach_detach(BPF_CGROUP_GETSOCKOPT);
if (test__start_subtest("preorder_prog_attach_detach"))
test_preorder_prog_attach_detach(BPF_CGROUP_GETSOCKOPT);
if (test__start_subtest("preorder_link_attach_detach"))
test_preorder_link_attach_detach(BPF_CGROUP_GETSOCKOPT);
if (test__start_subtest("invalid_attach_detach"))
test_invalid_attach_detach(BPF_CGROUP_GETSOCKOPT);
}

77
tools/testing/selftests/bpf/prog_tests/cgroup_mprog_ordering.c Normal file
View file

@ -0,0 +1,77 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
#include <test_progs.h>
#include "cgroup_helpers.h"
#include "cgroup_preorder.skel.h"
static int run_getsockopt_test(int cg_parent, int sock_fd, bool has_relative_fd)
{
LIBBPF_OPTS(bpf_prog_attach_opts, opts);
enum bpf_attach_type prog_p_atype, prog_p2_atype;
int prog_p_fd, prog_p2_fd;
struct cgroup_preorder *skel = NULL;
struct bpf_program *prog;
__u8 *result, buf;
socklen_t optlen = 1;
int err = 0;
skel = cgroup_preorder__open_and_load();
if (!ASSERT_OK_PTR(skel, "cgroup_preorder__open_and_load"))
return 0;
LIBBPF_OPTS_RESET(opts);
opts.flags = BPF_F_ALLOW_MULTI;
prog = skel->progs.parent;
prog_p_fd = bpf_program__fd(prog);
prog_p_atype = bpf_program__expected_attach_type(prog);
err = bpf_prog_attach_opts(prog_p_fd, cg_parent, prog_p_atype, &opts);
if (!ASSERT_OK(err, "bpf_prog_attach_opts-parent"))
goto close_skel;
opts.flags = BPF_F_ALLOW_MULTI | BPF_F_BEFORE;
if (has_relative_fd)
opts.relative_fd = prog_p_fd;
prog = skel->progs.parent_2;
prog_p2_fd = bpf_program__fd(prog);
prog_p2_atype = bpf_program__expected_attach_type(prog);
err = bpf_prog_attach_opts(prog_p2_fd, cg_parent, prog_p2_atype, &opts);
if (!ASSERT_OK(err, "bpf_prog_attach_opts-parent_2"))
goto detach_parent;
err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen);
if (!ASSERT_OK(err, "getsockopt"))
goto detach_parent_2;
result = skel->bss->result;
ASSERT_TRUE(result[0] == 4 && result[1] == 3, "result values");
detach_parent_2:
ASSERT_OK(bpf_prog_detach2(prog_p2_fd, cg_parent, prog_p2_atype),
"bpf_prog_detach2-parent_2");
detach_parent:
ASSERT_OK(bpf_prog_detach2(prog_p_fd, cg_parent, prog_p_atype),
"bpf_prog_detach2-parent");
close_skel:
cgroup_preorder__destroy(skel);
return err;
}
void test_cgroup_mprog_ordering(void)
{
int cg_parent = -1, sock_fd = -1;
cg_parent = test__join_cgroup("/parent");
if (!ASSERT_GE(cg_parent, 0, "join_cgroup /parent"))
goto out;
sock_fd = socket(AF_INET, SOCK_STREAM, 0);
if (!ASSERT_GE(sock_fd, 0, "socket"))
goto out;
ASSERT_OK(run_getsockopt_test(cg_parent, sock_fd, false), "getsockopt_test_1");
ASSERT_OK(run_getsockopt_test(cg_parent, sock_fd, true), "getsockopt_test_2");
out:
close(sock_fd);
close(cg_parent);
}

117
tools/testing/selftests/bpf/prog_tests/cgroup_xattr.c
View file

@ -7,133 +7,60 @@
#include <string.h>
#include <unistd.h>
#include <sys/socket.h>
#include <sys/xattr.h>
#include <test_progs.h>
#include "cgroup_helpers.h"
#include "read_cgroupfs_xattr.skel.h"
#include "cgroup_read_xattr.skel.h"
#define CGROUP_FS_ROOT "/sys/fs/cgroup/"
#define CGROUP_FS_PARENT CGROUP_FS_ROOT "foo/"
#define CGROUP_FS_PARENT "foo/"
#define CGROUP_FS_CHILD CGROUP_FS_PARENT "bar/"
static int move_pid_to_cgroup(const char *cgroup_folder, pid_t pid)
{
char filename[128];
char pid_str[64];
int procs_fd;
int ret;
snprintf(filename, sizeof(filename), "%scgroup.procs", cgroup_folder);
snprintf(pid_str, sizeof(pid_str), "%d", pid);
procs_fd = open(filename, O_WRONLY | O_APPEND);
if (!ASSERT_OK_FD(procs_fd, "open"))
return -1;
ret = write(procs_fd, pid_str, strlen(pid_str));
close(procs_fd);
if (!ASSERT_GT(ret, 0, "write cgroup.procs"))
return -1;
return 0;
}
static void reset_cgroups_and_lo(void)
{
rmdir(CGROUP_FS_CHILD);
rmdir(CGROUP_FS_PARENT);
system("ip addr del 1.1.1.1/32 dev lo");
system("ip link set dev lo down");
}
#define TMP_FILE "/tmp/selftests_cgroup_xattr"
static const char xattr_value_a[] = "bpf_selftest_value_a";
static const char xattr_value_b[] = "bpf_selftest_value_b";
static const char xattr_name[] = "user.bpf_test";
static int setup_cgroups_and_lo(void)
{
int err;
err = mkdir(CGROUP_FS_PARENT, 0755);
if (!ASSERT_OK(err, "mkdir 1"))
goto error;
err = mkdir(CGROUP_FS_CHILD, 0755);
if (!ASSERT_OK(err, "mkdir 2"))
goto error;
err = setxattr(CGROUP_FS_PARENT, xattr_name, xattr_value_a,
strlen(xattr_value_a) + 1, 0);
if (!ASSERT_OK(err, "setxattr 1"))
goto error;
err = setxattr(CGROUP_FS_CHILD, xattr_name, xattr_value_b,
strlen(xattr_value_b) + 1, 0);
if (!ASSERT_OK(err, "setxattr 2"))
goto error;
err = system("ip link set dev lo up");
if (!ASSERT_OK(err, "lo up"))
goto error;
err = system("ip addr add 1.1.1.1 dev lo");
if (!ASSERT_OK(err, "lo addr v4"))
goto error;
err = write_sysctl("/proc/sys/net/ipv4/ping_group_range", "0 0");
if (!ASSERT_OK(err, "write_sysctl"))
goto error;
return 0;
error:
reset_cgroups_and_lo();
return err;
}
static void test_read_cgroup_xattr(void)
{
struct sockaddr_in sa4 = {
.sin_family = AF_INET,
.sin_addr.s_addr = htonl(INADDR_LOOPBACK),
};
int tmp_fd, parent_cgroup_fd = -1, child_cgroup_fd = -1;
struct read_cgroupfs_xattr *skel = NULL;
pid_t pid = gettid();
int sock_fd = -1;
int connect_fd = -1;
if (!ASSERT_OK(setup_cgroups_and_lo(), "setup_cgroups_and_lo"))
parent_cgroup_fd = test__join_cgroup(CGROUP_FS_PARENT);
if (!ASSERT_OK_FD(parent_cgroup_fd, "create parent cgroup"))
return;
if (!ASSERT_OK(move_pid_to_cgroup(CGROUP_FS_CHILD, pid),
"move_pid_to_cgroup"))
if (!ASSERT_OK(set_cgroup_xattr(CGROUP_FS_PARENT, xattr_name, xattr_value_a),
"set parent xattr"))
goto out;
child_cgroup_fd = test__join_cgroup(CGROUP_FS_CHILD);
if (!ASSERT_OK_FD(child_cgroup_fd, "create child cgroup"))
goto out;
if (!ASSERT_OK(set_cgroup_xattr(CGROUP_FS_CHILD, xattr_name, xattr_value_b),
"set child xattr"))
goto out;
skel = read_cgroupfs_xattr__open_and_load();
if (!ASSERT_OK_PTR(skel, "read_cgroupfs_xattr__open_and_load"))
goto out;
skel->bss->target_pid = pid;
skel->bss->target_pid = gettid();
if (!ASSERT_OK(read_cgroupfs_xattr__attach(skel), "read_cgroupfs_xattr__attach"))
goto out;
sock_fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_ICMP);
if (!ASSERT_OK_FD(sock_fd, "sock create"))
goto out;
connect_fd = connect(sock_fd, &sa4, sizeof(sa4));
if (!ASSERT_OK_FD(connect_fd, "connect 1"))
goto out;
close(connect_fd);
tmp_fd = open(TMP_FILE, O_RDONLY | O_CREAT);
ASSERT_OK_FD(tmp_fd, "open tmp file");
close(tmp_fd);
ASSERT_TRUE(skel->bss->found_value_a, "found_value_a");
ASSERT_TRUE(skel->bss->found_value_b, "found_value_b");
out:
close(connect_fd);
close(sock_fd);
close(child_cgroup_fd);
close(parent_cgroup_fd);
read_cgroupfs_xattr__destroy(skel);
move_pid_to_cgroup(CGROUP_FS_ROOT, pid);
reset_cgroups_and_lo();
unlink(TMP_FILE);
}
void test_cgroup_xattr(void)

Some files were not shown because too many files have changed in this diff Show more