archive/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2025-08-15 14:11:42 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

LoongArch changes for v6.17

Browse source 
1, Complete KSave registers definition;
 2, Support the mem=<size> kernel parameter;
 3, Support BPF dynamic modification & trampoline;
 4, Add MMC/SDIO controller nodes in dts;
 5, Some bug fixes and other small changes.
 -----BEGIN PGP SIGNATURE-----
 
 iQJJBAABCAA0FiEEzOlt8mkP+tbeiYy5AoYrw/LiJnoFAmiR8aUWHGNoZW5odWFj
 YWlAa2VybmVsLm9yZwAKCRAChivD8uImesNJD/jFKBJ05ryQgz2mKERx6THyAvGz
 bWf1Fx3EcfJ0J/3cd8yIhP316dy6L+FElUNPJixz9OuWk/3LBDNzQb9F/QbBkNtg
 Ho2jaE3q90OXiZ/hwNmK2aRYP5Pi2YcAmIWpz65opj8SRTYq5i0mp+El7LzvQ+JO
 Y2Ghp13S5cQC7boL0iJf0jhYcf0PW7kpzIJKbZhHYT3vWsGA+gRBufanAuBPUo07
 zypaXcm54fni5n6ucOElTF8WZVoouFOeF7KW0tkL12ufkVyMnRfnu+tJgG9ohtjK
 EtrisF3scgC/pg9cTWC706C85FjNz3/+mi0AbQpH2NjJczmoeQwQfnAYxbeIwCJD
 C6wIY1V5IBVDCU5dTCqEOMvfaCjl0AzvZxn3NUh+OUyf3y9pENpnI1JHkyj4z0A4
 qDsFI+4W+XpvF/EAeDGCU/TlYEmEqEDZ+Y/eQkLT4s4pGIWjvX3vMLyugyE0nTR8
 y6BbyvJXmG5As/TQEGmvNaBnpDgCsxJWRtnuiUS9eAqhwGFpCdO5aZ3R7Pumke4l
 Meyn9TIUEsNZ20Fdvjo+Asa8FTtkV4OLkMPSXGb3ag8sNtnJSRl0PpXYZa7CZKa6
 6I4kmO8130TKOqwdxAlEdU9JnznFEMxEkoNdOuQKEHnGPUQbCVf9Zz5gg+vM1vxN
 Vhlmdjm3bSFcKNAY
 =SVHt
 -----END PGP SIGNATURE-----

Merge tag 'loongarch-6.17' of git://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson

Pull LoongArch updates from Huacai Chen:

 - Complete KSave registers definition

 - Support the mem=<size> kernel parameter

 - Support BPF dynamic modification & trampoline

 - Add MMC/SDIO controller nodes in dts

 - Some bug fixes and other small changes

* tag 'loongarch-6.17' of git://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson:
  LoongArch: vDSO: Remove -nostdlib complier flag
  LoongArch: dts: Add eMMC/SDIO controller support to Loongson-2K2000
  LoongArch: dts: Add SDIO controller support to Loongson-2K1000
  LoongArch: dts: Add SDIO controller support to Loongson-2K0500
  LoongArch: BPF: Set bpf_jit_bypass_spec_v1/v4()
  LoongArch: BPF: Fix the tailcall hierarchy
  LoongArch: BPF: Fix jump offset calculation in tailcall
  LoongArch: BPF: Add struct ops support for trampoline
  LoongArch: BPF: Add basic bpf trampoline support
  LoongArch: BPF: Add dynamic code modification support
  LoongArch: BPF: Rename and refactor validate_code()
  LoongArch: Add larch_insn_gen_{beq,bne} helpers
  LoongArch: Don't use %pK through printk() in unwinder
  LoongArch: Avoid in-place string operation on FDT content
  LoongArch: Support mem=<size> kernel parameter
  LoongArch: Make relocate_new_kernel_size be a .quad value
  LoongArch: Complete KSave registers definition
This commit is contained in:
Linus Torvalds 2025-08-08 06:36:48 +03:00
commit 83affacd18
16 changed files with 841 additions and 89 deletions
Download patch file Download diff file Expand all files Collapse all files

9
arch/loongarch/boot/dts/loongson-2k0500-ref.dts
View file

@ -41,6 +41,15 @@
};
};
&apbdma3 {
status = "okay";
};
&mmc0 {
status = "okay";
bus-width = <4>;
};
&gmac0 {
status = "okay";

28
arch/loongarch/boot/dts/loongson-2k0500.dtsi
View file

@ -104,7 +104,7 @@
status = "disabled";
};
dma-controller@1fe10c20 {
apbdma2: dma-controller@1fe10c20 {
compatible = "loongson,ls2k0500-apbdma", "loongson,ls2k1000-apbdma";
reg = <0 0x1fe10c20 0 0x8>;
interrupt-parent = <&eiointc>;
@ -114,7 +114,7 @@
status = "disabled";
};
dma-controller@1fe10c30 {
apbdma3: dma-controller@1fe10c30 {
compatible = "loongson,ls2k0500-apbdma", "loongson,ls2k1000-apbdma";
reg = <0 0x1fe10c30 0 0x8>;
interrupt-parent = <&eiointc>;
@ -437,6 +437,30 @@
status = "disabled";
};
mmc0: mmc@1ff64000 {
compatible = "loongson,ls2k0500-mmc";
reg = <0 0x1ff64000 0 0x2000>,
<0 0x1fe10100 0 0x4>;
interrupt-parent = <&eiointc>;
interrupts = <57>;
dmas = <&apbdma3 0>;
dma-names = "rx-tx";
clocks = <&clk LOONGSON2_APB_CLK>;
status = "disabled";
};
mmc@1ff66000 {
compatible = "loongson,ls2k0500-mmc";
reg = <0 0x1ff66000 0 0x2000>,
<0 0x1fe10100 0 0x4>;
interrupt-parent = <&eiointc>;
interrupts = <58>;
dmas = <&apbdma2 0>;
dma-names = "rx-tx";
clocks = <&clk LOONGSON2_APB_CLK>;
status = "disabled";
};
pmc: power-management@1ff6c000 {
compatible = "loongson,ls2k0500-pmc", "syscon";
reg = <0x0 0x1ff6c000 0x0 0x58>;

13
arch/loongarch/boot/dts/loongson-2k1000-ref.dts
View file

@ -48,6 +48,19 @@
};
};
&apbdma1 {
status = "okay";
};
&mmc {
status = "okay";
pinctrl-0 = <&sdio_pins_default>;
pinctrl-names = "default";
bus-width = <4>;
cd-gpios = <&gpio0 22 GPIO_ACTIVE_LOW>;
};
&gmac0 {
status = "okay";

24
arch/loongarch/boot/dts/loongson-2k1000.dtsi
View file

@ -187,14 +187,14 @@
<26 IRQ_TYPE_LEVEL_HIGH>,
<26 IRQ_TYPE_LEVEL_HIGH>,
<26 IRQ_TYPE_LEVEL_HIGH>,
<>,
<26 IRQ_TYPE_LEVEL_HIGH>,
<0 IRQ_TYPE_NONE>,
<26 IRQ_TYPE_LEVEL_HIGH>,
<26 IRQ_TYPE_LEVEL_HIGH>,
<26 IRQ_TYPE_LEVEL_HIGH>,
<26 IRQ_TYPE_LEVEL_HIGH>,
<26 IRQ_TYPE_LEVEL_HIGH>,
<26 IRQ_TYPE_LEVEL_HIGH>,
<26 IRQ_TYPE_NONE>,
<26 IRQ_TYPE_LEVEL_HIGH>,
<26 IRQ_TYPE_LEVEL_HIGH>,
<26 IRQ_TYPE_LEVEL_HIGH>,
@ -209,13 +209,13 @@
<27 IRQ_TYPE_LEVEL_HIGH>,
<27 IRQ_TYPE_LEVEL_HIGH>,
<27 IRQ_TYPE_LEVEL_HIGH>,
<>,
<0 IRQ_TYPE_NONE>,
<27 IRQ_TYPE_LEVEL_HIGH>,
<27 IRQ_TYPE_LEVEL_HIGH>,
<27 IRQ_TYPE_LEVEL_HIGH>,
<27 IRQ_TYPE_LEVEL_HIGH>,
<>,
<>,
<0 IRQ_TYPE_NONE>,
<0 IRQ_TYPE_NONE>,
<27 IRQ_TYPE_LEVEL_HIGH>,
<27 IRQ_TYPE_LEVEL_HIGH>,
<27 IRQ_TYPE_LEVEL_HIGH>,
@ -256,7 +256,7 @@
status = "disabled";
};
dma-controller@1fe00c10 {
apbdma1: dma-controller@1fe00c10 {
compatible = "loongson,ls2k1000-apbdma";
reg = <0x0 0x1fe00c10 0x0 0x8>;
interrupt-parent = <&liointc1>;
@ -405,6 +405,18 @@
status = "disabled";
};
mmc: mmc@1fe2c000 {
compatible = "loongson,ls2k1000-mmc";
reg = <0 0x1fe2c000 0 0x68>,
<0 0x1fe00438 0 0x8>;
interrupt-parent = <&liointc0>;
interrupts = <31 IRQ_TYPE_LEVEL_HIGH>;
clocks = <&clk LOONGSON2_APB_CLK>;
dmas = <&apbdma1 0>;
dma-names = "rx-tx";
status = "disabled";
};
spi0: spi@1fff0220 {
compatible = "loongson,ls2k1000-spi";
reg = <0x0 0x1fff0220 0x0 0x10>;

10
arch/loongarch/boot/dts/loongson-2k2000-ref.dts
View file

@ -39,6 +39,16 @@
};
};
&emmc {
status = "okay";
bus-width = <8>;
cap-mmc-highspeed;
mmc-hs200-1_8v;
no-sd;
no-sdio;
};
&sata {
status = "okay";
};

18
arch/loongarch/boot/dts/loongson-2k2000.dtsi
View file

@ -259,6 +259,24 @@
status = "disabled";
};
emmc: mmc@79990000 {
compatible = "loongson,ls2k2000-mmc";
reg = <0x0 0x79990000 0x0 0x1000>;
interrupt-parent = <&pic>;
interrupts = <51 IRQ_TYPE_LEVEL_HIGH>;
clocks = <&clk LOONGSON2_EMMC_CLK>;
status = "disabled";
};
mmc@79991000 {
compatible = "loongson,ls2k2000-mmc";
reg = <0x0 0x79991000 0x0 0x1000>;
interrupt-parent = <&pic>;
interrupts = <50 IRQ_TYPE_LEVEL_HIGH>;
clocks = <&clk LOONGSON2_EMMC_CLK>;
status = "disabled";
};
pcie@1a000000 {
compatible = "loongson,ls2k-pci";
reg = <0x0 0x1a000000 0x0 0x02000000>,

3
arch/loongarch/include/asm/inst.h
View file

@ -497,6 +497,7 @@ void arch_simulate_insn(union loongarch_instruction insn, struct pt_regs *regs);
int larch_insn_read(void *addr, u32 *insnp);
int larch_insn_write(void *addr, u32 insn);
int larch_insn_patch_text(void *addr, u32 insn);
int larch_insn_text_copy(void *dst, void *src, size_t len);
u32 larch_insn_gen_nop(void);
u32 larch_insn_gen_b(unsigned long pc, unsigned long dest);
@ -510,6 +511,8 @@ u32 larch_insn_gen_move(enum loongarch_gpr rd, enum loongarch_gpr rj);
u32 larch_insn_gen_lu12iw(enum loongarch_gpr rd, int imm);
u32 larch_insn_gen_lu32id(enum loongarch_gpr rd, int imm);
u32 larch_insn_gen_lu52id(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm);
u32 larch_insn_gen_beq(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm);
u32 larch_insn_gen_bne(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm);
u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm);
static inline bool signed_imm_check(long val, unsigned int bit)

7
arch/loongarch/include/asm/loongarch.h
View file

@ -451,6 +451,13 @@
#define LOONGARCH_CSR_KS6 0x36
#define LOONGARCH_CSR_KS7 0x37
#define LOONGARCH_CSR_KS8 0x38
#define LOONGARCH_CSR_KS9 0x39
#define LOONGARCH_CSR_KS10 0x3a
#define LOONGARCH_CSR_KS11 0x3b
#define LOONGARCH_CSR_KS12 0x3c
#define LOONGARCH_CSR_KS13 0x3d
#define LOONGARCH_CSR_KS14 0x3e
#define LOONGARCH_CSR_KS15 0x3f
/* Exception allocated KS0, KS1 and KS2 statically */
#define EXCEPTION_KS0 LOONGARCH_CSR_KS0

13
arch/loongarch/kernel/env.c
View file

@ -39,16 +39,19 @@ void __init init_environ(void)
static int __init init_cpu_fullname(void)
{
struct device_node *root;
int cpu, ret;
char *model;
char *cpuname;
const char *model;
struct device_node *root;
/* Parsing cpuname from DTS model property */
root = of_find_node_by_path("/");
ret = of_property_read_string(root, "model", (const char **)&model);
ret = of_property_read_string(root, "model", &model);
if (ret == 0) {
cpuname = kstrdup(model, GFP_KERNEL);
loongson_sysconf.cpuname = strsep(&cpuname, " ");
}
of_node_put(root);
if (ret == 0)
loongson_sysconf.cpuname = strsep(&model, " ");
if (loongson_sysconf.cpuname && !strncmp(loongson_sysconf.cpuname, "Loongson", 8)) {
for (cpu = 0; cpu < NR_CPUS; cpu++)

74
arch/loongarch/kernel/inst.c
View file

@ -4,6 +4,8 @@
*/
#include <linux/sizes.h>
#include <linux/uaccess.h>
#include <linux/set_memory.h>
#include <linux/stop_machine.h>
#include <asm/cacheflush.h>
#include <asm/inst.h>
@ -218,6 +220,50 @@ int larch_insn_patch_text(void *addr, u32 insn)
return ret;
}
struct insn_copy {
void *dst;
void *src;
size_t len;
unsigned int cpu;
};
static int text_copy_cb(void *data)
{
int ret = 0;
struct insn_copy *copy = data;
if (smp_processor_id() == copy->cpu) {
ret = copy_to_kernel_nofault(copy->dst, copy->src, copy->len);
if (ret)
pr_err("%s: operation failed\n", __func__);
}
flush_icache_range((unsigned long)copy->dst, (unsigned long)copy->dst + copy->len);
return ret;
}
int larch_insn_text_copy(void *dst, void *src, size_t len)
{
int ret = 0;
size_t start, end;
struct insn_copy copy = {
.dst = dst,
.src = src,
.len = len,
.cpu = smp_processor_id(),
};
start = round_down((size_t)dst, PAGE_SIZE);
end = round_up((size_t)dst + len, PAGE_SIZE);
set_memory_rw(start, (end - start) / PAGE_SIZE);
ret = stop_machine(text_copy_cb, &copy, cpu_online_mask);
set_memory_rox(start, (end - start) / PAGE_SIZE);
return ret;
}
u32 larch_insn_gen_nop(void)
{
return INSN_NOP;
@ -323,6 +369,34 @@ u32 larch_insn_gen_lu52id(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
return insn.word;
}
u32 larch_insn_gen_beq(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
{
union loongarch_instruction insn;
if ((imm & 3) || imm < -SZ_128K || imm >= SZ_128K) {
pr_warn("The generated beq instruction is out of range.\n");
return INSN_BREAK;
}
emit_beq(&insn, rj, rd, imm >> 2);
return insn.word;
}
u32 larch_insn_gen_bne(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
{
union loongarch_instruction insn;
if ((imm & 3) || imm < -SZ_128K || imm >= SZ_128K) {
pr_warn("The generated bne instruction is out of range.\n");
return INSN_BREAK;
}
emit_bne(&insn, rj, rd, imm >> 2);
return insn.word;
}
u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
{
union loongarch_instruction insn;

2
arch/loongarch/kernel/relocate_kernel.S
View file

@ -109,4 +109,4 @@ SYM_CODE_END(kexec_smp_wait)
relocate_new_kernel_end:
.section ".data"
SYM_DATA(relocate_new_kernel_size, .long relocate_new_kernel_end - relocate_new_kernel)
SYM_DATA(relocate_new_kernel_size, .quad relocate_new_kernel_end - relocate_new_kernel)

18
arch/loongarch/kernel/setup.c
View file

@ -191,6 +191,16 @@ static int __init early_parse_mem(char *p)
return -EINVAL;
}
start = 0;
size = memparse(p, &p);
if (*p == '@') /* Every mem=... should contain '@' */
start = memparse(p + 1, &p);
else { /* Only one mem=... is allowed if no '@' */
usermem = 1;
memblock_enforce_memory_limit(size);
return 0;
}
/*
* If a user specifies memory size, we
* blow away any automatically generated
@ -201,14 +211,6 @@ static int __init early_parse_mem(char *p)
memblock_remove(memblock_start_of_DRAM(),
memblock_end_of_DRAM() - memblock_start_of_DRAM());
}
start = 0;
size = memparse(p, &p);
if (*p == '@')
start = memparse(p + 1, &p);
else {
pr_err("Invalid format!\n");
return -EINVAL;
}
if (!IS_ENABLED(CONFIG_NUMA))
memblock_add(start, size);

2
arch/loongarch/kernel/unwind_orc.c
View file

@ -508,7 +508,7 @@ bool unwind_next_frame(struct unwind_state *state)
state->pc = bt_address(pc);
if (!state->pc) {
pr_err("cannot find unwind pc at %pK\n", (void *)pc);
pr_err("cannot find unwind pc at %p\n", (void *)pc);
goto err;
}

705
arch/loongarch/net/bpf_jit.c
View file

@ -4,13 +4,20 @@
*
* Copyright (C) 2022 Loongson Technology Corporation Limited
*/
#include <linux/memory.h>
#include "bpf_jit.h"
#define REG_TCC LOONGARCH_GPR_A6
#define TCC_SAVED LOONGARCH_GPR_S5
#define LOONGARCH_MAX_REG_ARGS 8
#define SAVE_RA BIT(0)
#define SAVE_TCC BIT(1)
#define LOONGARCH_LONG_JUMP_NINSNS 5
#define LOONGARCH_LONG_JUMP_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4)
#define LOONGARCH_FENTRY_NINSNS 2
#define LOONGARCH_FENTRY_NBYTES (LOONGARCH_FENTRY_NINSNS * 4)
#define LOONGARCH_BPF_FENTRY_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4)
#define REG_TCC LOONGARCH_GPR_A6
#define BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack) (round_up(stack, 16) - 80)
static const int regmap[] = {
/* return value from in-kernel function, and exit value for eBPF program */
@ -32,32 +39,57 @@ static const int regmap[] = {
[BPF_REG_AX] = LOONGARCH_GPR_T0,
};
static void mark_call(struct jit_ctx *ctx)
static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx, int *store_offset)
{
ctx->flags |= SAVE_RA;
const struct bpf_prog *prog = ctx->prog;
const bool is_main_prog = !bpf_is_subprog(prog);
if (is_main_prog) {
/*
* LOONGARCH_GPR_T3 = MAX_TAIL_CALL_CNT
* if (REG_TCC > T3 )
* std REG_TCC -> LOONGARCH_GPR_SP + store_offset
* else
* std REG_TCC -> LOONGARCH_GPR_SP + store_offset
* REG_TCC = LOONGARCH_GPR_SP + store_offset
*
* std REG_TCC -> LOONGARCH_GPR_SP + store_offset
*
* The purpose of this code is to first push the TCC into stack,
* and then push the address of TCC into stack.
* In cases where bpf2bpf and tailcall are used in combination,
* the value in REG_TCC may be a count or an address,
* these two cases need to be judged and handled separately.
*/
emit_insn(ctx, addid, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT);
*store_offset -= sizeof(long);
emit_cond_jmp(ctx, BPF_JGT, REG_TCC, LOONGARCH_GPR_T3, 4);
/*
* If REG_TCC < MAX_TAIL_CALL_CNT, the value in REG_TCC is a count,
* push tcc into stack
*/
emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
/* Push the address of TCC into the REG_TCC */
emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
emit_uncond_jmp(ctx, 2);
/*
* If REG_TCC > MAX_TAIL_CALL_CNT, the value in REG_TCC is an address,
* push tcc_ptr into stack
*/
emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
} else {
*store_offset -= sizeof(long);
emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
}
static void mark_tail_call(struct jit_ctx *ctx)
{
ctx->flags |= SAVE_TCC;
}
static bool seen_call(struct jit_ctx *ctx)
{
return (ctx->flags & SAVE_RA);
}
static bool seen_tail_call(struct jit_ctx *ctx)
{
return (ctx->flags & SAVE_TCC);
}
static u8 tail_call_reg(struct jit_ctx *ctx)
{
if (seen_call(ctx))
return TCC_SAVED;
return REG_TCC;
/* Push tcc_ptr into stack */
*store_offset -= sizeof(long);
emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
}
/*
@ -80,6 +112,10 @@ static u8 tail_call_reg(struct jit_ctx *ctx)
* | $s4 |
* +-------------------------+
* | $s5 |
* +-------------------------+
* | tcc |
* +-------------------------+
* | tcc_ptr |
* +-------------------------+ <--BPF_REG_FP
* | prog->aux->stack_depth |
* | (optional) |
@ -88,22 +124,32 @@ static u8 tail_call_reg(struct jit_ctx *ctx)
*/
static void build_prologue(struct jit_ctx *ctx)
{
int stack_adjust = 0, store_offset, bpf_stack_adjust;
int i, stack_adjust = 0, store_offset, bpf_stack_adjust;
const struct bpf_prog *prog = ctx->prog;
const bool is_main_prog = !bpf_is_subprog(prog);
bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
/* To store ra, fp, s0, s1, s2, s3, s4 and s5. */
/* To store ra, fp, s0, s1, s2, s3, s4, s5 */
stack_adjust += sizeof(long) * 8;
/* To store tcc and tcc_ptr */
stack_adjust += sizeof(long) * 2;
stack_adjust = round_up(stack_adjust, 16);
stack_adjust += bpf_stack_adjust;
/* Reserve space for the move_imm + jirl instruction */
for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++)
emit_insn(ctx, nop);
/*
* First instruction initializes the tail call count (TCC).
* On tail call we skip this instruction, and the TCC is
* passed in REG_TCC from the caller.
* First instruction initializes the tail call count (TCC)
* register to zero. On tail call we skip this instruction,
* and the TCC is passed in REG_TCC from the caller.
*/
emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT);
if (is_main_prog)
emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, 0);
emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust);
@ -131,20 +177,13 @@ static void build_prologue(struct jit_ctx *ctx)
store_offset -= sizeof(long);
emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset);
prepare_bpf_tail_call_cnt(ctx, &store_offset);
emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust);
if (bpf_stack_adjust)
emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust);
/*
* Program contains calls and tail calls, so REG_TCC need
* to be saved across calls.
*/
if (seen_tail_call(ctx) && seen_call(ctx))
move_reg(ctx, TCC_SAVED, REG_TCC);
else
emit_insn(ctx, nop);
ctx->stack_size = stack_adjust;
}
@ -177,6 +216,16 @@ static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call)
load_offset -= sizeof(long);
emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset);
/*
* When push into the stack, follow the order of tcc then tcc_ptr.
* When pop from the stack, first pop tcc_ptr then followed by tcc.
*/
load_offset -= 2 * sizeof(long);
emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset);
load_offset += sizeof(long);
emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset);
emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust);
if (!is_tail_call) {
@ -189,7 +238,7 @@ static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call)
* Call the next bpf prog and skip the first instruction
* of TCC initialization.
*/
emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T3, 1);
emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T3, 6);
}
}
@ -208,12 +257,10 @@ bool bpf_jit_supports_far_kfunc_call(void)
return true;
}
/* initialized on the first pass of build_body() */
static int out_offset = -1;
static int emit_bpf_tail_call(struct jit_ctx *ctx)
static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn)
{
int off;
u8 tcc = tail_call_reg(ctx);
int off, tc_ninsn = 0;
int tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size);
u8 a1 = LOONGARCH_GPR_A1;
u8 a2 = LOONGARCH_GPR_A2;
u8 t1 = LOONGARCH_GPR_T1;
@ -222,7 +269,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
const int idx0 = ctx->idx;
#define cur_offset (ctx->idx - idx0)
#define jmp_offset (out_offset - (cur_offset))
#define jmp_offset (tc_ninsn - (cur_offset))
/*
* a0: &ctx
@ -232,6 +279,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
* if (index >= array->map.max_entries)
* goto out;
*/
tc_ninsn = insn ? ctx->offset[insn+1] - ctx->offset[insn] : ctx->offset[0];
off = offsetof(struct bpf_array, map.max_entries);
emit_insn(ctx, ldwu, t1, a1, off);
/* bgeu $a2, $t1, jmp_offset */
@ -239,11 +287,15 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
goto toofar;
/*
* if (--TCC < 0)
* if ((*tcc_ptr)++ >= MAX_TAIL_CALL_CNT)
* goto out;
*/
emit_insn(ctx, addid, REG_TCC, tcc, -1);
if (emit_tailcall_jmp(ctx, BPF_JSLT, REG_TCC, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off);
emit_insn(ctx, ldd, t3, REG_TCC, 0);
emit_insn(ctx, addid, t3, t3, 1);
emit_insn(ctx, std, t3, REG_TCC, 0);
emit_insn(ctx, addid, t2, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT);
if (emit_tailcall_jmp(ctx, BPF_JSGT, t3, t2, jmp_offset) < 0)
goto toofar;
/*
@ -263,15 +315,6 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
emit_insn(ctx, ldd, t3, t2, off);
__build_epilogue(ctx, true);
/* out: */
if (out_offset == -1)
out_offset = cur_offset;
if (cur_offset != out_offset) {
pr_err_once("tail_call out_offset = %d, expected %d!\n",
cur_offset, out_offset);
return -1;
}
return 0;
toofar:
@ -463,7 +506,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
u64 func_addr;
bool func_addr_fixed, sign_extend;
int i = insn - ctx->prog->insnsi;
int ret, jmp_offset;
int ret, jmp_offset, tcc_ptr_off;
const u8 code = insn->code;
const u8 cond = BPF_OP(code);
const u8 t1 = LOONGARCH_GPR_T1;
@ -899,12 +942,16 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
/* function call */
case BPF_JMP | BPF_CALL:
mark_call(ctx);
ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass,
&func_addr, &func_addr_fixed);
if (ret < 0)
return ret;
if (insn->src_reg == BPF_PSEUDO_CALL) {
tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size);
emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off);
}
move_addr(ctx, t1, func_addr);
emit_insn(ctx, jirl, LOONGARCH_GPR_RA, t1, 0);
@ -915,8 +962,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
/* tail call */
case BPF_JMP | BPF_TAIL_CALL:
mark_tail_call(ctx);
if (emit_bpf_tail_call(ctx) < 0)
if (emit_bpf_tail_call(ctx, i) < 0)
return -EINVAL;
break;
@ -1180,12 +1226,528 @@ static int validate_code(struct jit_ctx *ctx)
return -1;
}
return 0;
}
static int validate_ctx(struct jit_ctx *ctx)
{
if (validate_code(ctx))
return -1;
if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries))
return -1;
return 0;
}
static int emit_jump_and_link(struct jit_ctx *ctx, u8 rd, u64 target)
{
if (!target) {
pr_err("bpf_jit: jump target address is error\n");
return -EFAULT;
}
move_imm(ctx, LOONGARCH_GPR_T1, target, false);
emit_insn(ctx, jirl, rd, LOONGARCH_GPR_T1, 0);
return 0;
}
static int emit_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call)
{
int i;
struct jit_ctx ctx;
ctx.idx = 0;
ctx.image = (union loongarch_instruction *)insns;
if (!target) {
for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++)
emit_insn((&ctx), nop);
return 0;
}
return emit_jump_and_link(&ctx, is_call ? LOONGARCH_GPR_T0 : LOONGARCH_GPR_ZERO, (u64)target);
}
static int emit_call(struct jit_ctx *ctx, u64 addr)
{
return emit_jump_and_link(ctx, LOONGARCH_GPR_RA, addr);
}
void *bpf_arch_text_copy(void *dst, void *src, size_t len)
{
int ret;
mutex_lock(&text_mutex);
ret = larch_insn_text_copy(dst, src, len);
mutex_unlock(&text_mutex);
return ret ? ERR_PTR(-EINVAL) : dst;
}
int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type,
void *old_addr, void *new_addr)
{
int ret;
bool is_call = (poke_type == BPF_MOD_CALL);
u32 old_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP};
u32 new_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP};
if (!is_kernel_text((unsigned long)ip) &&
!is_bpf_text_address((unsigned long)ip))
return -ENOTSUPP;
ret = emit_jump_or_nops(old_addr, ip, old_insns, is_call);
if (ret)
return ret;
if (memcmp(ip, old_insns, LOONGARCH_LONG_JUMP_NBYTES))
return -EFAULT;
ret = emit_jump_or_nops(new_addr, ip, new_insns, is_call);
if (ret)
return ret;
mutex_lock(&text_mutex);
if (memcmp(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES))
ret = larch_insn_text_copy(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES);
mutex_unlock(&text_mutex);
return ret;
}
int bpf_arch_text_invalidate(void *dst, size_t len)
{
int i;
int ret = 0;
u32 *inst;
inst = kvmalloc(len, GFP_KERNEL);
if (!inst)
return -ENOMEM;
for (i = 0; i < (len / sizeof(u32)); i++)
inst[i] = INSN_BREAK;
mutex_lock(&text_mutex);
if (larch_insn_text_copy(dst, inst, len))
ret = -EINVAL;
mutex_unlock(&text_mutex);
kvfree(inst);
return ret;
}
static void store_args(struct jit_ctx *ctx, int nargs, int args_off)
{
int i;
for (i = 0; i < nargs; i++) {
emit_insn(ctx, std, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off);
args_off -= 8;
}
}
static void restore_args(struct jit_ctx *ctx, int nargs, int args_off)
{
int i;
for (i = 0; i < nargs; i++) {
emit_insn(ctx, ldd, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off);
args_off -= 8;
}
}
static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l,
int args_off, int retval_off, int run_ctx_off, bool save_ret)
{
int ret;
u32 *branch;
struct bpf_prog *p = l->link.prog;
int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
if (l->cookie) {
move_imm(ctx, LOONGARCH_GPR_T1, l->cookie, false);
emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off);
} else {
emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off);
}
/* arg1: prog */
move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false);
/* arg2: &run_ctx */
emit_insn(ctx, addid, LOONGARCH_GPR_A1, LOONGARCH_GPR_FP, -run_ctx_off);
ret = emit_call(ctx, (const u64)bpf_trampoline_enter(p));
if (ret)
return ret;
/* store prog start time */
move_reg(ctx, LOONGARCH_GPR_S1, LOONGARCH_GPR_A0);
/*
* if (__bpf_prog_enter(prog) == 0)
* goto skip_exec_of_prog;
*/
branch = (u32 *)ctx->image + ctx->idx;
/* nop reserved for conditional jump */
emit_insn(ctx, nop);
/* arg1: &args_off */
emit_insn(ctx, addid, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -args_off);
if (!p->jited)
move_imm(ctx, LOONGARCH_GPR_A1, (const s64)p->insnsi, false);
ret = emit_call(ctx, (const u64)p->bpf_func);
if (ret)
return ret;
if (save_ret) {
emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
}
/* update branch with beqz */
if (ctx->image) {
int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branch;
*branch = larch_insn_gen_beq(LOONGARCH_GPR_A0, LOONGARCH_GPR_ZERO, offset);
}
/* arg1: prog */
move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false);
/* arg2: prog start time */
move_reg(ctx, LOONGARCH_GPR_A1, LOONGARCH_GPR_S1);
/* arg3: &run_ctx */
emit_insn(ctx, addid, LOONGARCH_GPR_A2, LOONGARCH_GPR_FP, -run_ctx_off);
ret = emit_call(ctx, (const u64)bpf_trampoline_exit(p));
return ret;
}
static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_links *tl,
int args_off, int retval_off, int run_ctx_off, u32 **branches)
{
int i;
emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off);
for (i = 0; i < tl->nr_links; i++) {
invoke_bpf_prog(ctx, tl->links[i], args_off, retval_off, run_ctx_off, true);
emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -retval_off);
branches[i] = (u32 *)ctx->image + ctx->idx;
emit_insn(ctx, nop);
}
}
void *arch_alloc_bpf_trampoline(unsigned int size)
{
return bpf_prog_pack_alloc(size, jit_fill_hole);
}
void arch_free_bpf_trampoline(void *image, unsigned int size)
{
bpf_prog_pack_free(image, size);
}
static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
const struct btf_func_model *m, struct bpf_tramp_links *tlinks,
void *func_addr, u32 flags)
{
int i, ret, save_ret;
int stack_size = 0, nargs = 0;
int retval_off, args_off, nargs_off, ip_off, run_ctx_off, sreg_off, tcc_ptr_off;
bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT;
void *orig_call = func_addr;
struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
u32 **branches = NULL;
if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY))
return -ENOTSUPP;
/*
* FP + 8 [ RA to parent func ] return address to parent
* function
* FP + 0 [ FP of parent func ] frame pointer of parent
* function
* FP - 8 [ T0 to traced func ] return address of traced
* function
* FP - 16 [ FP of traced func ] frame pointer of traced
* function
*
* FP - retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or
* BPF_TRAMP_F_RET_FENTRY_RET
* [ argN ]
* [ ... ]
* FP - args_off [ arg1 ]
*
* FP - nargs_off [ regs count ]
*
* FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG
*
* FP - run_ctx_off [ bpf_tramp_run_ctx ]
*
* FP - sreg_off [ callee saved reg ]
*
* FP - tcc_ptr_off [ tail_call_cnt_ptr ]
*/
if (m->nr_args > LOONGARCH_MAX_REG_ARGS)
return -ENOTSUPP;
if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY))
return -ENOTSUPP;
stack_size = 0;
/* Room of trampoline frame to store return address and frame pointer */
stack_size += 16;
save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET);
if (save_ret) {
/* Save BPF R0 and A0 */
stack_size += 16;
retval_off = stack_size;
}
/* Room of trampoline frame to store args */
nargs = m->nr_args;
stack_size += nargs * 8;
args_off = stack_size;
/* Room of trampoline frame to store args number */
stack_size += 8;
nargs_off = stack_size;
/* Room of trampoline frame to store ip address */
if (flags & BPF_TRAMP_F_IP_ARG) {
stack_size += 8;
ip_off = stack_size;
}
/* Room of trampoline frame to store struct bpf_tramp_run_ctx */
stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8);
run_ctx_off = stack_size;
stack_size += 8;
sreg_off = stack_size;
/* Room of trampoline frame to store tail_call_cnt_ptr */
if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) {
stack_size += 8;
tcc_ptr_off = stack_size;
}
stack_size = round_up(stack_size, 16);
if (is_struct_ops) {
/*
* For the trampoline called directly, just handle
* the frame of trampoline.
*/
emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size);
emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8);
emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size);
} else {
/*
* For the trampoline called from function entry,
* the frame of traced function and the frame of
* trampoline need to be considered.
*/
/* RA and FP for parent function */
emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -16);
emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8);
emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0);
emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 16);
/* RA and FP for traced function */
emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size);
emit_insn(ctx, std, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8);
emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size);
}
if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off);
/* callee saved register S1 to pass start time */
emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off);
/* store ip address of the traced function */
if (flags & BPF_TRAMP_F_IP_ARG) {
move_imm(ctx, LOONGARCH_GPR_T1, (const s64)func_addr, false);
emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -ip_off);
}
/* store nargs number */
move_imm(ctx, LOONGARCH_GPR_T1, nargs, false);
emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -nargs_off);
store_args(ctx, nargs, args_off);
/* To traced function */
/* Ftrace jump skips 2 NOP instructions */
if (is_kernel_text((unsigned long)orig_call))
orig_call += LOONGARCH_FENTRY_NBYTES;
/* Direct jump skips 5 NOP instructions */
else if (is_bpf_text_address((unsigned long)orig_call))
orig_call += LOONGARCH_BPF_FENTRY_NBYTES;
if (flags & BPF_TRAMP_F_CALL_ORIG) {
move_imm(ctx, LOONGARCH_GPR_A0, (const s64)im, false);
ret = emit_call(ctx, (const u64)__bpf_tramp_enter);
if (ret)
return ret;
}
for (i = 0; i < fentry->nr_links; i++) {
ret = invoke_bpf_prog(ctx, fentry->links[i], args_off, retval_off,
run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET);
if (ret)
return ret;
}
if (fmod_ret->nr_links) {
branches = kcalloc(fmod_ret->nr_links, sizeof(u32 *), GFP_KERNEL);
if (!branches)
return -ENOMEM;
invoke_bpf_mod_ret(ctx, fmod_ret, args_off, retval_off, run_ctx_off, branches);
}
if (flags & BPF_TRAMP_F_CALL_ORIG) {
restore_args(ctx, m->nr_args, args_off);
if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off);
ret = emit_call(ctx, (const u64)orig_call);
if (ret)
goto out;
emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
im->ip_after_call = ctx->ro_image + ctx->idx;
/* Reserve space for the move_imm + jirl instruction */
for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++)
emit_insn(ctx, nop);
}
for (i = 0; ctx->image && i < fmod_ret->nr_links; i++) {
int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branches[i];
*branches[i] = larch_insn_gen_bne(LOONGARCH_GPR_T1, LOONGARCH_GPR_ZERO, offset);
}
for (i = 0; i < fexit->nr_links; i++) {
ret = invoke_bpf_prog(ctx, fexit->links[i], args_off, retval_off, run_ctx_off, false);
if (ret)
goto out;
}
if (flags & BPF_TRAMP_F_CALL_ORIG) {
im->ip_epilogue = ctx->ro_image + ctx->idx;
move_imm(ctx, LOONGARCH_GPR_A0, (const s64)im, false);
ret = emit_call(ctx, (const u64)__bpf_tramp_exit);
if (ret)
goto out;
}
if (flags & BPF_TRAMP_F_RESTORE_REGS)
restore_args(ctx, m->nr_args, args_off);
if (save_ret) {
emit_insn(ctx, ldd, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off);
emit_insn(ctx, ldd, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8));
}
emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off);
if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off);
if (is_struct_ops) {
/* trampoline called directly */
emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8);
emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size);
emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0);
} else {
/* trampoline called from function entry */
emit_insn(ctx, ldd, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8);
emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16);
emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size);
emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8);
emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0);
emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, 16);
if (flags & BPF_TRAMP_F_SKIP_FRAME)
/* return to parent function */
emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0);
else
/* return to traced function */
emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T0, 0);
}
ret = ctx->idx;
out:
kfree(branches);
return ret;
}
int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image,
void *ro_image_end, const struct btf_func_model *m,
u32 flags, struct bpf_tramp_links *tlinks, void *func_addr)
{
int ret, size;
void *image, *tmp;
struct jit_ctx ctx;
size = ro_image_end - ro_image;
image = kvmalloc(size, GFP_KERNEL);
if (!image)
return -ENOMEM;
ctx.image = (union loongarch_instruction *)image;
ctx.ro_image = (union loongarch_instruction *)ro_image;
ctx.idx = 0;
jit_fill_hole(image, (unsigned int)(ro_image_end - ro_image));
ret = __arch_prepare_bpf_trampoline(&ctx, im, m, tlinks, func_addr, flags);
if (ret > 0 && validate_code(&ctx) < 0) {
ret = -EINVAL;
goto out;
}
tmp = bpf_arch_text_copy(ro_image, image, size);
if (IS_ERR(tmp)) {
ret = PTR_ERR(tmp);
goto out;
}
bpf_flush_icache(ro_image, ro_image_end);
out:
kvfree(image);
return ret < 0 ? ret : size;
}
int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
struct bpf_tramp_links *tlinks, void *func_addr)
{
int ret;
struct jit_ctx ctx;
struct bpf_tramp_image im;
ctx.image = NULL;
ctx.idx = 0;
ret = __arch_prepare_bpf_trampoline(&ctx, &im, m, tlinks, func_addr, flags);
/* Page align */
return ret < 0 ? ret : round_up(ret * LOONGARCH_INSN_SIZE, PAGE_SIZE);
}
struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
{
bool tmp_blinded = false, extra_pass = false;
@ -1288,7 +1850,7 @@ skip_init_ctx:
build_epilogue(&ctx);
/* 3. Extra pass to validate JITed code */
if (validate_code(&ctx)) {
if (validate_ctx(&ctx)) {
bpf_jit_binary_free(header);
prog = orig_prog;
goto out_offset;
@ -1342,7 +1904,6 @@ out:
if (tmp_blinded)
bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog);
out_offset = -1;
return prog;
@ -1354,6 +1915,16 @@ out_free:
goto out_offset;
}
bool bpf_jit_bypass_spec_v1(void)
{
return true;
}
bool bpf_jit_bypass_spec_v4(void)
{
return true;
}
/* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */
bool bpf_jit_supports_subprog_tailcalls(void)
{

6
arch/loongarch/net/bpf_jit.h
View file

@ -18,6 +18,7 @@ struct jit_ctx {
u32 *offset;
int num_exentries;
union loongarch_instruction *image;
union loongarch_instruction *ro_image;
u32 stack_size;
};
@ -308,3 +309,8 @@ static inline int emit_tailcall_jmp(struct jit_ctx *ctx, u8 cond, enum loongarch
return -EINVAL;
}
static inline void bpf_flush_icache(void *start, void *end)
{
flush_icache_range((unsigned long)start, (unsigned long)end);
}

2
arch/loongarch/vdso/Makefile
View file

@ -36,7 +36,7 @@ endif
# VDSO linker flags.
ldflags-y := -Bsymbolic --no-undefined -soname=linux-vdso.so.1 \
$(filter -E%,$(KBUILD_CFLAGS)) -nostdlib -shared --build-id -T
$(filter -E%,$(KBUILD_CFLAGS)) -shared --build-id -T
#
# Shared build commands.