diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index b2260e24f10211..0d6eef18aa4a8d 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -38,8 +38,10 @@ config ARC select HAVE_KERNEL_LZMA select HAVE_KPROBES select HAVE_KRETPROBES + select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_MOD_ARCH_SPECIFIC select HAVE_PERF_EVENTS + select HAVE_SYSCALL_TRACEPOINTS select IRQ_DOMAIN select MODULES_USE_ELF_RELA select OF diff --git a/arch/arc/include/asm/perf_event.h b/arch/arc/include/asm/perf_event.h index d9c1d678638019..ea4c463e549fc0 100644 --- a/arch/arc/include/asm/perf_event.h +++ b/arch/arc/include/asm/perf_event.h @@ -65,6 +65,10 @@ struct arc_reg_cc_build { #define PERF_COUNT_ARC_HW_MAX (PERF_COUNT_HW_MAX + 8) +#ifdef CONFIG_PERF_EVENTS +#define perf_arch_bpf_user_pt_regs(regs) (struct user_regs_struct *)regs +#endif + /* * Some ARC pct quirks: * diff --git a/arch/arc/include/asm/ptrace.h b/arch/arc/include/asm/ptrace.h index 549b49142d85e9..a70c7e79966e6d 100644 --- a/arch/arc/include/asm/ptrace.h +++ b/arch/arc/include/asm/ptrace.h @@ -8,6 +8,7 @@ #define __ASM_ARC_PTRACE_H #include +#include #ifndef __ASSEMBLY__ @@ -53,6 +54,8 @@ struct pt_regs { ecr_reg ecr; }; +#define MAX_REG_OFFSET offsetof(struct pt_regs, ecr) + struct callee_regs { unsigned long r25, r24, r23, r22, r21, r20, r19, r18, r17, r16, r15, r14, r13; }; @@ -95,6 +98,8 @@ struct pt_regs { unsigned long status32; }; +#define MAX_REG_OFFSET offsetof(struct pt_regs, status32) + struct callee_regs { unsigned long r25, r24, r23, r22, r21, r20, r19, r18, r17, r16, r15, r14, r13; }; @@ -119,6 +124,8 @@ struct pt_regs { unsigned long status32; }; +#define MAX_REG_OFFSET offsetof(struct pt_regs, status32) + /* ARCv3 callee regs start from r14; gp is r30 not r26 */ struct callee_regs { unsigned long r14, r15, r16, r17, r18, r19, r20, r21, r22, r23, r24, r25, r26; @@ -170,6 +177,27 @@ static inline void instruction_pointer_set(struct pt_regs *regs, { instruction_pointer(regs) = val; } + +static inline unsigned long kernel_stack_pointer(struct pt_regs *regs) +{ + return regs->sp; +} + +extern int regs_query_register_offset(const char *name); +extern const char *regs_query_register_name(unsigned int offset); +extern bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr); +extern unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, + unsigned int n); + +static inline unsigned long regs_get_register(struct pt_regs *regs, + unsigned int offset) +{ + if (unlikely(offset > MAX_REG_OFFSET)) + return 0; + + return *(unsigned long *)((unsigned long)regs + offset); +} + #endif /* !__ASSEMBLY__ */ #endif /* __ASM_PTRACE_H */ diff --git a/arch/arc/include/asm/syscall.h b/arch/arc/include/asm/syscall.h index 0e0fdbe6b2a118..42a8995fe2ee0b 100644 --- a/arch/arc/include/asm/syscall.h +++ b/arch/arc/include/asm/syscall.h @@ -12,6 +12,8 @@ #include #include /* in_syscall() */ +extern void *sys_call_table[]; + static inline long syscall_get_nr(struct task_struct *task, struct pt_regs *regs) { diff --git a/arch/arc/include/asm/thread_info.h b/arch/arc/include/asm/thread_info.h index c4c0b3492326b4..94c1f5241c0f5b 100644 --- a/arch/arc/include/asm/thread_info.h +++ b/arch/arc/include/asm/thread_info.h @@ -82,6 +82,7 @@ static inline __attribute_const__ struct thread_info *current_thread_info(void) #define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ #define TIF_SYSCALL_TRACE 15 /* syscall trace active */ #define TIF_MEMDIE 16 +#define TIF_SYSCALL_TRACEPOINT 17 /* syscall tracepoint instrumentation */ #define _TIF_SYSCALL_TRACE (1< + +typedef struct user_regs_struct bpf_user_pt_regs_t; + +#endif /* _UAPI__ASM_BPF_PERF_EVENT_H__ */ diff --git a/arch/arc/kernel/disasm.c b/arch/arc/kernel/disasm.c index 49be4e45b847de..4248ced7d18e5b 100644 --- a/arch/arc/kernel/disasm.c +++ b/arch/arc/kernel/disasm.c @@ -434,6 +434,7 @@ long __kprobes get_reg(int reg, struct pt_regs *regs, { long *p; +#if defined(CONFIG_ISA_ARCOMPACT) if (reg <= 12) { p = ®s->r0; return p[-reg]; @@ -441,11 +442,52 @@ long __kprobes get_reg(int reg, struct pt_regs *regs, if (cregs && (reg <= 25)) { p = &cregs->r13; - return p[13-reg]; + return p[13 - reg]; } if (reg == 26) return regs->gp; + +#elif defined(CONFIG_ISA_ARCV2) + if (reg <= 11) { + p = ®s->r0; + return p[reg]; + } + + if (reg == 12) + return regs->r12; + + if (cregs && (reg <= 25)) { + p = &cregs->r13; + return p[13 - reg]; + } + + if (reg == 26) + return regs->gp; + if (reg == 30) + return regs->r30; +#ifdef CONFIG_ARC_HAS_ACCL_REGS + if (reg == 58) + return regs->r58; + if (reg == 59) + return regs->r59; +#endif + +#else /* CONFIG_ISA_ARCV3 */ + if (reg <= 13) { + p = ®s->r0; + return p[reg]; + } + + if (cregs && (reg <= 26)) { + p = &cregs->r14; + return p[reg - 14]; + } + + if (reg == 30) + return regs->gp; +#endif + if (reg == 27) return regs->fp; if (reg == 28) @@ -461,6 +503,7 @@ void __kprobes set_reg(int reg, long val, struct pt_regs *regs, { long *p; +#if defined(CONFIG_ISA_ARCOMPACT) switch (reg) { case 0 ... 12: p = ®s->r0; @@ -469,7 +512,7 @@ void __kprobes set_reg(int reg, long val, struct pt_regs *regs, case 13 ... 25: if (cregs) { p = &cregs->r13; - p[13-reg] = val; + p[13 - reg] = val; } break; case 26: @@ -487,6 +530,76 @@ void __kprobes set_reg(int reg, long val, struct pt_regs *regs, default: break; } +#elif defined(CONFIG_ISA_ARCV2) + switch (reg) { + case 0 ... 11: + p = ®s->r0; + p[reg] = val; + break; + case 12: + regs->r12 = val; + break; + case 13 ... 25: + if (cregs) { + p = &cregs->r13; + p[13 - reg] = val; + } + break; + case 26: + regs->gp = val; + break; + case 27: + regs->fp = val; + break; + case 28: + regs->sp = val; + break; + case 30: + regs->r30 = val; + break; + case 31: + regs->blink = val; + break; +#ifdef CONFIG_ARC_HAS_ACCL_REGS + case 58: + regs->r58 = val; + break; + case 59: + regs->r59 = val; + break; +#endif + default: + break; + } +#else /* CONFIG_ISA_ARCV3 */ + switch (reg) { + case 0 ... 13: + p = ®s->r0; + p[reg] = val; + break; + case 14 ... 26: + if (cregs) { + p = &cregs->r14; + p[reg - 14] = val; + } + break; + case 27: + regs->fp = val; + break; + case 28: + regs->sp = val; + break; + case 30: + regs->gp = val; + break; + case 31: + regs->blink = val; + break; + default: + break; + } + +#endif } /* diff --git a/arch/arc/kernel/entry.S b/arch/arc/kernel/entry.S index 2061f7108a7293..e94590812f08ba 100644 --- a/arch/arc/kernel/entry.S +++ b/arch/arc/kernel/entry.S @@ -29,8 +29,8 @@ ENTRY(sys_clone_wrapper) DISCARD_CALLEE_SAVED_USER GET_CURR_THR_INFO_FLAGS r10 - btst r10, TIF_SYSCALL_TRACE - bnz tracesys_exit + and.f 0, r10, _TIF_SYSCALL_WORK + bnz tracesys_exit b .Lret_from_system_call END(sys_clone_wrapper) @@ -41,8 +41,8 @@ ENTRY(sys_clone3_wrapper) DISCARD_CALLEE_SAVED_USER GET_CURR_THR_INFO_FLAGS r10 - btst r10, TIF_SYSCALL_TRACE - bnz tracesys_exit + and.f 0, r10, _TIF_SYSCALL_WORK + bnz tracesys_exit b .Lret_from_system_call END(sys_clone3_wrapper) @@ -175,6 +175,7 @@ tracesys_exit: STR r0, sp, PT_r0 ; POST syscall trace hook + MOVR r0, sp ; pt_regs bl @syscall_trace_exit ; don't call ret_from_system_call as it saves r0, already done above @@ -223,8 +224,8 @@ ENTRY(EV_Trap) ; syscall tracing ongoing, invoke pre-post-hooks around syscall GET_CURR_THR_INFO_FLAGS r10 - btst r10, TIF_SYSCALL_TRACE - bnz tracesys ; this never comes back + and.f 0, r10, _TIF_SYSCALL_WORK + bnz tracesys ; this never comes back ;============ Normal syscall case diff --git a/arch/arc/kernel/ptrace.c b/arch/arc/kernel/ptrace.c index 72eea096a6ee00..bbcf1d46687237 100644 --- a/arch/arc/kernel/ptrace.c +++ b/arch/arc/kernel/ptrace.c @@ -11,6 +11,124 @@ #include #include +#define CREATE_TRACE_POINTS +#include + +struct pt_regs_offset { + const char *name; + int offset; +}; + +#define REG_OFFSET_NAME(r) {.name = #r, .offset = offsetof(struct pt_regs, r)} +#define REG_OFFSET_END {.name = NULL, .offset = 0} + +#ifdef CONFIG_ISA_ARCOMPACT +static const struct pt_regs_offset regoffset_table[] = { + REG_OFFSET_NAME(bta), + REG_OFFSET_NAME(lp_start), + REG_OFFSET_NAME(lp_end), + REG_OFFSET_NAME(lp_count), + REG_OFFSET_NAME(status32), + REG_OFFSET_NAME(ret), + REG_OFFSET_NAME(blink), + REG_OFFSET_NAME(fp), + REG_OFFSET_NAME(gp), + REG_OFFSET_NAME(r12), + REG_OFFSET_NAME(r11), + REG_OFFSET_NAME(r10), + REG_OFFSET_NAME(r9), + REG_OFFSET_NAME(r8), + REG_OFFSET_NAME(r7), + REG_OFFSET_NAME(r6), + REG_OFFSET_NAME(r5), + REG_OFFSET_NAME(r4), + REG_OFFSET_NAME(r3), + REG_OFFSET_NAME(r2), + REG_OFFSET_NAME(r1), + REG_OFFSET_NAME(r0), + REG_OFFSET_NAME(sp), + REG_OFFSET_NAME(orig_r0), + REG_OFFSET_NAME(ecr), + REG_OFFSET_END, +}; + +#elif defined(CONFIG_ISA_ARCV2) + +static const struct pt_regs_offset regoffset_table[] = { + REG_OFFSET_NAME(orig_r0), + REG_OFFSET_NAME(ecr), + REG_OFFSET_NAME(bta), + REG_OFFSET_NAME(fp), + REG_OFFSET_NAME(r30), + REG_OFFSET_NAME(r12), + REG_OFFSET_NAME(gp), +#ifdef CONFIG_ARC_HAS_ACCL_REGS + REG_OFFSET_NAME(r58), + REG_OFFSET_NAME(r59), +#endif +#ifdef CONFIG_ARC_DSP_SAVE_RESTORE_REGS + REG_OFFSET_NAME(DSP_CTRL), +#endif + REG_OFFSET_NAME(sp), + REG_OFFSET_NAME(r0), + REG_OFFSET_NAME(r1), + REG_OFFSET_NAME(r2), + REG_OFFSET_NAME(r3), + REG_OFFSET_NAME(r4), + REG_OFFSET_NAME(r5), + REG_OFFSET_NAME(r6), + REG_OFFSET_NAME(r7), + REG_OFFSET_NAME(r8), + REG_OFFSET_NAME(r9), + REG_OFFSET_NAME(r10), + REG_OFFSET_NAME(r11), + REG_OFFSET_NAME(blink), +#ifndef CONFIG_ARC_LACKS_ZOL + REG_OFFSET_NAME(lp_end), + REG_OFFSET_NAME(lp_start), + REG_OFFSET_NAME(lp_count), +#endif + REG_OFFSET_NAME(ei), + REG_OFFSET_NAME(ldi), + REG_OFFSET_NAME(jli), + REG_OFFSET_NAME(ret), + REG_OFFSET_NAME(status32), + REG_OFFSET_END, +}; + +#else /* CONFIG_ISA_ARCV3 */ + +static const struct pt_regs_offset regoffset_table[] = { + REG_OFFSET_NAME(orig_r0), + REG_OFFSET_NAME(r58), + REG_OFFSET_NAME(r59), + REG_OFFSET_NAME(ecr), + REG_OFFSET_NAME(bta), + REG_OFFSET_NAME(fp), + REG_OFFSET_NAME(gp), + REG_OFFSET_NAME(sp), + REG_OFFSET_NAME(r0), + REG_OFFSET_NAME(r1), + REG_OFFSET_NAME(r2), + REG_OFFSET_NAME(r3), + REG_OFFSET_NAME(r4), + REG_OFFSET_NAME(r5), + REG_OFFSET_NAME(r6), + REG_OFFSET_NAME(r7), + REG_OFFSET_NAME(r8), + REG_OFFSET_NAME(r9), + REG_OFFSET_NAME(r10), + REG_OFFSET_NAME(r11), + REG_OFFSET_NAME(r12), + REG_OFFSET_NAME(r13), + REG_OFFSET_NAME(blink), + REG_OFFSET_NAME(ret), + REG_OFFSET_NAME(status32), + REG_OFFSET_END, +}; + +#endif + static struct callee_regs *task_callee_regs(struct task_struct *tsk) { struct callee_regs *tmp = (struct callee_regs *)tsk->thread.callee_reg; @@ -69,7 +187,7 @@ static int genregs_get(struct task_struct *target, membuf_store(&to, cregs->r15); membuf_store(&to, cregs->r14); #ifdef CONFIG_ISA_ARCV3 - membuf_store(&to, &ptregs->r13); + membuf_store(&to, ptregs->r13); #else membuf_store(&to, cregs->r13); #endif @@ -280,13 +398,61 @@ long arch_ptrace(struct task_struct *child, long request, asmlinkage int syscall_trace_enter(struct pt_regs *regs) { - if (tracehook_report_syscall_entry(regs)) - return -1; + if (test_thread_flag(TIF_SYSCALL_TRACE)) + if (tracehook_report_syscall_entry(regs)) + return -1; + +#ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS + if (test_thread_flag(TIF_SYSCALL_TRACEPOINT)) + trace_sys_enter(regs, syscall_get_nr(current, regs)); +#endif return regs->r8; } asmlinkage void syscall_trace_exit(struct pt_regs *regs) { - tracehook_report_syscall_exit(regs, 0); + if (test_thread_flag(TIF_SYSCALL_TRACE)) + tracehook_report_syscall_exit(regs, 0); + +#ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS + if (test_thread_flag(TIF_SYSCALL_TRACEPOINT)) + trace_sys_exit(regs, regs_return_value(regs)); +#endif +} + +int regs_query_register_offset(const char *name) +{ + const struct pt_regs_offset *roff; + + for (roff = regoffset_table; roff->name != NULL; roff++) + if (!strcmp(roff->name, name)) + return roff->offset; + return -EINVAL; +} + +const char *regs_query_register_name(unsigned int offset) +{ + const struct pt_regs_offset *roff; + for (roff = regoffset_table; roff->name != NULL; roff++) + if (roff->offset == offset) + return roff->name; + return NULL; +} + +bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr) +{ + return (addr & ~(THREAD_SIZE - 1)) == + (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1)); +} + +unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n) +{ + unsigned long *addr = (unsigned long *)kernel_stack_pointer(regs); + + addr += n; + if (regs_within_kernel_stack(regs, (unsigned long)addr)) + return *addr; + else + return 0; } diff --git a/arch/arc/mm/cache-arcv3.c b/arch/arc/mm/cache-arcv3.c index 9dd2784a6c105d..8fcc0b201a02e4 100644 --- a/arch/arc/mm/cache-arcv3.c +++ b/arch/arc/mm/cache-arcv3.c @@ -315,14 +315,19 @@ void flush_icache_range(unsigned long kvaddr, unsigned long kvend) { unsigned int tot_sz = kvend - kvaddr; - BUG_ON((kvaddr < VMALLOC_START) || (kvend > VMALLOC_END)); + BUG_ON(is_vmalloc_addr((void *)kvaddr) && !is_vmalloc_addr((void *)kvend)); + BUG_ON(!is_vmalloc_addr((void *)kvaddr) && is_vmalloc_addr((void *)kvend)); + BUG_ON(kvaddr < TASK_SIZE); while (tot_sz > 0) { unsigned int off, sz; unsigned long paddr, pfn; off = kvaddr % PAGE_SIZE; - pfn = vmalloc_to_pfn((void *)kvaddr); + if (is_vmalloc_addr((void *)kvaddr)) + pfn = vmalloc_to_pfn((void *)kvaddr); + else + pfn = virt_to_pfn((void *)kvaddr); paddr = (pfn << PAGE_SHIFT) + off; sz = min_t(unsigned int, tot_sz, PAGE_SIZE - off); __sync_icache_dcache(paddr, kvaddr, sz); diff --git a/scripts/pahole-flags.sh b/scripts/pahole-flags.sh index e6093adf4c06d7..12cd096be97a3b 100755 --- a/scripts/pahole-flags.sh +++ b/scripts/pahole-flags.sh @@ -14,7 +14,7 @@ if [ "${pahole_ver}" -ge "118" ] && [ "${pahole_ver}" -le "121" ]; then extra_paholeopt="${extra_paholeopt} --skip_encoding_btf_vars" fi if [ "${pahole_ver}" -ge "121" ]; then - extra_paholeopt="${extra_paholeopt} --btf_gen_floats" + extra_paholeopt="${extra_paholeopt}" fi echo ${extra_paholeopt} diff --git a/tools/include/uapi/asm/bpf_perf_event.h b/tools/include/uapi/asm/bpf_perf_event.h index 39acc149d84320..d7dfeab0d71ad3 100644 --- a/tools/include/uapi/asm/bpf_perf_event.h +++ b/tools/include/uapi/asm/bpf_perf_event.h @@ -1,5 +1,7 @@ #if defined(__aarch64__) #include "../../arch/arm64/include/uapi/asm/bpf_perf_event.h" +#elif defined(__arc__) +#include "../../arch/arc/include/uapi/asm/bpf_perf_event.h" #elif defined(__s390__) #include "../../arch/s390/include/uapi/asm/bpf_perf_event.h" #elif defined(__riscv) diff --git a/tools/lib/bpf/bpf_tracing.h b/tools/lib/bpf/bpf_tracing.h index db05a593710560..7c7508600a6cc1 100644 --- a/tools/lib/bpf/bpf_tracing.h +++ b/tools/lib/bpf/bpf_tracing.h @@ -27,6 +27,9 @@ #elif defined(__TARGET_ARCH_riscv) #define bpf_target_riscv #define bpf_target_defined +#elif defined(__TARGET_ARCH_arc) + #define bpf_target_arc + #define bpf_target_defined #else /* Fall back to what the compiler says */ @@ -54,6 +57,9 @@ #elif defined(__riscv) && __riscv_xlen == 64 #define bpf_target_riscv #define bpf_target_defined +#elif defined(__arc__) + #define bpf_target_arc + #define bpf_target_defined #endif /* no compiler target */ #endif @@ -320,6 +326,30 @@ struct pt_regs; #define PT_REGS_SP_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), sp) #define PT_REGS_IP_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), epc) +#elif defined(bpf_target_arc) + +struct pt_regs; +#define PT_REGS_ARC const volatile struct user_regs_struct +#define PT_REGS_PARM1(x) (((PT_REGS_ARC *)(x))->scratch.r0) +#define PT_REGS_PARM2(x) (((PT_REGS_ARC *)(x))->scratch.r1) +#define PT_REGS_PARM3(x) (((PT_REGS_ARC *)(x))->scratch.r2) +#define PT_REGS_PARM4(x) (((PT_REGS_ARC *)(x))->scratch.r3) +#define PT_REGS_PARM5(x) (((PT_REGS_ARC *)(x))->scratch.r4) +#define PT_REGS_RET(x) (((PT_REGS_ARC *)(x))->scratch.blink) +#define PT_REGS_RC(x) (((PT_REGS_ARC *)(x))->scratch.r0) +#define PT_REGS_SP(x) (((PT_REGS_ARC *)(x))->scratch.sp) +#define PT_REGS_IP(x) (((PT_REGS_ARC *)(x))->scratch.ret) + +#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((PT_REGS_ARC *)(x), scratch.r0) +#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((PT_REGS_ARC *)(x), scratch.r1) +#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((PT_REGS_ARC *)(x), scratch.r2) +#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((PT_REGS_ARC *)(x), scratch.r3) +#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((PT_REGS_ARC *)(x), scratch.r4) +#define PT_REGS_RET_CORE(x) BPF_CORE_READ((PT_REGS_ARC *)(x), scratch.blink) +#define PT_REGS_RC_CORE(x) BPF_CORE_READ((PT_REGS_ARC *)(x), scratch.r0) +#define PT_REGS_SP_CORE(x) BPF_CORE_READ((PT_REGS_ARC *)(x), scratch.sp) +#define PT_REGS_IP_CORE(x) BPF_CORE_READ((PT_REGS_ARC *)(x), scratch.ret) + #endif #if defined(bpf_target_powerpc)