From 341b58f626ec749dfffb4c2d9e02e7893cb3d70f Mon Sep 17 00:00:00 2001 From: hujun5 Date: Tue, 26 Mar 2024 13:52:54 +0800 Subject: [PATCH 1/3] irq: use per-cpu reg to replace g_current_regs Signed-off-by: hujun5 --- arch/arm/include/arm/irq.h | 36 +++++++++++---- arch/arm/include/armv6-m/irq.h | 36 +++++++++++---- arch/arm/include/armv7-a/irq.h | 44 ++++++++++++++---- arch/arm/include/armv7-m/irq.h | 36 +++++++++++---- arch/arm/include/armv7-r/irq.h | 31 +++++++++---- arch/arm/include/armv8-m/irq.h | 36 +++++++++++---- arch/arm/include/armv8-r/irq.h | 31 +++++++++---- arch/arm/include/irq.h | 30 ------------- arch/arm/include/tlsr82/irq.h | 36 +++++++++++---- arch/arm/src/armv7-a/arm_cpuhead.S | 1 + arch/arm/src/armv7-a/arm_head.S | 1 + arch/arm/src/armv7-r/arm_cpuhead.S | 1 + arch/arm/src/armv7-r/arm_head.S | 1 + arch/arm/src/armv8-r/arm_head.S | 1 + arch/arm/src/common/arm_initialize.c | 3 ++ arch/arm64/include/irq.h | 45 +++++++++---------- arch/arm64/src/common/arm64_arch.h | 10 ----- arch/arm64/src/common/arm64_backtrace.c | 2 +- arch/arm64/src/common/arm64_copystate.c | 4 +- arch/arm64/src/common/arm64_cpustart.c | 3 +- arch/arm64/src/common/arm64_fork.c | 2 - arch/arm64/src/common/arm64_fpu.c | 6 +-- arch/arm64/src/common/arm64_head.S | 6 +++ arch/arm64/src/common/arm64_initialize.c | 12 ----- arch/arm64/src/common/arm64_initialstate.c | 4 -- arch/arm64/src/common/arm64_registerdump.c | 2 - .../src/common/arm64_schedulesigaction.c | 3 +- arch/arm64/src/common/arm64_vectors.S | 18 -------- 28 files changed, 256 insertions(+), 185 deletions(-) diff --git a/arch/arm/include/arm/irq.h b/arch/arm/include/arm/irq.h index 88ecca4210f05..eb0450ab8d925 100644 --- a/arch/arm/include/arm/irq.h +++ b/arch/arm/include/arm/irq.h @@ -157,13 +157,26 @@ struct xcptcontext uintptr_t far; #endif }; -#endif /**************************************************************************** - * Inline functions + * Public Data ****************************************************************************/ -#ifndef __ASSEMBLY__ +/* g_current_regs[] holds a references to the current interrupt level + * register storage structure. If is non-NULL only during interrupt + * processing. Access to g_current_regs[] must be through the + * [get/set]_current_regs for portability. + */ + +/* For the case of architectures with multiple CPUs, then there must be one + * such value for each processor that can receive an interrupt. + */ + +extern volatile uint32_t *g_current_regs[CONFIG_SMP_NCPUS]; + +/**************************************************************************** + * Inline functions + ****************************************************************************/ /* Name: up_irq_save, up_irq_restore, and friends. * @@ -242,17 +255,22 @@ int up_cpu_index(void) noinstrument_function; # define up_cpu_index() 0 #endif /* CONFIG_SMP */ -#endif /* __ASSEMBLY__ */ +noinstrument_function +static inline_function uint32_t *up_current_regs(void) +{ + return (uint32_t *)g_current_regs[up_cpu_index()]; +} -/**************************************************************************** - * Public Data - ****************************************************************************/ +noinstrument_function +static inline_function void up_set_current_regs(uint32_t *regs) +{ + g_current_regs[up_cpu_index()] = regs; +} /**************************************************************************** * Public Function Prototypes ****************************************************************************/ -#ifndef __ASSEMBLY__ #ifdef __cplusplus #define EXTERN extern "C" extern "C" @@ -265,6 +283,6 @@ extern "C" #ifdef __cplusplus } #endif -#endif +#endif /* __ASSEMBLY__ */ #endif /* __ARCH_ARM_INCLUDE_ARM_IRQ_H */ diff --git a/arch/arm/include/armv6-m/irq.h b/arch/arm/include/armv6-m/irq.h index 9a0426a2dfb29..cc6fc4fd86198 100644 --- a/arch/arm/include/armv6-m/irq.h +++ b/arch/arm/include/armv6-m/irq.h @@ -189,13 +189,26 @@ struct xcptcontext uint32_t *regs; }; -#endif /**************************************************************************** - * Inline functions + * Public Data ****************************************************************************/ -#ifndef __ASSEMBLY__ +/* g_current_regs[] holds a references to the current interrupt level + * register storage structure. If is non-NULL only during interrupt + * processing. Access to g_current_regs[] must be through the + * [get/set]_current_regs for portability. + */ + +/* For the case of architectures with multiple CPUs, then there must be one + * such value for each processor that can receive an interrupt. + */ + +extern volatile uint32_t *g_current_regs[CONFIG_SMP_NCPUS]; + +/**************************************************************************** + * Inline functions + ****************************************************************************/ /* Name: up_irq_save, up_irq_restore, and friends. * @@ -366,17 +379,22 @@ static inline_function uint32_t up_getsp(void) return sp; } -#endif /* __ASSEMBLY__ */ +noinstrument_function +static inline_function uint32_t *up_current_regs(void) +{ + return (uint32_t *)g_current_regs[up_cpu_index()]; +} -/**************************************************************************** - * Public Data - ****************************************************************************/ +noinstrument_function +static inline_function void up_set_current_regs(uint32_t *regs) +{ + g_current_regs[up_cpu_index()] = regs; +} /**************************************************************************** * Public Function Prototypes ****************************************************************************/ -#ifndef __ASSEMBLY__ #ifdef __cplusplus #define EXTERN extern "C" extern "C" @@ -389,6 +407,6 @@ extern "C" #ifdef __cplusplus } #endif -#endif +#endif /* __ASSEMBLY__ */ #endif /* __ARCH_ARM_INCLUDE_ARMV6_M_IRQ_H */ diff --git a/arch/arm/include/armv7-a/irq.h b/arch/arm/include/armv7-a/irq.h index cb8f14831cf66..61c28ef0690b5 100644 --- a/arch/arm/include/armv7-a/irq.h +++ b/arch/arm/include/armv7-a/irq.h @@ -249,7 +249,6 @@ struct xcpt_syscall_s * For a total of 17 (XCPTCONTEXT_REGS) */ -#ifndef __ASSEMBLY__ struct xcptcontext { /* The following function pointer is non-zero if there are pending signals @@ -326,16 +325,11 @@ struct xcptcontext #endif #endif }; -#endif - -#endif /* __ASSEMBLY__ */ /**************************************************************************** * Inline functions ****************************************************************************/ -#ifndef __ASSEMBLY__ - /* Name: up_irq_save, up_irq_restore, and friends. * * NOTE: This function should never be called from application code and, @@ -490,13 +484,45 @@ static inline_function uint32_t up_getsp(void) return sp; } -#endif /* __ASSEMBLY__ */ +/**************************************************************************** + * Name: + * up_current_regs/up_set_current_regs + * + * Description: + * We use the following code to manipulate the TPIDRPRW register, + * which exists uniquely for each CPU and is primarily designed to store + * current thread information. Currently, we leverage it to store interrupt + * information, with plans to further optimize its use for storing both + * thread and interrupt information in the future. + * + ****************************************************************************/ + +noinstrument_function +static inline_function uint32_t *up_current_regs(void) +{ + uint32_t *regs; + __asm__ __volatile__ + ( + "mrc " "p15, " "0" ", %0, " "c13" ", " "c0" ", " "4" "\n" + : "=r"(regs) + ); + return regs; +} + +noinstrument_function +static inline_function void up_set_current_regs(uint32_t *regs) +{ + __asm__ __volatile__ + ( + "mcr " "p15, " "0" ", %0, " "c13" ", " "c0" ", " "4" "\n" + :: "r"(regs) + ); +} /**************************************************************************** * Public Data ****************************************************************************/ -#ifndef __ASSEMBLY__ #ifdef __cplusplus #define EXTERN extern "C" extern "C" @@ -513,6 +539,6 @@ extern "C" #ifdef __cplusplus } #endif -#endif +#endif /* __ASSEMBLY__ */ #endif /* __ARCH_ARM_INCLUDE_ARMV7_A_IRQ_H */ diff --git a/arch/arm/include/armv7-m/irq.h b/arch/arm/include/armv7-m/irq.h index 1c78a60140e54..333b95d3ad454 100644 --- a/arch/arm/include/armv7-m/irq.h +++ b/arch/arm/include/armv7-m/irq.h @@ -251,13 +251,26 @@ struct xcptcontext uint32_t *regs; }; -#endif /**************************************************************************** - * Inline functions + * Public Data ****************************************************************************/ -#ifndef __ASSEMBLY__ +/* g_current_regs[] holds a references to the current interrupt level + * register storage structure. If is non-NULL only during interrupt + * processing. Access to g_current_regs[] must be through the + * [get/set]_current_regs for portability. + */ + +/* For the case of architectures with multiple CPUs, then there must be one + * such value for each processor that can receive an interrupt. + */ + +extern volatile uint32_t *g_current_regs[CONFIG_SMP_NCPUS]; + +/**************************************************************************** + * Inline functions + ****************************************************************************/ /* Name: up_irq_save, up_irq_restore, and friends. * @@ -571,17 +584,22 @@ static inline_function uint32_t up_getsp(void) return sp; } -#endif /* __ASSEMBLY__ */ +noinstrument_function +static inline_function uint32_t *up_current_regs(void) +{ + return (uint32_t *)g_current_regs[up_cpu_index()]; +} -/**************************************************************************** - * Public Data - ****************************************************************************/ +noinstrument_function +static inline_function void up_set_current_regs(uint32_t *regs) +{ + g_current_regs[up_cpu_index()] = regs; +} /**************************************************************************** * Public Function Prototypes ****************************************************************************/ -#ifndef __ASSEMBLY__ #ifdef __cplusplus #define EXTERN extern "C" extern "C" @@ -594,6 +612,6 @@ extern "C" #ifdef __cplusplus } #endif -#endif +#endif /* __ASSEMBLY__ */ #endif /* __ARCH_ARM_INCLUDE_ARMV7_M_IRQ_H */ diff --git a/arch/arm/include/armv7-r/irq.h b/arch/arm/include/armv7-r/irq.h index a19070b79abd8..fb2e684f827ef 100644 --- a/arch/arm/include/armv7-r/irq.h +++ b/arch/arm/include/armv7-r/irq.h @@ -249,7 +249,6 @@ struct xcpt_syscall_s * For a total of 17 (XCPTCONTEXT_REGS) */ -#ifndef __ASSEMBLY__ struct xcptcontext { /* The following function pointer is non-zero if there are pending signals @@ -325,16 +324,11 @@ struct xcptcontext #endif #endif }; -#endif - -#endif /* __ASSEMBLY__ */ /**************************************************************************** * Inline functions ****************************************************************************/ -#ifndef __ASSEMBLY__ - /* Name: up_irq_save, up_irq_restore, and friends. * * NOTE: This function should never be called from application code and, @@ -485,13 +479,32 @@ static inline_function uint32_t up_getsp(void) return sp; } -#endif /* __ASSEMBLY__ */ +noinstrument_function +static inline_function uint32_t *up_current_regs(void) +{ + uint32_t *regs; + __asm__ __volatile__ + ( + "mrc " "p15, " "0" ", %0, " "c13" ", " "c0" ", " "4" "\n" + : "=r"(regs) + ); + return regs; +} + +noinstrument_function +static inline_function void up_set_current_regs(uint32_t *regs) +{ + __asm__ __volatile__ + ( + "mcr " "p15, " "0" ", %0, " "c13" ", " "c0" ", " "4" "\n" + :: "r"(regs) + ); +} /**************************************************************************** * Public Data ****************************************************************************/ -#ifndef __ASSEMBLY__ #ifdef __cplusplus #define EXTERN extern "C" extern "C" @@ -508,6 +521,6 @@ extern "C" #ifdef __cplusplus } #endif -#endif +#endif /* __ASSEMBLY__ */ #endif /* __ARCH_ARM_INCLUDE_ARMV7_R_IRQ_H */ diff --git a/arch/arm/include/armv8-m/irq.h b/arch/arm/include/armv8-m/irq.h index 60c43a6149d7e..4148f07644173 100644 --- a/arch/arm/include/armv8-m/irq.h +++ b/arch/arm/include/armv8-m/irq.h @@ -262,13 +262,26 @@ struct xcptcontext uint32_t *regs; }; -#endif /**************************************************************************** - * Inline functions + * Public Data ****************************************************************************/ -#ifndef __ASSEMBLY__ +/* g_current_regs[] holds a references to the current interrupt level + * register storage structure. If is non-NULL only during interrupt + * processing. Access to g_current_regs[] must be through the + * [get/set]_current_regs for portability. + */ + +/* For the case of architectures with multiple CPUs, then there must be one + * such value for each processor that can receive an interrupt. + */ + +extern volatile uint32_t *g_current_regs[CONFIG_SMP_NCPUS]; + +/**************************************************************************** + * Inline functions + ****************************************************************************/ /* Name: up_irq_save, up_irq_restore, and friends. * @@ -544,17 +557,22 @@ static inline_function uint32_t up_getsp(void) return sp; } -#endif /* __ASSEMBLY__ */ +noinstrument_function +static inline_function uint32_t *up_current_regs(void) +{ + return (uint32_t *)g_current_regs[up_cpu_index()]; +} -/**************************************************************************** - * Public Data - ****************************************************************************/ +noinstrument_function +static inline_function void up_set_current_regs(uint32_t *regs) +{ + g_current_regs[up_cpu_index()] = regs; +} /**************************************************************************** * Public Function Prototypes ****************************************************************************/ -#ifndef __ASSEMBLY__ #ifdef __cplusplus #define EXTERN extern "C" extern "C" @@ -567,6 +585,6 @@ extern "C" #ifdef __cplusplus } #endif -#endif +#endif /* __ASSEMBLY__ */ #endif /* __ARCH_ARM_INCLUDE_ARMV8_M_IRQ_H */ diff --git a/arch/arm/include/armv8-r/irq.h b/arch/arm/include/armv8-r/irq.h index 25cd71c3f3e41..4b7f7ac63e464 100644 --- a/arch/arm/include/armv8-r/irq.h +++ b/arch/arm/include/armv8-r/irq.h @@ -249,7 +249,6 @@ struct xcpt_syscall_s * For a total of 17 (XCPTCONTEXT_REGS) */ -#ifndef __ASSEMBLY__ struct xcptcontext { /* The following function pointer is non-zero if there are pending signals @@ -325,16 +324,11 @@ struct xcptcontext #endif #endif }; -#endif - -#endif /* __ASSEMBLY__ */ /**************************************************************************** * Inline functions ****************************************************************************/ -#ifndef __ASSEMBLY__ - /* Name: up_irq_save, up_irq_restore, and friends. * * NOTE: This function should never be called from application code and, @@ -485,13 +479,32 @@ static inline_function uint32_t up_getsp(void) return sp; } -#endif /* __ASSEMBLY__ */ +noinstrument_function +static inline_function uint32_t *up_current_regs(void) +{ + uint32_t *regs; + __asm__ __volatile__ + ( + "mrc " "p15, " "0" ", %0, " "c13" ", " "c0" ", " "4" "\n" + : "=r"(regs) + ); + return regs; +} + +noinstrument_function +static inline_function void up_set_current_regs(uint32_t *regs) +{ + __asm__ __volatile__ + ( + "mcr " "p15, " "0" ", %0, " "c13" ", " "c0" ", " "4" "\n" + :: "r"(regs) + ); +} /**************************************************************************** * Public Data ****************************************************************************/ -#ifndef __ASSEMBLY__ #ifdef __cplusplus #define EXTERN extern "C" extern "C" @@ -508,6 +521,6 @@ extern "C" #ifdef __cplusplus } #endif -#endif +#endif /* __ASSEMBLY__ */ #endif /* __ARCH_ARM_INCLUDE_ARMV8_R_IRQ_H */ diff --git a/arch/arm/include/irq.h b/arch/arm/include/irq.h index 19f835b84565e..c8c3cfd2c5fb9 100644 --- a/arch/arm/include/irq.h +++ b/arch/arm/include/irq.h @@ -84,40 +84,10 @@ extern "C" #define EXTERN extern #endif -/**************************************************************************** - * Public Data - ****************************************************************************/ - -/* g_current_regs[] holds a references to the current interrupt level - * register storage structure. If is non-NULL only during interrupt - * processing. Access to g_current_regs[] must be through the - * [get/set]_current_regs for portability. - */ - -/* For the case of architectures with multiple CPUs, then there must be one - * such value for each processor that can receive an interrupt. - */ - -EXTERN volatile uint32_t *g_current_regs[CONFIG_SMP_NCPUS]; - -/**************************************************************************** - * Public Function Prototypes - ****************************************************************************/ - /**************************************************************************** * Inline functions ****************************************************************************/ -static inline_function uint32_t *up_current_regs(void) -{ - return (uint32_t *)g_current_regs[up_cpu_index()]; -} - -static inline_function void up_set_current_regs(uint32_t *regs) -{ - g_current_regs[up_cpu_index()] = regs; -} - /**************************************************************************** * Name: up_interrupt_context * diff --git a/arch/arm/include/tlsr82/irq.h b/arch/arm/include/tlsr82/irq.h index 077ef8c0ed5b4..869ddf5bc1873 100644 --- a/arch/arm/include/tlsr82/irq.h +++ b/arch/arm/include/tlsr82/irq.h @@ -174,13 +174,26 @@ struct xcptcontext uint32_t *regs; }; -#endif /**************************************************************************** - * Inline functions + * Public Data ****************************************************************************/ -#ifndef __ASSEMBLY__ +/* g_current_regs[] holds a references to the current interrupt level + * register storage structure. If is non-NULL only during interrupt + * processing. Access to g_current_regs[] must be through the + * [get/set]_current_regs for portability. + */ + +/* For the case of architectures with multiple CPUs, then there must be one + * such value for each processor that can receive an interrupt. + */ + +extern volatile uint32_t *g_current_regs[CONFIG_SMP_NCPUS]; + +/**************************************************************************** + * Inline functions + ****************************************************************************/ /* Name: up_irq_save, up_irq_restore, and friends. * @@ -271,17 +284,22 @@ static inline_function uint32_t up_getsp(void) return sp; } -#endif /* __ASSEMBLY__ */ +noinstrument_function +static inline_function uint32_t *up_current_regs(void) +{ + return (uint32_t *)g_current_regs[up_cpu_index()]; +} -/**************************************************************************** - * Public Data - ****************************************************************************/ +noinstrument_function +static inline_function void up_set_current_regs(uint32_t *regs) +{ + g_current_regs[up_cpu_index()] = regs; +} /**************************************************************************** * Public Function Prototypes ****************************************************************************/ -#ifndef __ASSEMBLY__ #ifdef __cplusplus #define EXTERN extern "C" extern "C" @@ -294,6 +312,6 @@ extern "C" #ifdef __cplusplus } #endif -#endif +#endif /* __ASSEMBLY__ */ #endif /* __ARCH_ARM_INCLUDE_TLSR82_IRQ_H */ diff --git a/arch/arm/src/armv7-a/arm_cpuhead.S b/arch/arm/src/armv7-a/arm_cpuhead.S index bed632ae629ea..68369dae6b2b4 100644 --- a/arch/arm/src/armv7-a/arm_cpuhead.S +++ b/arch/arm/src/armv7-a/arm_cpuhead.S @@ -194,6 +194,7 @@ __cpu3_start: */ mov r0, #0 + mcr CP15_TPIDRPRW(r0) /* Initialize percpu reg TPIDRPRW */ #ifdef CONFIG_ARM_HAVE_MPCORE mcr CP15_TLBIALLIS(r0) /* Invalidate the entire unified TLB */ mcr CP15_BPIALLIS(r0) /* Invalidate entire branch prediction array */ diff --git a/arch/arm/src/armv7-a/arm_head.S b/arch/arm/src/armv7-a/arm_head.S index a469f9af52d61..804d096a2ce81 100644 --- a/arch/arm/src/armv7-a/arm_head.S +++ b/arch/arm/src/armv7-a/arm_head.S @@ -367,6 +367,7 @@ __cpu0_start: */ mov r0, #0 + mcr CP15_TPIDRPRW(r0) /* Initialize percpu reg TPIDRPRW */ #ifdef CONFIG_ARM_HAVE_MPCORE mcr CP15_TLBIALLIS(r0) /* Invalidate the entire unified TLB */ mcr CP15_BPIALLIS(r0) /* Invalidate entire branch prediction array */ diff --git a/arch/arm/src/armv7-r/arm_cpuhead.S b/arch/arm/src/armv7-r/arm_cpuhead.S index 338506c68b30e..473e743c36e06 100644 --- a/arch/arm/src/armv7-r/arm_cpuhead.S +++ b/arch/arm/src/armv7-r/arm_cpuhead.S @@ -220,6 +220,7 @@ __cpu3_start: */ mov r0, #0 + mcr CP15_TPIDRPRW(r0) /* Initialize percpu reg TPIDRPRW */ mcr CP15_BPIALL(r0) /* Invalidate entire branch prediction array */ mcr CP15_ICIALLU(r0) /* Invalidate I-cache */ mcr CP15_DCIALLU(r0) /* Invalidate D-cache */ diff --git a/arch/arm/src/armv7-r/arm_head.S b/arch/arm/src/armv7-r/arm_head.S index cd298688dc91d..180902d809092 100644 --- a/arch/arm/src/armv7-r/arm_head.S +++ b/arch/arm/src/armv7-r/arm_head.S @@ -174,6 +174,7 @@ __cpu0_start: */ mov r0, #0 + mcr CP15_TPIDRPRW(r0) /* Initialize percpu reg TPIDRPRW */ mcr CP15_BPIALL(r0) /* Invalidate entire branch prediction array */ mcr CP15_ICIALLU(r0) /* Invalidate I-cache */ mcr CP15_DCIALLU(r0) /* Invalidate D-cache */ diff --git a/arch/arm/src/armv8-r/arm_head.S b/arch/arm/src/armv8-r/arm_head.S index 16819dc32c824..02dbfc14a060f 100644 --- a/arch/arm/src/armv8-r/arm_head.S +++ b/arch/arm/src/armv8-r/arm_head.S @@ -197,6 +197,7 @@ __cpu0_start: */ mov r0, #0 + mcr CP15_TPIDRPRW(r0) /* Initialize percpu reg TPIDRPRW */ mcr CP15_BPIALL(r0) /* Invalidate entire branch prediction array */ mcr CP15_ICIALLU(r0) /* Invalidate I-cache */ mov r1, CP15_CACHE_INVALIDATE diff --git a/arch/arm/src/common/arm_initialize.c b/arch/arm/src/common/arm_initialize.c index 89eff35d38a5f..fa79353698e67 100644 --- a/arch/arm/src/common/arm_initialize.c +++ b/arch/arm/src/common/arm_initialize.c @@ -38,7 +38,10 @@ * [get/set]_current_regs for portability. */ +#if defined(CONFIG_ARCH_ARMV7M) || defined(CONFIG_ARCH_ARMV8M) || \ + defined(CONFIG_ARCH_ARMV6M) || defined(CONFIG_ARCH_ARM) volatile uint32_t *g_current_regs[CONFIG_SMP_NCPUS]; +#endif /**************************************************************************** * Private Functions diff --git a/arch/arm64/include/irq.h b/arch/arm64/include/irq.h index 5708be61c0187..c5d941f46798d 100644 --- a/arch/arm64/include/irq.h +++ b/arch/arm64/include/irq.h @@ -145,12 +145,10 @@ #define REG_SPSR (33) #define REG_SP_EL0 (34) #define REG_EXE_DEPTH (35) -#define REG_TPIDR_EL0 (36) -#define REG_TPIDR_EL1 (37) /* In Armv8-A Architecture, the stack must align with 16 byte */ -#define XCPTCONTEXT_GP_REGS (38) +#define XCPTCONTEXT_GP_REGS (36) #define XCPTCONTEXT_GP_SIZE (8 * XCPTCONTEXT_GP_REGS) #ifdef CONFIG_ARCH_FPU @@ -242,24 +240,6 @@ extern "C" * Public Data ****************************************************************************/ -/* g_current_regs[] holds a references to the current interrupt level - * register storage structure. It is non-NULL only during interrupt - * processing. Access to g_current_regs[] must be through the macro - * current_regs for portability. - */ - -/* For the case of architectures with multiple CPUs, then there must be one - * such value for each processor that can receive an interrupt. - */ - -EXTERN volatile uint64_t *g_current_regs[CONFIG_SMP_NCPUS]; - -/**************************************************************************** - * Public Types - ****************************************************************************/ - -#ifndef __ASSEMBLY__ - struct xcptcontext { /* The following function pointer is non-zero if there are pending signals @@ -328,8 +308,6 @@ struct xcptcontext #endif }; -#endif /* __ASSEMBLY__ */ - /* Name: up_irq_save, up_irq_restore, and friends. * * NOTE: This function should never be called from application code and, @@ -418,14 +396,31 @@ static inline void up_irq_restore(irqstate_t flags) # define up_cpu_index() (0) #endif +/**************************************************************************** + * Name: + * up_current_regs/up_set_current_regs + * + * Description: + * We use the following code to manipulate the tpidr_el1 register, + * which exists uniquely for each CPU and is primarily designed to store + * current thread information. Currently, we leverage it to store interrupt + * information, with plans to further optimize its use for storing both + * thread and interrupt information in the future. + * + ****************************************************************************/ + +noinstrument_function static inline_function uint64_t *up_current_regs(void) { - return (uint64_t *)g_current_regs[up_cpu_index()]; + uint64_t *regs; + __asm__ volatile ("mrs %0, " "tpidr_el1" : "=r" (regs)); + return regs; } +noinstrument_function static inline_function void up_set_current_regs(uint64_t *regs) { - g_current_regs[up_cpu_index()] = regs; + __asm__ volatile ("msr " "tpidr_el1" ", %0" : : "r" (regs)); } /**************************************************************************** diff --git a/arch/arm64/src/common/arm64_arch.h b/arch/arm64/src/common/arm64_arch.h index 5b02f815faeff..06935472fb595 100644 --- a/arch/arm64/src/common/arm64_arch.h +++ b/arch/arm64/src/common/arm64_arch.h @@ -299,8 +299,6 @@ struct regs_context uint64_t spsr; uint64_t sp_el0; uint64_t exe_depth; - uint64_t tpidr_el0; - uint64_t tpidr_el1; }; /**************************************************************************** @@ -449,13 +447,10 @@ void modifyreg32(unsigned int addr, uint32_t clearbits, uint32_t setbits); /**************************************************************************** * Name: * arch_get_exception_depth - * arch_get_current_tcb * * Description: * tpidrro_el0 is used to record exception depth, it's used for fpu trap * happened at exception context (like IRQ). - * tpidr_el1 is used to record TCB at present, it's used for fpu and task - * switch propose * ****************************************************************************/ @@ -464,11 +459,6 @@ static inline int arch_get_exception_depth(void) return read_sysreg(tpidrro_el0); } -static inline uint64_t arch_get_current_tcb(void) -{ - return read_sysreg(tpidr_el1); -} - void arch_cpu_idle(void); /**************************************************************************** diff --git a/arch/arm64/src/common/arm64_backtrace.c b/arch/arm64/src/common/arm64_backtrace.c index 206da6c47de3f..e3cfe0d8cbc79 100644 --- a/arch/arm64/src/common/arm64_backtrace.c +++ b/arch/arm64/src/common/arm64_backtrace.c @@ -115,7 +115,7 @@ static int backtrace(uintptr_t *base, uintptr_t *limit, int up_backtrace(struct tcb_s *tcb, void **buffer, int size, int skip) { - struct tcb_s *rtcb = (struct tcb_s *)arch_get_current_tcb(); + struct tcb_s *rtcb = running_task(); struct regs_context * p_regs; int ret; diff --git a/arch/arm64/src/common/arm64_copystate.c b/arch/arm64/src/common/arm64_copystate.c index 8f3716f915e20..fbb04f2c733cc 100644 --- a/arch/arm64/src/common/arm64_copystate.c +++ b/arch/arm64/src/common/arm64_copystate.c @@ -35,6 +35,7 @@ #include #include "arm64_internal.h" +#include "sched/sched.h" #ifdef CONFIG_ARCH_FPU #include "arm64_fpu.h" @@ -84,8 +85,7 @@ int arm64_syscall_save_context(uint64_t * regs) #ifdef CONFIG_ARCH_FPU uint64_t *p_fpu; struct tcb_s *rtcb; - struct tcb_s *rtcb_cur = - (struct tcb_s *)arch_get_current_tcb(); + struct tcb_s *rtcb_cur = running_task(); #endif DEBUGASSERT(regs); diff --git a/arch/arm64/src/common/arm64_cpustart.c b/arch/arm64/src/common/arm64_cpustart.c index a87ba4dbd0e85..975fffdfee7e2 100644 --- a/arch/arm64/src/common/arm64_cpustart.c +++ b/arch/arm64/src/common/arm64_cpustart.c @@ -137,8 +137,7 @@ static void arm64_smp_init_top(void *arg) /* core n, idle n */ write_sysreg(0, tpidrro_el0); - write_sysreg(tcb, tpidr_el1); - write_sysreg(tcb, tpidr_el0); + UNUSED(tcb); cpu_boot_params.cpu_ready_flag = 1; SP_SEV(); diff --git a/arch/arm64/src/common/arm64_fork.c b/arch/arm64/src/common/arm64_fork.c index 7fdd290063b89..faddc4d4d3ffc 100644 --- a/arch/arm64/src/common/arm64_fork.c +++ b/arch/arm64/src/common/arm64_fork.c @@ -244,8 +244,6 @@ pid_t arm64_fork(const struct fork_s *context) #else pforkctx->sp_el0 = (uint64_t)pforkctx; #endif - pforkctx->tpidr_el0 = (uint64_t)(&child->cmn); - pforkctx->tpidr_el1 = (uint64_t)(&child->cmn); child->cmn.xcp.regs = (uint64_t *)pforkctx; diff --git a/arch/arm64/src/common/arm64_fpu.c b/arch/arm64/src/common/arm64_fpu.c index 69b458a9c524f..c1accb48ebfa5 100644 --- a/arch/arm64/src/common/arm64_fpu.c +++ b/arch/arm64/src/common/arm64_fpu.c @@ -348,7 +348,7 @@ void arm64_fpu_trap(struct regs_context *regs) } else { - owner = (struct tcb_s *)arch_get_current_tcb(); + owner = running_task(); } /* restore our context */ @@ -363,7 +363,7 @@ void arm64_fpu_trap(struct regs_context *regs) void arm64_fpu_context_restore(void) { - struct tcb_s *new_tcb = (struct tcb_s *)arch_get_current_tcb(); + struct tcb_s *new_tcb = running_task(); arm64_fpu_access_trap_disable(); @@ -384,7 +384,7 @@ void arm64_fpu_context_restore(void) #ifdef CONFIG_SMP void arm64_fpu_context_save(void) { - struct tcb_s *tcb = (struct tcb_s *)arch_get_current_tcb(); + struct tcb_s *tcb = running_task(); if (tcb == g_cpu_fpu_ctx[this_cpu()].fpu_owner) { diff --git a/arch/arm64/src/common/arm64_head.S b/arch/arm64/src/common/arm64_head.S index 53ceeb93c4b97..7e8a9e9e18845 100644 --- a/arch/arm64/src/common/arm64_head.S +++ b/arch/arm64/src/common/arm64_head.S @@ -173,6 +173,9 @@ real_start: bne .loop # endif + /* Initialize percpu reg tpidr_el1 */ + + msr tpidr_el1, xzr ldr x25, =arm64_boot_secondary_c_routine bl __reset_prep_c @@ -201,6 +204,9 @@ primary_core: ldr x24, =(g_idle_stack + CONFIG_IDLETHREAD_STACKSIZE) #endif /* CONFIG_SMP */ + /* Initialize percpu reg tpidr_el1 */ + + msr tpidr_el1, xzr ldr x25, =arm64_boot_primary_c_routine /* Prepare for calling C code */ diff --git a/arch/arm64/src/common/arm64_initialize.c b/arch/arm64/src/common/arm64_initialize.c index 3f731f7d759f6..fdceba25e973b 100644 --- a/arch/arm64/src/common/arm64_initialize.c +++ b/arch/arm64/src/common/arm64_initialize.c @@ -50,18 +50,6 @@ * Public data ****************************************************************************/ -/* g_current_regs[] holds a references to the current interrupt level - * register storage structure. It is non-NULL only during interrupt - * processing. Access to g_current_regs[] must be through the macro - * current_regs for portability. - */ - -/* For the case of configurations with multiple CPUs, then there must be one - * such value for each processor that can receive an interrupt. - */ - -volatile uint64_t *g_current_regs[CONFIG_SMP_NCPUS]; - #ifdef CONFIG_ARCH_FPU static struct notifier_block g_fpu_panic_block; #endif diff --git a/arch/arm64/src/common/arm64_initialstate.c b/arch/arm64/src/common/arm64_initialstate.c index 6e10432a43659..93aa822fff6ec 100644 --- a/arch/arm64/src/common/arm64_initialstate.c +++ b/arch/arm64/src/common/arm64_initialstate.c @@ -104,8 +104,6 @@ void arm64_new_task(struct tcb_s * tcb) pinitctx->sp_el0 = (uint64_t)pinitctx; #endif pinitctx->exe_depth = 0; - pinitctx->tpidr_el0 = (uint64_t)tcb; - pinitctx->tpidr_el1 = (uint64_t)tcb; tcb->xcp.regs = (uint64_t *)pinitctx; @@ -162,8 +160,6 @@ void up_initial_state(struct tcb_s *tcb) */ write_sysreg(0, tpidrro_el0); - write_sysreg(tcb, tpidr_el1); - write_sysreg(tcb, tpidr_el0); #ifdef CONFIG_STACK_COLORATION diff --git a/arch/arm64/src/common/arm64_registerdump.c b/arch/arm64/src/common/arm64_registerdump.c index 5db120a5524b7..76e6e05635573 100644 --- a/arch/arm64/src/common/arm64_registerdump.c +++ b/arch/arm64/src/common/arm64_registerdump.c @@ -101,7 +101,5 @@ void up_dump_register(void *dumpregs) _alert("ELR: 0x%-16"PRIx64"\n", regs->elr); _alert("SP_EL0: 0x%-16"PRIx64"\n", regs->sp_el0); _alert("SP_ELX: 0x%-16"PRIx64"\n", regs->sp_elx); - _alert("TPIDR_EL0: 0x%-16"PRIx64"\n", regs->tpidr_el0); - _alert("TPIDR_EL1: 0x%-16"PRIx64"\n", regs->tpidr_el1); _alert("EXE_DEPTH: 0x%-16"PRIx64"\n", regs->exe_depth); } diff --git a/arch/arm64/src/common/arm64_schedulesigaction.c b/arch/arm64/src/common/arm64_schedulesigaction.c index bc35117fe6335..888c8106268dd 100644 --- a/arch/arm64/src/common/arm64_schedulesigaction.c +++ b/arch/arm64/src/common/arm64_schedulesigaction.c @@ -86,8 +86,7 @@ void arm64_init_signal_process(struct tcb_s *tcb, struct regs_context *regs) psigctx->sp_el0 = (uint64_t)psigctx; #endif psigctx->exe_depth = 1; - psigctx->tpidr_el0 = (uint64_t)tcb; - psigctx->tpidr_el1 = (uint64_t)tcb; + tcb->xcp.regs = (uint64_t *)psigctx; } diff --git a/arch/arm64/src/common/arm64_vectors.S b/arch/arm64/src/common/arm64_vectors.S index 025e4cb3372d4..16f75106b2095 100644 --- a/arch/arm64/src/common/arm64_vectors.S +++ b/arch/arm64/src/common/arm64_vectors.S @@ -48,12 +48,6 @@ mrs \xreg1, tpidrro_el0 stp \xreg0, \xreg1, [\xfp, #8 * REG_SP_EL0] - /* Save the TPIDR0/TPIDR1, which is the current tcb */ - - mrs \xreg0, tpidr_el0 - mrs \xreg1, tpidr_el1 - stp \xreg0, \xreg1, [\xfp, #8 * REG_TPIDR_EL0] - .endm /**************************************************************************** @@ -141,12 +135,6 @@ SECTION_FUNC(text, arm64_context_switch) mrs x5, tpidrro_el0 stp x4, x5, [x1, #8 * REG_SP_EL0] - /* Save the TPIDR0/TPIDR1, which is the current tcb */ - - mrs x4, tpidr_el0 - mrs x5, tpidr_el1 - stp x4, x5, [x1, #8 * REG_TPIDR_EL0] - restore_new: /* Restore SP_EL0 and thread's exception dept */ @@ -154,12 +142,6 @@ restore_new: msr tpidrro_el0, x5 msr sp_el0, x4 - /* restore the TPIDR0/TPIDR1 */ - - ldp x4, x5, [x0, #8 * REG_TPIDR_EL0] - msr tpidr_el0, x4 - msr tpidr_el1, x5 - /* retrieve new thread's SP_ELx */ ldr x4, [x0, #8 * REG_SP_ELX] sub sp, x4, #8 * XCPTCONTEXT_GP_REGS From 5b1700bbc9a2f590a923a3530db010b7b251fb27 Mon Sep 17 00:00:00 2001 From: hujun5 Date: Thu, 21 Mar 2024 09:57:05 +0800 Subject: [PATCH 2/3] arch: move up_interrupt_context to arch specific irq.h Signed-off-by: hujun5 --- arch/arm/include/arm/irq.h | 16 ++++++++++++++++ arch/arm/include/armv6-m/irq.h | 16 ++++++++++++++++ arch/arm/include/armv7-a/irq.h | 6 ++++++ arch/arm/include/armv7-m/irq.h | 16 ++++++++++++++++ arch/arm/include/armv7-r/irq.h | 6 ++++++ arch/arm/include/armv8-m/irq.h | 16 ++++++++++++++++ arch/arm/include/armv8-r/irq.h | 6 ++++++ arch/arm/include/irq.h | 28 ---------------------------- arch/arm/include/tlsr82/irq.h | 16 ++++++++++++++++ arch/arm64/include/irq.h | 12 +----------- 10 files changed, 99 insertions(+), 39 deletions(-) diff --git a/arch/arm/include/arm/irq.h b/arch/arm/include/arm/irq.h index eb0450ab8d925..a3321d7c62c05 100644 --- a/arch/arm/include/arm/irq.h +++ b/arch/arm/include/arm/irq.h @@ -267,6 +267,22 @@ static inline_function void up_set_current_regs(uint32_t *regs) g_current_regs[up_cpu_index()] = regs; } +noinstrument_function +static inline_function bool up_interrupt_context(void) +{ +#ifdef CONFIG_SMP + irqstate_t flags = up_irq_save(); +#endif + + bool ret = up_current_regs() != NULL; + +#ifdef CONFIG_SMP + up_irq_restore(flags); +#endif + + return ret; +} + /**************************************************************************** * Public Function Prototypes ****************************************************************************/ diff --git a/arch/arm/include/armv6-m/irq.h b/arch/arm/include/armv6-m/irq.h index cc6fc4fd86198..ee40eb119b73f 100644 --- a/arch/arm/include/armv6-m/irq.h +++ b/arch/arm/include/armv6-m/irq.h @@ -391,6 +391,22 @@ static inline_function void up_set_current_regs(uint32_t *regs) g_current_regs[up_cpu_index()] = regs; } +noinstrument_function +static inline_function bool up_interrupt_context(void) +{ +#ifdef CONFIG_SMP + irqstate_t flags = up_irq_save(); +#endif + + bool ret = up_current_regs() != NULL; + +#ifdef CONFIG_SMP + up_irq_restore(flags); +#endif + + return ret; +} + /**************************************************************************** * Public Function Prototypes ****************************************************************************/ diff --git a/arch/arm/include/armv7-a/irq.h b/arch/arm/include/armv7-a/irq.h index 61c28ef0690b5..c7876f5f4f3d9 100644 --- a/arch/arm/include/armv7-a/irq.h +++ b/arch/arm/include/armv7-a/irq.h @@ -519,6 +519,12 @@ static inline_function void up_set_current_regs(uint32_t *regs) ); } +noinstrument_function +static inline_function bool up_interrupt_context(void) +{ + return up_current_regs() != NULL; +} + /**************************************************************************** * Public Data ****************************************************************************/ diff --git a/arch/arm/include/armv7-m/irq.h b/arch/arm/include/armv7-m/irq.h index 333b95d3ad454..53900703b7721 100644 --- a/arch/arm/include/armv7-m/irq.h +++ b/arch/arm/include/armv7-m/irq.h @@ -596,6 +596,22 @@ static inline_function void up_set_current_regs(uint32_t *regs) g_current_regs[up_cpu_index()] = regs; } +noinstrument_function +static inline_function bool up_interrupt_context(void) +{ +#ifdef CONFIG_SMP + irqstate_t flags = up_irq_save(); +#endif + + bool ret = up_current_regs() != NULL; + +#ifdef CONFIG_SMP + up_irq_restore(flags); +#endif + + return ret; +} + /**************************************************************************** * Public Function Prototypes ****************************************************************************/ diff --git a/arch/arm/include/armv7-r/irq.h b/arch/arm/include/armv7-r/irq.h index fb2e684f827ef..1b6d6e1b46131 100644 --- a/arch/arm/include/armv7-r/irq.h +++ b/arch/arm/include/armv7-r/irq.h @@ -501,6 +501,12 @@ static inline_function void up_set_current_regs(uint32_t *regs) ); } +noinstrument_function +static inline_function bool up_interrupt_context(void) +{ + return up_current_regs() != NULL; +} + /**************************************************************************** * Public Data ****************************************************************************/ diff --git a/arch/arm/include/armv8-m/irq.h b/arch/arm/include/armv8-m/irq.h index 4148f07644173..69763c866448e 100644 --- a/arch/arm/include/armv8-m/irq.h +++ b/arch/arm/include/armv8-m/irq.h @@ -569,6 +569,22 @@ static inline_function void up_set_current_regs(uint32_t *regs) g_current_regs[up_cpu_index()] = regs; } +noinstrument_function +static inline_function bool up_interrupt_context(void) +{ +#ifdef CONFIG_SMP + irqstate_t flags = up_irq_save(); +#endif + + bool ret = up_current_regs() != NULL; + +#ifdef CONFIG_SMP + up_irq_restore(flags); +#endif + + return ret; +} + /**************************************************************************** * Public Function Prototypes ****************************************************************************/ diff --git a/arch/arm/include/armv8-r/irq.h b/arch/arm/include/armv8-r/irq.h index 4b7f7ac63e464..53fb064d84db3 100644 --- a/arch/arm/include/armv8-r/irq.h +++ b/arch/arm/include/armv8-r/irq.h @@ -501,6 +501,12 @@ static inline_function void up_set_current_regs(uint32_t *regs) ); } +noinstrument_function +static inline_function bool up_interrupt_context(void) +{ + return up_current_regs() != NULL; +} + /**************************************************************************** * Public Data ****************************************************************************/ diff --git a/arch/arm/include/irq.h b/arch/arm/include/irq.h index c8c3cfd2c5fb9..e859604c6b18a 100644 --- a/arch/arm/include/irq.h +++ b/arch/arm/include/irq.h @@ -84,34 +84,6 @@ extern "C" #define EXTERN extern #endif -/**************************************************************************** - * Inline functions - ****************************************************************************/ - -/**************************************************************************** - * Name: up_interrupt_context - * - * Description: - * Return true is we are currently executing in the interrupt - * handler context. - * - ****************************************************************************/ - -noinstrument_function -static inline bool up_interrupt_context(void) -{ -#ifdef CONFIG_SMP - irqstate_t flags = up_irq_save(); -#endif - - bool ret = up_current_regs() != NULL; - -#ifdef CONFIG_SMP - up_irq_restore(flags); -#endif - - return ret; -} #endif /* __ASSEMBLY__ */ #undef EXTERN diff --git a/arch/arm/include/tlsr82/irq.h b/arch/arm/include/tlsr82/irq.h index 869ddf5bc1873..72cb08c9bfcf9 100644 --- a/arch/arm/include/tlsr82/irq.h +++ b/arch/arm/include/tlsr82/irq.h @@ -296,6 +296,22 @@ static inline_function void up_set_current_regs(uint32_t *regs) g_current_regs[up_cpu_index()] = regs; } +noinstrument_function +static inline_function bool up_interrupt_context(void) +{ +#ifdef CONFIG_SMP + irqstate_t flags = up_irq_save(); +#endif + + bool ret = up_current_regs() != NULL; + +#ifdef CONFIG_SMP + up_irq_restore(flags); +#endif + + return ret; +} + /**************************************************************************** * Public Function Prototypes ****************************************************************************/ diff --git a/arch/arm64/include/irq.h b/arch/arm64/include/irq.h index c5d941f46798d..19ed567ad18e6 100644 --- a/arch/arm64/include/irq.h +++ b/arch/arm64/include/irq.h @@ -433,17 +433,7 @@ static inline_function void up_set_current_regs(uint64_t *regs) static inline bool up_interrupt_context(void) { -#ifdef CONFIG_SMP - irqstate_t flags = up_irq_save(); -#endif - - bool ret = (up_current_regs() != NULL); - -#ifdef CONFIG_SMP - up_irq_restore(flags); -#endif - - return ret; + return up_current_regs() != NULL; } #undef EXTERN From 9f45b975fd529761770941e8f2a3b37bd523c3b4 Mon Sep 17 00:00:00 2001 From: hujun5 Date: Thu, 28 Mar 2024 09:32:14 +0800 Subject: [PATCH 3/3] arm: optimize up_interrupt_context used in armv[6/7/8]-m resson: using percpu storage for g_current_regs or leveraging interrupt status registers to determine if code is running within an interrupt context can enhance performance. Signed-off-by: hujun5 --- arch/arm/include/armv6-m/irq.h | 12 +----------- arch/arm/include/armv7-m/irq.h | 12 +----------- arch/arm/include/armv8-m/irq.h | 12 +----------- 3 files changed, 3 insertions(+), 33 deletions(-) diff --git a/arch/arm/include/armv6-m/irq.h b/arch/arm/include/armv6-m/irq.h index ee40eb119b73f..752e3ed394008 100644 --- a/arch/arm/include/armv6-m/irq.h +++ b/arch/arm/include/armv6-m/irq.h @@ -394,17 +394,7 @@ static inline_function void up_set_current_regs(uint32_t *regs) noinstrument_function static inline_function bool up_interrupt_context(void) { -#ifdef CONFIG_SMP - irqstate_t flags = up_irq_save(); -#endif - - bool ret = up_current_regs() != NULL; - -#ifdef CONFIG_SMP - up_irq_restore(flags); -#endif - - return ret; + return getipsr() != 0; } /**************************************************************************** diff --git a/arch/arm/include/armv7-m/irq.h b/arch/arm/include/armv7-m/irq.h index 53900703b7721..88db5d3ba36d0 100644 --- a/arch/arm/include/armv7-m/irq.h +++ b/arch/arm/include/armv7-m/irq.h @@ -599,17 +599,7 @@ static inline_function void up_set_current_regs(uint32_t *regs) noinstrument_function static inline_function bool up_interrupt_context(void) { -#ifdef CONFIG_SMP - irqstate_t flags = up_irq_save(); -#endif - - bool ret = up_current_regs() != NULL; - -#ifdef CONFIG_SMP - up_irq_restore(flags); -#endif - - return ret; + return getipsr() != 0; } /**************************************************************************** diff --git a/arch/arm/include/armv8-m/irq.h b/arch/arm/include/armv8-m/irq.h index 69763c866448e..f14d84f6553e8 100644 --- a/arch/arm/include/armv8-m/irq.h +++ b/arch/arm/include/armv8-m/irq.h @@ -572,17 +572,7 @@ static inline_function void up_set_current_regs(uint32_t *regs) noinstrument_function static inline_function bool up_interrupt_context(void) { -#ifdef CONFIG_SMP - irqstate_t flags = up_irq_save(); -#endif - - bool ret = up_current_regs() != NULL; - -#ifdef CONFIG_SMP - up_irq_restore(flags); -#endif - - return ret; + return getipsr() != 0; } /****************************************************************************