diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index ef5e60d6d57709e7be6b1208efdfe700ae180670..b221f2a2f52e9c0ad239b0d673eb8097d28c602b 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -672,6 +672,9 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU .set .Lframe_regcount, \regcount .set .Lframe_extra, \extra .set .Lframe_local_offset, ((\regcount + 3) / 2) * 16 +#ifdef CONFIG_ARM64_PTR_AUTH_BACK_CFI + pacib x30, sp +#endif stp x29, x30, [sp, #-.Lframe_local_offset - .Lframe_extra]! mov x29, sp .endif @@ -687,6 +690,9 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU .error "frame_push/frame_pop may not be nested" .endif ldp x29, x30, [sp], #.Lframe_local_offset + .Lframe_extra +#ifdef CONFIG_ARM64_PTR_AUTH_BACK_CFI + autib x30, sp +#endif .set .Lframe_regcount, -1 .endif .endm diff --git a/arch/arm64/kernel/efi-rt-wrapper.S b/arch/arm64/kernel/efi-rt-wrapper.S index 75691a2641c1c0f8ec05604ae6b47345fcb93e75..2da6b57dc514ba5f438b2182874f4e97ac6ee66c 100644 --- a/arch/arm64/kernel/efi-rt-wrapper.S +++ b/arch/arm64/kernel/efi-rt-wrapper.S @@ -6,6 +6,9 @@ #include SYM_FUNC_START(__efi_rt_asm_wrapper) +#ifdef CONFIG_ARM64_PTR_AUTH_BACK_CFI + pacib x30, sp +#endif stp x29, x30, [sp, #-32]! mov x29, sp @@ -32,6 +35,9 @@ SYM_FUNC_START(__efi_rt_asm_wrapper) ldp x1, x2, [sp, #16] cmp x2, x18 ldp x29, x30, [sp], #32 +#ifdef CONFIG_ARM64_PTR_AUTH_BACK_CFI + autib x30, sp +#endif b.ne 0f ret 0: diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S index 67f68c9ef94c43fb918cd493360a802c7134ad90..7f4be2b2c058f495b6cf82ef982257d1e73d3f04 100644 --- a/arch/arm64/kernel/entry-ftrace.S +++ b/arch/arm64/kernel/entry-ftrace.S @@ -177,12 +177,18 @@ SYM_CODE_END(ftrace_graph_caller) */ .macro mcount_enter +#ifdef CONFIG_ARM64_PTR_AUTH_BACK_CFI + pacib x30, sp +#endif stp x29, x30, [sp, #-16]! mov x29, sp .endm .macro mcount_exit ldp x29, x30, [sp], #16 +#ifdef CONFIG_ARM64_PTR_AUTH_BACK_CFI + autib x30, sp +#endif ret .endm diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index d5bc1dbdd2fda84cd1e6e9508c07be5b41fae793..91f06c195f46a729912d961bf8d1b8982c75eddb 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -1137,6 +1137,9 @@ SYM_CODE_START(__sdei_asm_handler) stp x26, x27, [x1, #SDEI_EVENT_INTREGS + 16 * 13] stp x28, x29, [x1, #SDEI_EVENT_INTREGS + 16 * 14] mov x4, sp +#ifdef CONFIG_ARM64_PTR_AUTH_BACK_CFI + pacib lr, sp +#endif stp lr, x4, [x1, #SDEI_EVENT_INTREGS + S_LR] mov x19, x1 @@ -1198,6 +1201,9 @@ SYM_CODE_START(__sdei_asm_handler) ldp x18, x19, [x4, #SDEI_EVENT_INTREGS + 16 * 9] ldp lr, x1, [x4, #SDEI_EVENT_INTREGS + S_LR] mov sp, x1 +#ifdef CONFIG_ARM64_PTR_AUTH_BACK_CFI + autib lr, sp +#endif mov x1, x0 // address to complete_and_resume /* x0 = (x0 <= 1) ? EVENT_COMPLETE:EVENT_COMPLETE_AND_RESUME */ diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index e1c25fa3b8e6ca6566876ece29871cefdcd73d1b..6700ab80df5bbc673fe61d7cc0c28f9c8122b62e 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -427,7 +427,9 @@ SYM_FUNC_START_LOCAL(__primary_switched) adr_l x8, vectors // load VBAR_EL1 with virtual msr vbar_el1, x8 // vector table address isb - +#ifdef CONFIG_ARM64_PTR_AUTH_BACK_CFI + pacib x30, sp +#endif stp xzr, x30, [sp, #-16]! mov x29, sp @@ -460,6 +462,9 @@ SYM_FUNC_START_LOCAL(__primary_switched) cbz x0, 0f // KASLR disabled? just proceed orr x23, x23, x0 // record KASLR offset ldp x29, x30, [sp], #16 // we must enable KASLR, return +#ifdef CONFIG_ARM64_PTR_AUTH_BACK_CFI + autib x30, sp +#endif ret // to __primary_switch() 0: #endif diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S index ba40d57757d63c10f671ba8f7e674d1e2430e161..978c0ba41ee4f01f8bf1abd80c903d3cfbe43017 100644 --- a/arch/arm64/kernel/sleep.S +++ b/arch/arm64/kernel/sleep.S @@ -63,6 +63,9 @@ * x0 = struct sleep_stack_data area */ SYM_FUNC_START(__cpu_suspend_enter) +#ifdef CONFIG_ARM64_PTR_AUTH_BACK_CFI + pacib lr, sp +#endif stp x29, lr, [x0, #SLEEP_STACK_DATA_CALLEE_REGS] stp x19, x20, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+16] stp x21, x22, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+32] @@ -90,9 +93,15 @@ SYM_FUNC_START(__cpu_suspend_enter) str x0, [x1] add x0, x0, #SLEEP_STACK_DATA_SYSTEM_REGS +#ifdef CONFIG_ARM64_PTR_AUTH_BACK_CFI + pacib lr, sp +#endif stp x29, lr, [sp, #-16]! bl cpu_do_suspend ldp x29, lr, [sp], #16 +#ifdef CONFIG_ARM64_PTR_AUTH_BACK_CFI + autib lr, sp +#endif mov x0, #1 ret SYM_FUNC_END(__cpu_suspend_enter) @@ -144,6 +153,9 @@ SYM_FUNC_START(_cpu_resume) ldp x25, x26, [x29, #64] ldp x27, x28, [x29, #80] ldp x29, lr, [x29] +#ifdef CONFIG_ARM64_PTR_AUTH_BACK_CFI + autib x30, sp +#endif mov x0, #0 ret SYM_FUNC_END(_cpu_resume)