diff options
| author | Catalin Marinas <catalin.marinas@arm.com> | 2021-04-15 15:00:48 +0200 |
|---|---|---|
| committer | Catalin Marinas <catalin.marinas@arm.com> | 2021-04-15 15:00:48 +0200 |
| commit | a27a8816568964fcef62a3ae5f9d2228ec1ebc68 (patch) | |
| tree | 7197c94bcf5b600bb43e7aa70ba2b05a7fb82cd1 /arch/arm64/kernel/entry.S | |
| parent | Merge branch 'for-next/mte-async-kernel-mode' into for-next/core (diff) | |
| parent | arm64: pac: Optimize kernel entry/exit key installation code paths (diff) | |
| download | linux-a27a8816568964fcef62a3ae5f9d2228ec1ebc68.tar.xz linux-a27a8816568964fcef62a3ae5f9d2228ec1ebc68.zip | |
Merge branch 'for-next/pac-set-get-enabled-keys' into for-next/core
* for-next/pac-set-get-enabled-keys:
: Introduce arm64 prctl(PR_PAC_{SET,GET}_ENABLED_KEYS).
arm64: pac: Optimize kernel entry/exit key installation code paths
arm64: Introduce prctl(PR_PAC_{SET,GET}_ENABLED_KEYS)
arm64: mte: make the per-task SCTLR_EL1 field usable elsewhere
Diffstat (limited to 'arch/arm64/kernel/entry.S')
| -rw-r--r-- | arch/arm64/kernel/entry.S | 46 |
1 files changed, 43 insertions, 3 deletions
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index a45b4ebbfe7d..2df32a516ffe 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -247,7 +247,29 @@ alternative_else_nop_endif check_mte_async_tcf x19, x22 apply_ssbd 1, x22, x23 - ptrauth_keys_install_kernel tsk, x20, x22, x23 +#ifdef CONFIG_ARM64_PTR_AUTH +alternative_if ARM64_HAS_ADDRESS_AUTH + /* + * Enable IA for in-kernel PAC if the task had it disabled. Although + * this could be implemented with an unconditional MRS which would avoid + * a load, this was measured to be slower on Cortex-A75 and Cortex-A76. + * + * Install the kernel IA key only if IA was enabled in the task. If IA + * was disabled on kernel exit then we would have left the kernel IA + * installed so there is no need to install it again. + */ + ldr x0, [tsk, THREAD_SCTLR_USER] + tbz x0, SCTLR_ELx_ENIA_SHIFT, 1f + __ptrauth_keys_install_kernel_nosync tsk, x20, x22, x23 + b 2f +1: + mrs x0, sctlr_el1 + orr x0, x0, SCTLR_ELx_ENIA + msr sctlr_el1, x0 +2: + isb +alternative_else_nop_endif +#endif mte_set_kernel_gcr x22, x23 @@ -351,8 +373,26 @@ alternative_else_nop_endif 3: scs_save tsk, x0 - /* No kernel C function calls after this as user keys are set. */ - ptrauth_keys_install_user tsk, x0, x1, x2 +#ifdef CONFIG_ARM64_PTR_AUTH +alternative_if ARM64_HAS_ADDRESS_AUTH + /* + * IA was enabled for in-kernel PAC. Disable it now if needed, or + * alternatively install the user's IA. All other per-task keys and + * SCTLR bits were updated on task switch. + * + * No kernel C function calls after this. + */ + ldr x0, [tsk, THREAD_SCTLR_USER] + tbz x0, SCTLR_ELx_ENIA_SHIFT, 1f + __ptrauth_keys_install_user tsk, x0, x1, x2 + b 2f +1: + mrs x0, sctlr_el1 + bic x0, x0, SCTLR_ELx_ENIA + msr sctlr_el1, x0 +2: +alternative_else_nop_endif +#endif mte_set_user_gcr tsk, x0, x1 |