1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
|
// SPDX-License-Identifier: GPL-2.0
#include <linux/ftrace.h>
#include <linux/percpu.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/pgtable.h>
#include <linux/cpuidle.h>
#include <asm/alternative.h>
#include <asm/cacheflush.h>
#include <asm/cpufeature.h>
#include <asm/cpuidle.h>
#include <asm/daifflags.h>
#include <asm/debug-monitors.h>
#include <asm/exec.h>
#include <asm/mte.h>
#include <asm/memory.h>
#include <asm/mmu_context.h>
#include <asm/smp_plat.h>
#include <asm/suspend.h>
/*
* This is allocated by cpu_suspend_init(), and used to store a pointer to
* the 'struct sleep_stack_data' the contains a particular CPUs state.
*/
unsigned long *sleep_save_stash;
/*
* This hook is provided so that cpu_suspend code can restore HW
* breakpoints as early as possible in the resume path, before reenabling
* debug exceptions. Code cannot be run from a CPU PM notifier since by the
* time the notifier runs debug exceptions might have been enabled already,
* with HW breakpoints registers content still in an unknown state.
*/
static int (*hw_breakpoint_restore)(unsigned int);
void __init cpu_suspend_set_dbg_restorer(int (*hw_bp_restore)(unsigned int))
{
/* Prevent multiple restore hook initializations */
if (WARN_ON(hw_breakpoint_restore))
return;
hw_breakpoint_restore = hw_bp_restore;
}
void notrace __cpu_suspend_exit(void)
{
unsigned int cpu = smp_processor_id();
mte_suspend_exit();
/*
* We are resuming from reset with the idmap active in TTBR0_EL1.
* We must uninstall the idmap and restore the expected MMU
* state before we can possibly return to userspace.
*/
cpu_uninstall_idmap();
/* Restore CnP bit in TTBR1_EL1 */
if (system_supports_cnp())
cpu_enable_swapper_cnp();
/*
* PSTATE was not saved over suspend/resume, re-enable any detected
* features that might not have been set correctly.
*/
if (alternative_has_cap_unlikely(ARM64_HAS_DIT))
set_pstate_dit(1);
__uaccess_enable_hw_pan();
/*
* Restore HW breakpoint registers to sane values
* before debug exceptions are possibly reenabled
* by cpu_suspend()s local_daif_restore() call.
*/
if (hw_breakpoint_restore)
hw_breakpoint_restore(cpu);
/*
* On resume, firmware implementing dynamic mitigation will
* have turned the mitigation on. If the user has forcefully
* disabled it, make sure their wishes are obeyed.
*/
spectre_v4_enable_mitigation(NULL);
/* Restore additional feature-specific configuration */
ptrauth_suspend_exit();
}
/*
* cpu_suspend
*
* arg: argument to pass to the finisher function
* fn: finisher function pointer
*
*/
int cpu_suspend(unsigned long arg, int (*fn)(unsigned long))
{
int ret = 0;
unsigned long flags;
struct sleep_stack_data state;
struct arm_cpuidle_irq_context context;
/*
* Some portions of CPU state (e.g. PSTATE.{PAN,DIT}) are initialized
* before alternatives are patched, but are only restored by
* __cpu_suspend_exit() after alternatives are patched. To avoid
* accidentally losing these bits we must not attempt to suspend until
* after alternatives have been patched.
*/
WARN_ON(!system_capabilities_finalized());
/* Report any MTE async fault before going to suspend */
mte_suspend_enter();
/*
* From this point debug exceptions are disabled to prevent
* updates to mdscr register (saved and restored along with
* general purpose registers) from kernel debuggers.
*
* Strictly speaking the trace_hardirqs_off() here is superfluous,
* hardirqs should be firmly off by now. This really ought to use
* something like raw_local_daif_save().
*/
flags = local_daif_save();
/*
* Function graph tracer state gets inconsistent when the kernel
* calls functions that never return (aka suspend finishers) hence
* disable graph tracing during their execution.
*/
pause_graph_tracing();
/*
* Switch to using DAIF.IF instead of PMR in order to reliably
* resume if we're using pseudo-NMIs.
*/
arm_cpuidle_save_irq_context(&context);
ct_cpuidle_enter();
if (__cpu_suspend_enter(&state)) {
/* Call the suspend finisher */
ret = fn(arg);
/*
* Never gets here, unless the suspend finisher fails.
* Successful cpu_suspend() should return from cpu_resume(),
* returning through this code path is considered an error
* If the return value is set to 0 force ret = -EOPNOTSUPP
* to make sure a proper error condition is propagated
*/
if (!ret)
ret = -EOPNOTSUPP;
ct_cpuidle_exit();
} else {
ct_cpuidle_exit();
__cpu_suspend_exit();
}
arm_cpuidle_restore_irq_context(&context);
unpause_graph_tracing();
/*
* Restore pstate flags. OS lock and mdscr have been already
* restored, so from this point onwards, debugging is fully
* reenabled if it was enabled when core started shutdown.
*/
local_daif_restore(flags);
return ret;
}
static int __init cpu_suspend_init(void)
{
/* ctx_ptr is an array of physical addresses */
sleep_save_stash = kcalloc(mpidr_hash_size(), sizeof(*sleep_save_stash),
GFP_KERNEL);
if (WARN_ON(!sleep_save_stash))
return -ENOMEM;
return 0;
}
early_initcall(cpu_suspend_init);
|
