1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
|
/*
* x86 FPU boot time init code
*/
#include <asm/fpu/internal.h>
#include <asm/tlbflush.h>
/*
* The earliest FPU detection code.
*
* Set the X86_FEATURE_FPU CPU-capability bit based on
* trying to execute an actual sequence of FPU instructions:
*/
static void fpu__init_system_early_generic(struct cpuinfo_x86 *c)
{
unsigned long cr0;
u16 fsw, fcw;
fsw = fcw = 0xffff;
cr0 = read_cr0();
cr0 &= ~(X86_CR0_TS | X86_CR0_EM);
write_cr0(cr0);
asm volatile("fninit ; fnstsw %0 ; fnstcw %1"
: "+m" (fsw), "+m" (fcw));
if (fsw == 0 && (fcw & 0x103f) == 0x003f)
set_cpu_cap(c, X86_FEATURE_FPU);
else
clear_cpu_cap(c, X86_FEATURE_FPU);
#ifndef CONFIG_MATH_EMULATION
if (!cpu_has_fpu) {
pr_emerg("No FPU found and no math emulation present\n");
pr_emerg("Giving up\n");
for (;;)
asm volatile("hlt");
}
#endif
}
/*
* Boot time FPU feature detection code:
*/
unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu;
unsigned int xstate_size;
EXPORT_SYMBOL_GPL(xstate_size);
static void fpu__init_system_mxcsr(void)
{
unsigned int mask = 0;
if (cpu_has_fxsr) {
struct i387_fxsave_struct fx_tmp __aligned(32) = { };
asm volatile("fxsave %0" : "+m" (fx_tmp));
mask = fx_tmp.mxcsr_mask;
/*
* If zero then use the default features mask,
* which has all features set, except the
* denormals-are-zero feature bit:
*/
if (mask == 0)
mask = 0x0000ffbf;
}
mxcsr_feature_mask &= mask;
}
/*
* Once per bootup FPU initialization sequences that will run on most x86 CPUs:
*/
static void fpu__init_system_generic(void)
{
/*
* Set up the legacy init FPU context. (xstate init might overwrite this
* with a more modern format, if the CPU supports it.)
*/
fx_finit(&init_xstate_ctx.i387);
fpu__init_system_mxcsr();
}
static void fpstate_xstate_init_size(void)
{
static bool on_boot_cpu = 1;
if (!on_boot_cpu)
return;
on_boot_cpu = 0;
/*
* Note that xstate_size might be overwriten later during
* fpu__init_system_xstate().
*/
if (!cpu_has_fpu) {
/*
* Disable xsave as we do not support it if i387
* emulation is enabled.
*/
setup_clear_cpu_cap(X86_FEATURE_XSAVE);
setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
xstate_size = sizeof(struct i387_soft_struct);
} else {
if (cpu_has_fxsr)
xstate_size = sizeof(struct i387_fxsave_struct);
else
xstate_size = sizeof(struct i387_fsave_struct);
}
}
/*
* Initialize the TS bit in CR0 according to the style of context-switches
* we are using:
*/
static void fpu__init_cpu_ctx_switch(void)
{
if (!cpu_has_eager_fpu)
stts();
else
clts();
}
/*
* Initialize the registers found in all CPUs, CR0 and CR4:
*/
static void fpu__init_cpu_generic(void)
{
unsigned long cr0;
unsigned long cr4_mask = 0;
if (cpu_has_fxsr)
cr4_mask |= X86_CR4_OSFXSR;
if (cpu_has_xmm)
cr4_mask |= X86_CR4_OSXMMEXCPT;
if (cr4_mask)
cr4_set_bits(cr4_mask);
cr0 = read_cr0();
cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
if (!cpu_has_fpu)
cr0 |= X86_CR0_EM;
write_cr0(cr0);
}
/*
* Enable all supported FPU features. Called when a CPU is brought online.
*/
void fpu__init_cpu(void)
{
fpu__init_cpu_generic();
fpu__init_cpu_xstate();
fpu__init_cpu_ctx_switch();
}
static enum { AUTO, ENABLE, DISABLE } eagerfpu = AUTO;
static int __init eager_fpu_setup(char *s)
{
if (!strcmp(s, "on"))
eagerfpu = ENABLE;
else if (!strcmp(s, "off"))
eagerfpu = DISABLE;
else if (!strcmp(s, "auto"))
eagerfpu = AUTO;
return 1;
}
__setup("eagerfpu=", eager_fpu_setup);
/*
* setup_init_fpu_buf() is __init and it is OK to call it here because
* init_xstate_ctx will be unset only once during boot.
*/
static void fpu__init_system_ctx_switch(void)
{
WARN_ON(current->thread.fpu.fpstate_active);
current_thread_info()->status = 0;
/* Auto enable eagerfpu for xsaveopt */
if (cpu_has_xsaveopt && eagerfpu != DISABLE)
eagerfpu = ENABLE;
if (xfeatures_mask & XSTATE_EAGER) {
if (eagerfpu == DISABLE) {
pr_err("x86/fpu: eagerfpu switching disabled, disabling the following xstate features: 0x%llx.\n",
xfeatures_mask & XSTATE_EAGER);
xfeatures_mask &= ~XSTATE_EAGER;
} else {
eagerfpu = ENABLE;
}
}
if (eagerfpu == ENABLE)
setup_force_cpu_cap(X86_FEATURE_EAGER_FPU);
printk_once(KERN_INFO "x86/fpu: Using '%s' FPU context switches.\n", eagerfpu == ENABLE ? "eager" : "lazy");
}
/*
* Called on the boot CPU once per system bootup, to set up the initial FPU state that
* is later cloned into all processes.
*/
void fpu__init_system(struct cpuinfo_x86 *c)
{
fpu__init_system_early_generic(c);
/* The FPU has to be operational for some of the later FPU init activities: */
fpu__init_cpu();
/*
* But don't leave CR0::TS set yet, as some of the FPU setup methods depend
* on being able to execute FPU instructions that will fault on a set TS,
* such as the FXSAVE in fpu__init_system_mxcsr().
*/
clts();
fpu__init_system_generic();
fpstate_xstate_init_size();
fpu__init_system_xstate();
fpu__init_system_ctx_switch();
}
static int __init no_387(char *s)
{
setup_clear_cpu_cap(X86_FEATURE_FPU);
return 1;
}
__setup("no387", no_387);
void fpu__detect(struct cpuinfo_x86 *c)
{
fpu__init_system(c);
/* The final cr0 value is set later, in fpu_init() */
}
|
