1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
|
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_S390_ALTERNATIVE_H
#define _ASM_S390_ALTERNATIVE_H
/*
* Each alternative comes with a 32 bit feature field:
* union {
* u32 feature;
* struct {
* u32 ctx : 4;
* u32 type : 8;
* u32 data : 20;
* };
* }
*
* @ctx is a bitfield, where only one bit must be set. Each bit defines
* in which context an alternative is supposed to be applied to the
* kernel image:
*
* - from the decompressor before the kernel itself is executed
* - from early kernel code from within the kernel
*
* @type is a number which defines the type and with that the type
* specific alternative patching.
*
* @data is additional type specific information which defines if an
* alternative should be applied.
*/
#define ALT_CTX_EARLY 1
#define ALT_CTX_LATE 2
#define ALT_CTX_ALL (ALT_CTX_EARLY | ALT_CTX_LATE)
#define ALT_TYPE_FACILITY 0
#define ALT_TYPE_SPEC 1
#define ALT_TYPE_LOWCORE 2
#define ALT_DATA_SHIFT 0
#define ALT_TYPE_SHIFT 20
#define ALT_CTX_SHIFT 28
#define ALT_FACILITY_EARLY(facility) (ALT_CTX_EARLY << ALT_CTX_SHIFT | \
ALT_TYPE_FACILITY << ALT_TYPE_SHIFT | \
(facility) << ALT_DATA_SHIFT)
#define ALT_FACILITY(facility) (ALT_CTX_LATE << ALT_CTX_SHIFT | \
ALT_TYPE_FACILITY << ALT_TYPE_SHIFT | \
(facility) << ALT_DATA_SHIFT)
#define ALT_SPEC(facility) (ALT_CTX_LATE << ALT_CTX_SHIFT | \
ALT_TYPE_SPEC << ALT_TYPE_SHIFT | \
(facility) << ALT_DATA_SHIFT)
#define ALT_LOWCORE (ALT_CTX_EARLY << ALT_CTX_SHIFT | \
ALT_TYPE_LOWCORE << ALT_TYPE_SHIFT)
#ifndef __ASSEMBLY__
#include <linux/types.h>
#include <linux/stddef.h>
#include <linux/stringify.h>
struct alt_instr {
s32 instr_offset; /* original instruction */
s32 repl_offset; /* offset to replacement instruction */
union {
u32 feature; /* feature required for replacement */
struct {
u32 ctx : 4; /* context */
u32 type : 8; /* type of alternative */
u32 data : 20; /* patching information */
};
};
u8 instrlen; /* length of original instruction */
} __packed;
extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
void __apply_alternatives(struct alt_instr *start, struct alt_instr *end, unsigned int ctx);
static inline void apply_alternative_instructions(void)
{
__apply_alternatives(__alt_instructions, __alt_instructions_end, ALT_CTX_LATE);
}
static inline void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
{
__apply_alternatives(start, end, ALT_CTX_ALL);
}
/*
* +---------------------------------+
* |661: |662:
* | oldinstr |
* +---------------------------------+
*
* .altinstr_replacement section
* +---------------------------------+
* |6641: |6651:
* | alternative instr 1 |
* +---------------------------------+
* |6642: |6652:
* | alternative instr 2 |
* +---------------------------------+
*
* .altinstructions section
* +---------------------------------+
* | alt_instr entries for each |
* | alternative instr |
* +---------------------------------+
*/
#define b_altinstr(num) "664"#num
#define e_altinstr(num) "665"#num
#define oldinstr_len "662b-661b"
#define altinstr_len(num) e_altinstr(num)"b-"b_altinstr(num)"b"
#define OLDINSTR(oldinstr) \
"661:\n\t" oldinstr "\n662:\n"
#define ALTINSTR_ENTRY(feature, num) \
"\t.long 661b - .\n" /* old instruction */ \
"\t.long " b_altinstr(num)"b - .\n" /* alt instruction */ \
"\t.long " __stringify(feature) "\n" /* feature */ \
"\t.byte " oldinstr_len "\n" /* instruction len */ \
"\t.org . - (" oldinstr_len ") & 1\n" \
"\t.org . - (" oldinstr_len ") + (" altinstr_len(num) ")\n" \
"\t.org . - (" altinstr_len(num) ") + (" oldinstr_len ")\n"
#define ALTINSTR_REPLACEMENT(altinstr, num) /* replacement */ \
b_altinstr(num)":\n\t" altinstr "\n" e_altinstr(num) ":\n"
/* alternative assembly primitive: */
#define ALTERNATIVE(oldinstr, altinstr, feature) \
".pushsection .altinstr_replacement, \"ax\"\n" \
ALTINSTR_REPLACEMENT(altinstr, 1) \
".popsection\n" \
OLDINSTR(oldinstr) \
".pushsection .altinstructions,\"a\"\n" \
ALTINSTR_ENTRY(feature, 1) \
".popsection\n"
#define ALTERNATIVE_2(oldinstr, altinstr1, feature1, altinstr2, feature2)\
".pushsection .altinstr_replacement, \"ax\"\n" \
ALTINSTR_REPLACEMENT(altinstr1, 1) \
ALTINSTR_REPLACEMENT(altinstr2, 2) \
".popsection\n" \
OLDINSTR(oldinstr) \
".pushsection .altinstructions,\"a\"\n" \
ALTINSTR_ENTRY(feature1, 1) \
ALTINSTR_ENTRY(feature2, 2) \
".popsection\n"
/*
* Alternative instructions for different CPU types or capabilities.
*
* This allows to use optimized instructions even on generic binary
* kernels.
*
* oldinstr is padded with jump and nops at compile time if altinstr is
* longer. altinstr is padded with jump and nops at run-time during patching.
*
* For non barrier like inlines please define new variants
* without volatile and memory clobber.
*/
#define alternative(oldinstr, altinstr, feature) \
asm_inline volatile(ALTERNATIVE(oldinstr, altinstr, feature) : : : "memory")
#define alternative_2(oldinstr, altinstr1, feature1, altinstr2, feature2) \
asm_inline volatile(ALTERNATIVE_2(oldinstr, altinstr1, feature1, \
altinstr2, feature2) ::: "memory")
/* Alternative inline assembly with input. */
#define alternative_input(oldinstr, newinstr, feature, input...) \
asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature) \
: : input)
/* Like alternative_input, but with a single output argument */
#define alternative_io(oldinstr, altinstr, feature, output, input...) \
asm_inline volatile(ALTERNATIVE(oldinstr, altinstr, feature) \
: output : input)
/* Use this macro if more than one output parameter is needed. */
#define ASM_OUTPUT2(a...) a
/* Use this macro if clobbers are needed without inputs. */
#define ASM_NO_INPUT_CLOBBER(clobber...) : clobber
#else /* __ASSEMBLY__ */
/*
* Issue one struct alt_instr descriptor entry (need to put it into
* the section .altinstructions, see below). This entry contains
* enough information for the alternatives patching code to patch an
* instruction. See apply_alternatives().
*/
.macro alt_entry orig_start, orig_end, alt_start, alt_end, feature
.long \orig_start - .
.long \alt_start - .
.long \feature
.byte \orig_end - \orig_start
.org . - ( \orig_end - \orig_start ) & 1
.org . - ( \orig_end - \orig_start ) + ( \alt_end - \alt_start )
.org . - ( \alt_end - \alt_start ) + ( \orig_end - \orig_start )
.endm
/*
* Define an alternative between two instructions. If @feature is
* present, early code in apply_alternatives() replaces @oldinstr with
* @newinstr.
*/
.macro ALTERNATIVE oldinstr, newinstr, feature
.pushsection .altinstr_replacement,"ax"
770: \newinstr
771: .popsection
772: \oldinstr
773: .pushsection .altinstructions,"a"
alt_entry 772b, 773b, 770b, 771b, \feature
.popsection
.endm
/*
* Define an alternative between two instructions. If @feature is
* present, early code in apply_alternatives() replaces @oldinstr with
* @newinstr.
*/
.macro ALTERNATIVE_2 oldinstr, newinstr1, feature1, newinstr2, feature2
.pushsection .altinstr_replacement,"ax"
770: \newinstr1
771: \newinstr2
772: .popsection
773: \oldinstr
774: .pushsection .altinstructions,"a"
alt_entry 773b, 774b, 770b, 771b,\feature1
alt_entry 773b, 774b, 771b, 772b,\feature2
.popsection
.endm
#endif /* __ASSEMBLY__ */
#endif /* _ASM_S390_ALTERNATIVE_H */
|
