1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
|
/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
/*
* rseq.h
*
* (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
*/
#ifndef RSEQ_H
#define RSEQ_H
#include <stdint.h>
#include <stdbool.h>
#include <pthread.h>
#include <signal.h>
#include <sched.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <linux/rseq.h>
/*
* Empty code injection macros, override when testing.
* It is important to consider that the ASM injection macros need to be
* fully reentrant (e.g. do not modify the stack).
*/
#ifndef RSEQ_INJECT_ASM
#define RSEQ_INJECT_ASM(n)
#endif
#ifndef RSEQ_INJECT_C
#define RSEQ_INJECT_C(n)
#endif
#ifndef RSEQ_INJECT_INPUT
#define RSEQ_INJECT_INPUT
#endif
#ifndef RSEQ_INJECT_CLOBBER
#define RSEQ_INJECT_CLOBBER
#endif
#ifndef RSEQ_INJECT_FAILED
#define RSEQ_INJECT_FAILED
#endif
extern __thread volatile struct rseq __rseq_abi;
#define rseq_likely(x) __builtin_expect(!!(x), 1)
#define rseq_unlikely(x) __builtin_expect(!!(x), 0)
#define rseq_barrier() __asm__ __volatile__("" : : : "memory")
#define RSEQ_ACCESS_ONCE(x) (*(__volatile__ __typeof__(x) *)&(x))
#define RSEQ_WRITE_ONCE(x, v) __extension__ ({ RSEQ_ACCESS_ONCE(x) = (v); })
#define RSEQ_READ_ONCE(x) RSEQ_ACCESS_ONCE(x)
#define __rseq_str_1(x) #x
#define __rseq_str(x) __rseq_str_1(x)
#define rseq_log(fmt, args...) \
fprintf(stderr, fmt "(in %s() at " __FILE__ ":" __rseq_str(__LINE__)"\n", \
## args, __func__)
#define rseq_bug(fmt, args...) \
do { \
rseq_log(fmt, ##args); \
abort(); \
} while (0)
#if defined(__x86_64__) || defined(__i386__)
#include <rseq-x86.h>
#elif defined(__ARMEL__)
#include <rseq-arm.h>
#elif defined (__AARCH64EL__)
#include <rseq-arm64.h>
#elif defined(__PPC__)
#include <rseq-ppc.h>
#elif defined(__mips__)
#include <rseq-mips.h>
#elif defined(__s390__)
#include <rseq-s390.h>
#else
#error unsupported target
#endif
/*
* Register rseq for the current thread. This needs to be called once
* by any thread which uses restartable sequences, before they start
* using restartable sequences, to ensure restartable sequences
* succeed. A restartable sequence executed from a non-registered
* thread will always fail.
*/
int rseq_register_current_thread(void);
/*
* Unregister rseq for current thread.
*/
int rseq_unregister_current_thread(void);
/*
* Restartable sequence fallback for reading the current CPU number.
*/
int32_t rseq_fallback_current_cpu(void);
/*
* Values returned can be either the current CPU number, -1 (rseq is
* uninitialized), or -2 (rseq initialization has failed).
*/
static inline int32_t rseq_current_cpu_raw(void)
{
return RSEQ_ACCESS_ONCE(__rseq_abi.cpu_id);
}
/*
* Returns a possible CPU number, which is typically the current CPU.
* The returned CPU number can be used to prepare for an rseq critical
* section, which will confirm whether the cpu number is indeed the
* current one, and whether rseq is initialized.
*
* The CPU number returned by rseq_cpu_start should always be validated
* by passing it to a rseq asm sequence, or by comparing it to the
* return value of rseq_current_cpu_raw() if the rseq asm sequence
* does not need to be invoked.
*/
static inline uint32_t rseq_cpu_start(void)
{
return RSEQ_ACCESS_ONCE(__rseq_abi.cpu_id_start);
}
static inline uint32_t rseq_current_cpu(void)
{
int32_t cpu;
cpu = rseq_current_cpu_raw();
if (rseq_unlikely(cpu < 0))
cpu = rseq_fallback_current_cpu();
return cpu;
}
static inline void rseq_clear_rseq_cs(void)
{
#ifdef __LP64__
__rseq_abi.rseq_cs.ptr = 0;
#else
__rseq_abi.rseq_cs.ptr.ptr32 = 0;
#endif
}
/*
* rseq_prepare_unload() should be invoked by each thread executing a rseq
* critical section at least once between their last critical section and
* library unload of the library defining the rseq critical section
* (struct rseq_cs). This also applies to use of rseq in code generated by
* JIT: rseq_prepare_unload() should be invoked at least once by each
* thread executing a rseq critical section before reclaim of the memory
* holding the struct rseq_cs.
*/
static inline void rseq_prepare_unload(void)
{
rseq_clear_rseq_cs();
}
#endif /* RSEQ_H_ */
|
