1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
|
/*
* Stress test for transparent huge pages, memory compaction and migration.
*
* Authors: Konstantin Khlebnikov <koct9i@gmail.com>
*
* This is free and unencumbered software released into the public domain.
*/
#include <stdlib.h>
#include <stdio.h>
#include <stdint.h>
#include <err.h>
#include <time.h>
#include <unistd.h>
#include <fcntl.h>
#include <string.h>
#include <sys/mman.h>
#define PAGE_SHIFT 12
#define HPAGE_SHIFT 21
#define PAGE_SIZE (1 << PAGE_SHIFT)
#define HPAGE_SIZE (1 << HPAGE_SHIFT)
#define PAGEMAP_PRESENT(ent) (((ent) & (1ull << 63)) != 0)
#define PAGEMAP_PFN(ent) ((ent) & ((1ull << 55) - 1))
int pagemap_fd;
int64_t allocate_transhuge(void *ptr)
{
uint64_t ent[2];
/* drop pmd */
if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE,
MAP_FIXED | MAP_ANONYMOUS |
MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr)
errx(2, "mmap transhuge");
if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE))
err(2, "MADV_HUGEPAGE");
/* allocate transparent huge page */
*(volatile void **)ptr = ptr;
if (pread(pagemap_fd, ent, sizeof(ent),
(uintptr_t)ptr >> (PAGE_SHIFT - 3)) != sizeof(ent))
err(2, "read pagemap");
if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) &&
PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) &&
!(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - PAGE_SHIFT)) - 1)))
return PAGEMAP_PFN(ent[0]);
return -1;
}
int main(int argc, char **argv)
{
size_t ram, len;
void *ptr, *p;
struct timespec a, b;
double s;
uint8_t *map;
size_t map_len;
ram = sysconf(_SC_PHYS_PAGES);
if (ram > SIZE_MAX / sysconf(_SC_PAGESIZE) / 4)
ram = SIZE_MAX / 4;
else
ram *= sysconf(_SC_PAGESIZE);
if (argc == 1)
len = ram;
else if (!strcmp(argv[1], "-h"))
errx(1, "usage: %s [size in MiB]", argv[0]);
else
len = atoll(argv[1]) << 20;
warnx("allocate %zd transhuge pages, using %zd MiB virtual memory"
" and %zd MiB of ram", len >> HPAGE_SHIFT, len >> 20,
ram >> (20 + HPAGE_SHIFT - PAGE_SHIFT - 1));
pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
if (pagemap_fd < 0)
err(2, "open pagemap");
len -= len % HPAGE_SIZE;
ptr = mmap(NULL, len + HPAGE_SIZE, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0);
if (ptr == MAP_FAILED)
err(2, "initial mmap");
ptr += HPAGE_SIZE - (uintptr_t)ptr % HPAGE_SIZE;
if (madvise(ptr, len, MADV_HUGEPAGE))
err(2, "MADV_HUGEPAGE");
map_len = ram >> (HPAGE_SHIFT - 1);
map = malloc(map_len);
if (!map)
errx(2, "map malloc");
while (1) {
int nr_succeed = 0, nr_failed = 0, nr_pages = 0;
memset(map, 0, map_len);
clock_gettime(CLOCK_MONOTONIC, &a);
for (p = ptr; p < ptr + len; p += HPAGE_SIZE) {
int64_t pfn;
pfn = allocate_transhuge(p);
if (pfn < 0) {
nr_failed++;
} else {
size_t idx = pfn >> (HPAGE_SHIFT - PAGE_SHIFT);
nr_succeed++;
if (idx >= map_len) {
map = realloc(map, idx + 1);
if (!map)
errx(2, "map realloc");
memset(map + map_len, 0, idx + 1 - map_len);
map_len = idx + 1;
}
if (!map[idx])
nr_pages++;
map[idx] = 1;
}
/* split transhuge page, keep last page */
if (madvise(p, HPAGE_SIZE - PAGE_SIZE, MADV_DONTNEED))
err(2, "MADV_DONTNEED");
}
clock_gettime(CLOCK_MONOTONIC, &b);
s = b.tv_sec - a.tv_sec + (b.tv_nsec - a.tv_nsec) / 1000000000.;
warnx("%.3f s/loop, %.3f ms/page, %10.3f MiB/s\t"
"%4d succeed, %4d failed, %4d different pages",
s, s * 1000 / (len >> HPAGE_SHIFT), len / s / (1 << 20),
nr_succeed, nr_failed, nr_pages);
}
}
|
