diff options
| author | Dennis Zhou <dennis@kernel.org> | 2019-02-25 23:10:15 +0100 |
|---|---|---|
| committer | Dennis Zhou <dennis@kernel.org> | 2019-03-13 20:25:31 +0100 |
| commit | da3afdd5bb5428fd38b4b64f2d5e897c3bb78354 (patch) | |
| tree | 561003eb63655cdbb8f8bae8912b46a8f0f5e93b /mm/percpu.c | |
| parent | percpu: remember largest area skipped during allocation (diff) | |
| download | linux-da3afdd5bb5428fd38b4b64f2d5e897c3bb78354.tar.xz linux-da3afdd5bb5428fd38b4b64f2d5e897c3bb78354.zip | |
percpu: use block scan_hint to only scan forward
Blocks now remember the latest scan_hint. This can be used on the
allocation path as when a contig_hint is broken, we can promote the
scan_hint to the contig_hint and scan forward from there. This works
because pcpu_block_refresh_hint() is only called on the allocation path
while block free regions are updated manually in
pcpu_block_update_hint_free().
Signed-off-by: Dennis Zhou <dennis@kernel.org>
Diffstat (limited to 'mm/percpu.c')
| -rw-r--r-- | mm/percpu.c | 23 |
1 files changed, 17 insertions, 6 deletions
diff --git a/mm/percpu.c b/mm/percpu.c index b82f3cacf2ed..c5250e162d4d 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -761,14 +761,23 @@ static void pcpu_block_refresh_hint(struct pcpu_chunk *chunk, int index) { struct pcpu_block_md *block = chunk->md_blocks + index; unsigned long *alloc_map = pcpu_index_alloc_map(chunk, index); - int rs, re; /* region start, region end */ + int rs, re, start; /* region start, region end */ + + /* promote scan_hint to contig_hint */ + if (block->scan_hint) { + start = block->scan_hint_start + block->scan_hint; + block->contig_hint_start = block->scan_hint_start; + block->contig_hint = block->scan_hint; + block->scan_hint = 0; + } else { + start = block->first_free; + block->contig_hint = 0; + } - /* clear hints */ - block->contig_hint = block->scan_hint = 0; - block->left_free = block->right_free = 0; + block->right_free = 0; /* iterate over free areas and update the contig hints */ - pcpu_for_each_unpop_region(alloc_map, rs, re, block->first_free, + pcpu_for_each_unpop_region(alloc_map, rs, re, start, PCPU_BITMAP_BLOCK_BITS) { pcpu_block_update(block, rs, re); } @@ -833,6 +842,8 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, s_off, s_off + bits)) { /* block contig hint is broken - scan to fix it */ + if (!s_off) + s_block->left_free = 0; pcpu_block_refresh_hint(chunk, s_index); } else { /* update left and right contig manually */ @@ -866,11 +877,11 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off, if (e_off > e_block->scan_hint_start) e_block->scan_hint = 0; + e_block->left_free = 0; if (e_off > e_block->contig_hint_start) { /* contig hint is broken - scan to fix it */ pcpu_block_refresh_hint(chunk, e_index); } else { - e_block->left_free = 0; e_block->right_free = min_t(int, e_block->right_free, PCPU_BITMAP_BLOCK_BITS - e_off); |