From 5b40998ae35cf64561868370e6c9f3d3e94b6bf7 Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Wed, 11 Sep 2013 14:22:33 -0700 Subject: mm: munlock: remove redundant get_page/put_page pair on the fast path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The performance of the fast path in munlock_vma_range() can be further improved by avoiding atomic ops of a redundant get_page()/put_page() pair. When calling get_page() during page isolation, we already have the pin from follow_page_mask(). This pin will be then returned by __pagevec_lru_add(), after which we do not reference the pages anymore. After this patch, an 8% speedup was measured for munlocking a 56GB large memory area with THP disabled. Signed-off-by: Vlastimil Babka Reviewed-by: Jörn Engel Acked-by: Mel Gorman Cc: Michel Lespinasse Cc: Hugh Dickins Cc: Rik van Riel Cc: Johannes Weiner Cc: Michal Hocko Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/mlock.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) (limited to 'mm/mlock.c') diff --git a/mm/mlock.c b/mm/mlock.c index abdc612b042d..19a934dce5d6 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -303,8 +303,10 @@ static void __munlock_pagevec(struct pagevec *pvec, struct zone *zone) if (PageLRU(page)) { lruvec = mem_cgroup_page_lruvec(page, zone); lru = page_lru(page); - - get_page(page); + /* + * We already have pin from follow_page_mask() + * so we can spare the get_page() here. + */ ClearPageLRU(page); del_page_from_lru_list(page, lruvec, lru); } else { @@ -336,25 +338,25 @@ skip_munlock: lock_page(page); if (!__putback_lru_fast_prepare(page, &pvec_putback, &pgrescued)) { - /* Slow path */ + /* + * Slow path. We don't want to lose the last + * pin before unlock_page() + */ + get_page(page); /* for putback_lru_page() */ __munlock_isolated_page(page); unlock_page(page); + put_page(page); /* from follow_page_mask() */ } } } - /* Phase 3: page putback for pages that qualified for the fast path */ + /* + * Phase 3: page putback for pages that qualified for the fast path + * This will also call put_page() to return pin from follow_page_mask() + */ if (pagevec_count(&pvec_putback)) __putback_lru_fast(&pvec_putback, pgrescued); - /* Phase 4: put_page to return pin from follow_page_mask() */ - for (i = 0; i < nr; i++) { - struct page *page = pvec->pages[i]; - - if (page) - put_page(page); - } - pagevec_reinit(pvec); } -- cgit v1.2.3