diff options
| -rw-r--r-- | mm/ksm.c | 234 |
1 files changed, 112 insertions, 122 deletions
@@ -356,8 +356,10 @@ static int break_ksm(struct vm_area_struct *vma, unsigned long addr) return (ret & VM_FAULT_OOM) ? -ENOMEM : 0; } -static void break_cow(struct mm_struct *mm, unsigned long addr) +static void break_cow(struct rmap_item *rmap_item) { + struct mm_struct *mm = rmap_item->mm; + unsigned long addr = rmap_item->address; struct vm_area_struct *vma; down_read(&mm->mmap_sem); @@ -665,15 +667,15 @@ out: /** * replace_page - replace page in vma by new ksm page - * @vma: vma that holds the pte pointing to oldpage - * @oldpage: the page we are replacing by newpage - * @newpage: the ksm page we replace oldpage by + * @vma: vma that holds the pte pointing to page + * @page: the page we are replacing by kpage + * @kpage: the ksm page we replace page by * @orig_pte: the original value of the pte * * Returns 0 on success, -EFAULT on failure. */ -static int replace_page(struct vm_area_struct *vma, struct page *oldpage, - struct page *newpage, pte_t orig_pte) +static int replace_page(struct vm_area_struct *vma, struct page *page, + struct page *kpage, pte_t orig_pte) { struct mm_struct *mm = vma->vm_mm; pgd_t *pgd; @@ -684,7 +686,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *oldpage, unsigned long addr; int err = -EFAULT; - addr = page_address_in_vma(oldpage, vma); + addr = page_address_in_vma(page, vma); if (addr == -EFAULT) goto out; @@ -706,15 +708,15 @@ static int replace_page(struct vm_area_struct *vma, struct page *oldpage, goto out; } - get_page(newpage); - page_add_ksm_rmap(newpage); + get_page(kpage); + page_add_ksm_rmap(kpage); flush_cache_page(vma, addr, pte_pfn(*ptep)); ptep_clear_flush(vma, addr, ptep); - set_pte_at_notify(mm, addr, ptep, mk_pte(newpage, vma->vm_page_prot)); + set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot)); - page_remove_rmap(oldpage); - put_page(oldpage); + page_remove_rmap(page); + put_page(page); pte_unmap_unlock(ptep, ptl); err = 0; @@ -724,26 +726,22 @@ out: /* * try_to_merge_one_page - take two pages and merge them into one - * @vma: the vma that hold the pte pointing into oldpage - * @oldpage: the page that we want to replace with newpage - * @newpage: the page that we want to map instead of oldpage - * - * Note: - * oldpage should be a PageAnon page, while newpage should be a PageKsm page, - * or a newly allocated kernel page which page_add_ksm_rmap will make PageKsm. + * @vma: the vma that holds the pte pointing to page + * @page: the PageAnon page that we want to replace with kpage + * @kpage: the PageKsm page (or newly allocated page which page_add_ksm_rmap + * will make PageKsm) that we want to map instead of page * * This function returns 0 if the pages were merged, -EFAULT otherwise. */ static int try_to_merge_one_page(struct vm_area_struct *vma, - struct page *oldpage, - struct page *newpage) + struct page *page, struct page *kpage) { pte_t orig_pte = __pte(0); int err = -EFAULT; if (!(vma->vm_flags & VM_MERGEABLE)) goto out; - if (!PageAnon(oldpage)) + if (!PageAnon(page)) goto out; /* @@ -753,7 +751,7 @@ static int try_to_merge_one_page(struct vm_area_struct *vma, * prefer to continue scanning and merging different pages, * then come back to this page when it is unlocked. */ - if (!trylock_page(oldpage)) + if (!trylock_page(page)) goto out; /* * If this anonymous page is mapped only here, its pte may need @@ -761,11 +759,11 @@ static int try_to_merge_one_page(struct vm_area_struct *vma, * ptes are necessarily already write-protected. But in either * case, we need to lock and check page_count is not raised. */ - if (write_protect_page(vma, oldpage, &orig_pte) == 0 && - pages_identical(oldpage, newpage)) - err = replace_page(vma, oldpage, newpage, orig_pte); + if (write_protect_page(vma, page, &orig_pte) == 0 && + pages_identical(page, kpage)) + err = replace_page(vma, page, kpage, orig_pte); - unlock_page(oldpage); + unlock_page(page); out: return err; } @@ -773,26 +771,26 @@ out: /* * try_to_merge_with_ksm_page - like try_to_merge_two_pages, * but no new kernel page is allocated: kpage must already be a ksm page. + * + * This function returns 0 if the pages were merged, -EFAULT otherwise. */ -static int try_to_merge_with_ksm_page(struct mm_struct *mm1, - unsigned long addr1, - struct page *page1, - struct page *kpage) +static int try_to_merge_with_ksm_page(struct rmap_item *rmap_item, + struct page *page, struct page *kpage) { + struct mm_struct *mm = rmap_item->mm; struct vm_area_struct *vma; int err = -EFAULT; - down_read(&mm1->mmap_sem); - if (ksm_test_exit(mm1)) + down_read(&mm->mmap_sem); + if (ksm_test_exit(mm)) goto out; - - vma = find_vma(mm1, addr1); - if (!vma || vma->vm_start > addr1) + vma = find_vma(mm, rmap_item->address); + if (!vma || vma->vm_start > rmap_item->address) goto out; - err = try_to_merge_one_page(vma, page1, kpage); + err = try_to_merge_one_page(vma, page, kpage); out: - up_read(&mm1->mmap_sem); + up_read(&mm->mmap_sem); return err; } @@ -800,16 +798,18 @@ out: * try_to_merge_two_pages - take two identical pages and prepare them * to be merged into one page. * - * This function returns 0 if we successfully mapped two identical pages - * into one page, -EFAULT otherwise. + * This function returns the kpage if we successfully merged two identical + * pages into one ksm page, NULL otherwise. * * Note that this function allocates a new kernel page: if one of the pages * is already a ksm page, try_to_merge_with_ksm_page should be used. */ -static int try_to_merge_two_pages(struct mm_struct *mm1, unsigned long addr1, - struct page *page1, struct mm_struct *mm2, - unsigned long addr2, struct page *page2) +static struct page *try_to_merge_two_pages(struct rmap_item *rmap_item, + struct page *page, + struct rmap_item *tree_rmap_item, + struct page *tree_page) { + struct mm_struct *mm = rmap_item->mm; struct vm_area_struct *vma; struct page *kpage; int err = -EFAULT; @@ -820,47 +820,43 @@ static int try_to_merge_two_pages(struct mm_struct *mm1, unsigned long addr1, */ if (ksm_max_kernel_pages && ksm_max_kernel_pages <= ksm_pages_shared) - return err; + return NULL; kpage = alloc_page(GFP_HIGHUSER); if (!kpage) - return err; - - down_read(&mm1->mmap_sem); - if (ksm_test_exit(mm1)) { - up_read(&mm1->mmap_sem); - goto out; - } - vma = find_vma(mm1, addr1); - if (!vma || vma->vm_start > addr1) { - up_read(&mm1->mmap_sem); - goto out; - } + return NULL; - copy_user_highpage(kpage, page1, addr1, vma); - err = try_to_merge_one_page(vma, page1, kpage); - up_read(&mm1->mmap_sem); + down_read(&mm->mmap_sem); + if (ksm_test_exit(mm)) + goto up; + vma = find_vma(mm, rmap_item->address); + if (!vma || vma->vm_start > rmap_item->address) + goto up; + + copy_user_highpage(kpage, page, rmap_item->address, vma); + err = try_to_merge_one_page(vma, page, kpage); +up: + up_read(&mm->mmap_sem); if (!err) { - err = try_to_merge_with_ksm_page(mm2, addr2, page2, kpage); + err = try_to_merge_with_ksm_page(tree_rmap_item, + tree_page, kpage); /* * If that fails, we have a ksm page with only one pte * pointing to it: so break it. */ if (err) - break_cow(mm1, addr1); + break_cow(rmap_item); } -out: - put_page(kpage); - return err; + if (err) { + put_page(kpage); + kpage = NULL; + } + return kpage; } /* - * stable_tree_search - search page inside the stable tree - * @page: the page that we are searching identical pages to. - * @page2: pointer into identical page that we are holding inside the stable - * tree that we have found. - * @rmap_item: the reverse mapping item + * stable_tree_search - search for page inside the stable tree * * This function checks if there is a page inside the stable tree * with identical content to the page that we are scanning right now. @@ -869,21 +865,21 @@ out: * NULL otherwise. */ static struct rmap_item *stable_tree_search(struct page *page, - struct page **page2, - struct rmap_item *rmap_item) + struct page **tree_pagep) { struct rb_node *node = root_stable_tree.rb_node; while (node) { struct rmap_item *tree_rmap_item, *next_rmap_item; + struct page *tree_page; int ret; tree_rmap_item = rb_entry(node, struct rmap_item, node); while (tree_rmap_item) { BUG_ON(!in_stable_tree(tree_rmap_item)); cond_resched(); - page2[0] = get_ksm_page(tree_rmap_item); - if (page2[0]) + tree_page = get_ksm_page(tree_rmap_item); + if (tree_page) break; next_rmap_item = tree_rmap_item->next; remove_rmap_item_from_tree(tree_rmap_item); @@ -892,15 +888,16 @@ static struct rmap_item *stable_tree_search(struct page *page, if (!tree_rmap_item) return NULL; - ret = memcmp_pages(page, page2[0]); + ret = memcmp_pages(page, tree_page); if (ret < 0) { - put_page(page2[0]); + put_page(tree_page); node = node->rb_left; } else if (ret > 0) { - put_page(page2[0]); + put_page(tree_page); node = node->rb_right; } else { + *tree_pagep = tree_page; return tree_rmap_item; } } @@ -912,13 +909,9 @@ static struct rmap_item *stable_tree_search(struct page *page, * stable_tree_insert - insert rmap_item pointing to new ksm page * into the stable tree. * - * @page: the page that we are searching identical page to inside the stable - * tree. - * @rmap_item: pointer to the reverse mapping item. - * * This function returns rmap_item if success, NULL otherwise. */ -static struct rmap_item *stable_tree_insert(struct page *page, +static struct rmap_item *stable_tree_insert(struct page *kpage, struct rmap_item *rmap_item) { struct rb_node **new = &root_stable_tree.rb_node; @@ -943,7 +936,7 @@ static struct rmap_item *stable_tree_insert(struct page *page, if (!tree_rmap_item) return NULL; - ret = memcmp_pages(page, tree_page); + ret = memcmp_pages(kpage, tree_page); put_page(tree_page); parent = *new; @@ -971,12 +964,8 @@ static struct rmap_item *stable_tree_insert(struct page *page, } /* - * unstable_tree_search_insert - search and insert items into the unstable tree. - * - * @page: the page that we are going to search for identical page or to insert - * into the unstable tree - * @page2: pointer into identical page that was found inside the unstable tree - * @rmap_item: the reverse mapping item of page + * unstable_tree_search_insert - search for identical page, + * else insert rmap_item into the unstable tree. * * This function searches for a page in the unstable tree identical to the * page currently being scanned; and if no identical page is found in the @@ -988,42 +977,45 @@ static struct rmap_item *stable_tree_insert(struct page *page, * This function does both searching and inserting, because they share * the same walking algorithm in an rbtree. */ -static struct rmap_item *unstable_tree_search_insert(struct page *page, - struct page **page2, - struct rmap_item *rmap_item) +static +struct rmap_item *unstable_tree_search_insert(struct rmap_item *rmap_item, + struct page *page, + struct page **tree_pagep) + { struct rb_node **new = &root_unstable_tree.rb_node; struct rb_node *parent = NULL; while (*new) { struct rmap_item *tree_rmap_item; + struct page *tree_page; int ret; cond_resched(); tree_rmap_item = rb_entry(*new, struct rmap_item, node); - page2[0] = get_mergeable_page(tree_rmap_item); - if (!page2[0]) + tree_page = get_mergeable_page(tree_rmap_item); + if (!tree_page) return NULL; /* - * Don't substitute an unswappable ksm page - * just for one good swappable forked page. + * Don't substitute a ksm page for a forked page. */ - if (page == page2[0]) { - put_page(page2[0]); + if (page == tree_page) { + put_page(tree_page); return NULL; } - ret = memcmp_pages(page, page2[0]); + ret = memcmp_pages(page, tree_page); parent = *new; if (ret < 0) { - put_page(page2[0]); + put_page(tree_page); new = &parent->rb_left; } else if (ret > 0) { - put_page(page2[0]); + put_page(tree_page); new = &parent->rb_right; } else { + *tree_pagep = tree_page; return tree_rmap_item; } } @@ -1068,24 +1060,23 @@ static void stable_tree_append(struct rmap_item *rmap_item, */ static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item) { - struct page *page2[1]; struct rmap_item *tree_rmap_item; + struct page *tree_page = NULL; + struct page *kpage; unsigned int checksum; int err; remove_rmap_item_from_tree(rmap_item); /* We first start with searching the page inside the stable tree */ - tree_rmap_item = stable_tree_search(page, page2, rmap_item); + tree_rmap_item = stable_tree_search(page, &tree_page); if (tree_rmap_item) { - if (page == page2[0]) /* forked */ + kpage = tree_page; + if (page == kpage) /* forked */ err = 0; else - err = try_to_merge_with_ksm_page(rmap_item->mm, - rmap_item->address, - page, page2[0]); - put_page(page2[0]); - + err = try_to_merge_with_ksm_page(rmap_item, + page, kpage); if (!err) { /* * The page was successfully merged: @@ -1093,6 +1084,7 @@ static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item) */ stable_tree_append(rmap_item, tree_rmap_item); } + put_page(kpage); return; } @@ -1103,7 +1095,7 @@ static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item) * when the mem_cgroup had reached its limit: try again now. */ if (PageKsm(page)) - break_cow(rmap_item->mm, rmap_item->address); + break_cow(rmap_item); /* * In case the hash value of the page was changed from the last time we @@ -1117,18 +1109,18 @@ static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item) return; } - tree_rmap_item = unstable_tree_search_insert(page, page2, rmap_item); + tree_rmap_item = + unstable_tree_search_insert(rmap_item, page, &tree_page); if (tree_rmap_item) { - err = try_to_merge_two_pages(rmap_item->mm, - rmap_item->address, page, - tree_rmap_item->mm, - tree_rmap_item->address, page2[0]); + kpage = try_to_merge_two_pages(rmap_item, page, + tree_rmap_item, tree_page); + put_page(tree_page); /* * As soon as we merge this page, we want to remove the * rmap_item of the page we have merged with from the unstable * tree, and insert it instead as new node in the stable tree. */ - if (!err) { + if (kpage) { remove_rmap_item_from_tree(tree_rmap_item); /* @@ -1137,16 +1129,14 @@ static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item) * to a ksm page left outside the stable tree, * in which case we need to break_cow on both. */ - if (stable_tree_insert(page2[0], tree_rmap_item)) + if (stable_tree_insert(kpage, tree_rmap_item)) stable_tree_append(rmap_item, tree_rmap_item); else { - break_cow(tree_rmap_item->mm, - tree_rmap_item->address); - break_cow(rmap_item->mm, rmap_item->address); + break_cow(tree_rmap_item); + break_cow(rmap_item); } + put_page(kpage); } - - put_page(page2[0]); } } @@ -1308,7 +1298,7 @@ static void ksm_do_scan(unsigned int scan_npages) /* * Replace now-unshared ksm page by ordinary page. */ - break_cow(rmap_item->mm, rmap_item->address); + break_cow(rmap_item); remove_rmap_item_from_tree(rmap_item); rmap_item->oldchecksum = calc_checksum(page); } |