diff options
authorLinus Torvalds <>2008-06-20 11:18:25 -0700
committerLinus Torvalds <>2008-06-20 11:18:25 -0700
commit89f5b7da2a6bad2e84670422ab8192382a5aeb9f (patch)
parent9bedbcb207ed9a571b239231d99c8fd4a34ae24d (diff)
Reinstate ZERO_PAGE optimization in 'get_user_pages()' and fix XIP
KAMEZAWA Hiroyuki and Oleg Nesterov point out that since the commit 557ed1fa2620dc119adb86b34c614e152a629a80 ("remove ZERO_PAGE") removed the ZERO_PAGE from the VM mappings, any users of get_user_pages() will generally now populate the VM with real empty pages needlessly. We used to get the ZERO_PAGE when we did the "handle_mm_fault()", but since fault handling no longer uses ZERO_PAGE for new anonymous pages, we now need to handle that special case in follow_page() instead. In particular, the removal of ZERO_PAGE effectively removed the core file writing optimization where we would skip writing pages that had not been populated at all, and increased memory pressure a lot by allocating all those useless newly zeroed pages. This reinstates the optimization by making the unmapped PTE case the same as for a non-existent page table, which already did this correctly. While at it, this also fixes the XIP case for follow_page(), where the caller could not differentiate between the case of a page that simply could not be used (because it had no "struct page" associated with it) and a page that just wasn't mapped. We do that by simply returning an error pointer for pages that could not be turned into a "struct page *". The error is arbitrarily picked to be EFAULT, since that was what get_user_pages() already used for the equivalent IO-mapped page case. [ Also removed an impossible test for pte_offset_map_lock() failing: that's not how that function works ] Acked-by: Oleg Nesterov <> Acked-by: Nick Piggin <> Cc: KAMEZAWA Hiroyuki <> Cc: Hugh Dickins <> Cc: Andrew Morton <> Cc: Ingo Molnar <> Cc: Roland McGrath <> Signed-off-by: Linus Torvalds <>
3 files changed, 24 insertions, 5 deletions
diff --git a/arch/powerpc/kernel/vdso.c b/arch/powerpc/kernel/vdso.c
index c21a626af67..ce245a850db 100644
--- a/arch/powerpc/kernel/vdso.c
+++ b/arch/powerpc/kernel/vdso.c
@@ -142,7 +142,7 @@ static void dump_one_vdso_page(struct page *pg, struct page *upg)
printk("kpg: %p (c:%d,f:%08lx)", __va(page_to_pfn(pg) << PAGE_SHIFT),
- if (upg/* && pg != upg*/) {
+ if (upg && !IS_ERR(upg) /* && pg != upg*/) {
printk(" upg: %p (c:%d,f:%08lx)", __va(page_to_pfn(upg)
diff --git a/mm/memory.c b/mm/memory.c
index 19e0ae9beec..9aefaae4685 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -999,17 +999,15 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
goto no_page_table;
ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
- if (!ptep)
- goto out;
pte = *ptep;
if (!pte_present(pte))
- goto unlock;
+ goto no_page;
if ((flags & FOLL_WRITE) && !pte_write(pte))
goto unlock;
page = vm_normal_page(vma, address, pte);
if (unlikely(!page))
- goto unlock;
+ goto bad_page;
if (flags & FOLL_GET)
@@ -1024,6 +1022,15 @@ unlock:
return page;
+ pte_unmap_unlock(ptep, ptl);
+ return ERR_PTR(-EFAULT);
+ pte_unmap_unlock(ptep, ptl);
+ if (!pte_none(pte))
+ return page;
+ /* Fall through to ZERO_PAGE handling */
* When core dumping an enormous anonymous area that nobody
@@ -1159,6 +1166,8 @@ int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
+ if (IS_ERR(page))
+ return i ? i : PTR_ERR(page);
if (pages) {
pages[i] = page;
diff --git a/mm/migrate.c b/mm/migrate.c
index 449d77d409f..112bcaeaa10 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -865,6 +865,11 @@ static int do_move_pages(struct mm_struct *mm, struct page_to_node *pm,
goto set_status;
page = follow_page(vma, pp->addr, FOLL_GET);
+ err = PTR_ERR(page);
+ if (IS_ERR(page))
+ goto set_status;
err = -ENOENT;
if (!page)
goto set_status;
@@ -928,6 +933,11 @@ static int do_pages_stat(struct mm_struct *mm, struct page_to_node *pm)
goto set_status;
page = follow_page(vma, pm->addr, 0);
+ err = PTR_ERR(page);
+ if (IS_ERR(page))
+ goto set_status;
err = -ENOENT;
/* Use PageReserved to check for zero page */
if (!page || PageReserved(page))