blob: 98e88a6112baa3afea1b7913e4151e1f673239de [file] [log] [blame]
Paul Mundt26ff6c12006-09-27 15:13:36 +09001/*
2 * Page fault handler for SH with an MMU.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 * Copyright (C) 1999 Niibe Yutaka
Paul Mundt0f60bb22009-07-05 03:18:47 +09005 * Copyright (C) 2003 - 2009 Paul Mundt
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 *
7 * Based on linux/arch/i386/mm/fault.c:
8 * Copyright (C) 1995 Linus Torvalds
Paul Mundt26ff6c12006-09-27 15:13:36 +09009 *
10 * This file is subject to the terms and conditions of the GNU General Public
11 * License. See the file "COPYING" in the main directory of this archive
12 * for more details.
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/mm.h>
Paul Mundt0f08f332006-09-27 17:03:56 +090016#include <linux/hardirq.h>
17#include <linux/kprobes.h>
Ingo Molnarcdd6c482009-09-21 12:02:48 +020018#include <linux/perf_event.h>
Magnus Damme7cc9a72008-02-07 20:18:21 +090019#include <asm/io_trapped.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <asm/mmu_context.h>
Paul Mundtdb2e1fa2007-02-14 14:13:10 +090021#include <asm/tlbflush.h>
David Howellse839ca52012-03-28 18:30:03 +010022#include <asm/traps.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023
Paul Mundt7433ab7702009-06-25 02:30:10 +090024static inline int notify_page_fault(struct pt_regs *regs, int trap)
25{
26 int ret = 0;
27
Paul Mundtc63c3102009-07-05 02:50:10 +090028 if (kprobes_built_in() && !user_mode(regs)) {
Paul Mundt7433ab7702009-06-25 02:30:10 +090029 preempt_disable();
30 if (kprobe_running() && kprobe_fault_handler(regs, trap))
31 ret = 1;
32 preempt_enable();
33 }
Paul Mundt7433ab7702009-06-25 02:30:10 +090034
35 return ret;
36}
37
Paul Mundt0f60bb22009-07-05 03:18:47 +090038static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
39{
40 unsigned index = pgd_index(address);
41 pgd_t *pgd_k;
42 pud_t *pud, *pud_k;
43 pmd_t *pmd, *pmd_k;
44
45 pgd += index;
46 pgd_k = init_mm.pgd + index;
47
48 if (!pgd_present(*pgd_k))
49 return NULL;
50
51 pud = pud_offset(pgd, address);
52 pud_k = pud_offset(pgd_k, address);
53 if (!pud_present(*pud_k))
54 return NULL;
55
Matt Fleming5d9b4b12009-12-13 14:38:50 +000056 if (!pud_present(*pud))
57 set_pud(pud, *pud_k);
58
Paul Mundt0f60bb22009-07-05 03:18:47 +090059 pmd = pmd_offset(pud, address);
60 pmd_k = pmd_offset(pud_k, address);
61 if (!pmd_present(*pmd_k))
62 return NULL;
63
64 if (!pmd_present(*pmd))
65 set_pmd(pmd, *pmd_k);
Matt Fleming05dd2cd2009-07-13 11:38:04 +000066 else {
67 /*
68 * The page tables are fully synchronised so there must
69 * be another reason for the fault. Return NULL here to
70 * signal that we have not taken care of the fault.
71 */
Paul Mundt0f60bb22009-07-05 03:18:47 +090072 BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k));
Matt Fleming05dd2cd2009-07-13 11:38:04 +000073 return NULL;
74 }
Paul Mundt0f60bb22009-07-05 03:18:47 +090075
76 return pmd_k;
77}
78
79/*
80 * Handle a fault on the vmalloc or module mapping area
81 */
82static noinline int vmalloc_fault(unsigned long address)
83{
84 pgd_t *pgd_k;
85 pmd_t *pmd_k;
86 pte_t *pte_k;
87
Paul Mundt0906a3a2009-09-03 17:21:10 +090088 /* Make sure we are in vmalloc/module/P3 area: */
89 if (!(address >= VMALLOC_START && address < P3_ADDR_MAX))
Paul Mundt0f60bb22009-07-05 03:18:47 +090090 return -1;
91
92 /*
93 * Synchronize this task's top level page-table
94 * with the 'reference' page table.
95 *
96 * Do _not_ use "current" here. We might be inside
97 * an interrupt in the middle of a task switch..
98 */
99 pgd_k = get_TTB();
Matt Fleming05dd2cd2009-07-13 11:38:04 +0000100 pmd_k = vmalloc_sync_one(pgd_k, address);
Paul Mundt0f60bb22009-07-05 03:18:47 +0900101 if (!pmd_k)
102 return -1;
103
104 pte_k = pte_offset_kernel(pmd_k, address);
105 if (!pte_present(*pte_k))
106 return -1;
107
108 return 0;
109}
110
111static int fault_in_kernel_space(unsigned long address)
112{
113 return address >= TASK_SIZE;
114}
115
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116/*
117 * This routine handles page faults. It determines the address,
118 * and the problem, and then passes it off to one of the appropriate
119 * routines.
120 */
Stuart Menefyb5a1bcb2006-11-21 13:34:04 +0900121asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
122 unsigned long writeaccess,
123 unsigned long address)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700124{
Paul Mundt0f60bb22009-07-05 03:18:47 +0900125 unsigned long vec;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126 struct task_struct *tsk;
127 struct mm_struct *mm;
128 struct vm_area_struct * vma;
Stuart Menefyb5a1bcb2006-11-21 13:34:04 +0900129 int si_code;
Nick Piggin83c54072007-07-19 01:47:05 -0700130 int fault;
Stuart Menefyb5a1bcb2006-11-21 13:34:04 +0900131 siginfo_t info;
Kautuk Consul11fd9822012-03-31 08:06:11 -0400132 unsigned int flags = (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE |
133 (writeaccess ? FAULT_FLAG_WRITE : 0));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134
Linus Torvalds1da177e2005-04-16 15:20:36 -0700135 tsk = current;
Paul Mundt0f60bb22009-07-05 03:18:47 +0900136 mm = tsk->mm;
Stuart Menefyb5a1bcb2006-11-21 13:34:04 +0900137 si_code = SEGV_MAPERR;
Paul Mundt0f60bb22009-07-05 03:18:47 +0900138 vec = lookup_exception_vector();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139
Paul Mundt0f60bb22009-07-05 03:18:47 +0900140 /*
141 * We fault-in kernel-space virtual memory on-demand. The
142 * 'reference' page table is init_mm.pgd.
143 *
144 * NOTE! We MUST NOT take any locks for this case. We may
145 * be in an interrupt or a critical region, and should
146 * only copy the information from the master page table,
147 * nothing more.
148 */
149 if (unlikely(fault_in_kernel_space(address))) {
150 if (vmalloc_fault(address) >= 0)
Stuart Menefy99a596f2006-11-21 15:38:05 +0900151 return;
Paul Mundt0f60bb22009-07-05 03:18:47 +0900152 if (notify_page_fault(regs, vec))
Stuart Menefy96e14e52008-09-05 16:17:15 +0900153 return;
Stuart Menefy99a596f2006-11-21 15:38:05 +0900154
Paul Mundt0f60bb22009-07-05 03:18:47 +0900155 goto bad_area_nosemaphore;
Stuart Menefy99a596f2006-11-21 15:38:05 +0900156 }
157
Paul Mundt0f60bb22009-07-05 03:18:47 +0900158 if (unlikely(notify_page_fault(regs, vec)))
Paul Mundt7433ab7702009-06-25 02:30:10 +0900159 return;
160
161 /* Only enable interrupts if they were on before the fault */
162 if ((regs->sr & SR_IMASK) != SR_IMASK)
163 local_irq_enable();
164
Peter Zijlstraa8b0ca12011-06-27 14:41:57 +0200165 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
Paul Mundt7433ab7702009-06-25 02:30:10 +0900166
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167 /*
Paul Mundt0f60bb22009-07-05 03:18:47 +0900168 * If we're in an interrupt, have no user context or are running
169 * in an atomic region then we must not take the fault:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170 */
171 if (in_atomic() || !mm)
172 goto no_context;
173
Kautuk Consul11fd9822012-03-31 08:06:11 -0400174retry:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175 down_read(&mm->mmap_sem);
176
177 vma = find_vma(mm, address);
178 if (!vma)
179 goto bad_area;
180 if (vma->vm_start <= address)
181 goto good_area;
182 if (!(vma->vm_flags & VM_GROWSDOWN))
183 goto bad_area;
184 if (expand_stack(vma, address))
185 goto bad_area;
Paul Mundt0f60bb22009-07-05 03:18:47 +0900186
187 /*
188 * Ok, we have a good vm_area for this memory access, so
189 * we can handle it..
190 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191good_area:
Stuart Menefyb5a1bcb2006-11-21 13:34:04 +0900192 si_code = SEGV_ACCERR;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193 if (writeaccess) {
194 if (!(vma->vm_flags & VM_WRITE))
195 goto bad_area;
196 } else {
Jason Barondf67b3d2006-09-29 01:58:58 -0700197 if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198 goto bad_area;
199 }
200
201 /*
202 * If for any reason at all we couldn't handle the fault,
203 * make sure we exit gracefully rather than endlessly redo
204 * the fault.
205 */
Kautuk Consul11fd9822012-03-31 08:06:11 -0400206 fault = handle_mm_fault(mm, vma, address, flags);
207
208 if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
209 return;
210
Nick Piggin83c54072007-07-19 01:47:05 -0700211 if (unlikely(fault & VM_FAULT_ERROR)) {
212 if (fault & VM_FAULT_OOM)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213 goto out_of_memory;
Nick Piggin83c54072007-07-19 01:47:05 -0700214 else if (fault & VM_FAULT_SIGBUS)
215 goto do_sigbus;
216 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217 }
Kautuk Consul11fd9822012-03-31 08:06:11 -0400218
219 if (flags & FAULT_FLAG_ALLOW_RETRY) {
220 if (fault & VM_FAULT_MAJOR) {
221 tsk->maj_flt++;
222 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
223 regs, address);
224 } else {
225 tsk->min_flt++;
226 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
227 regs, address);
228 }
229 if (fault & VM_FAULT_RETRY) {
230 flags &= ~FAULT_FLAG_ALLOW_RETRY;
231
232 /*
233 * No need to up_read(&mm->mmap_sem) as we would
234 * have already released it in __lock_page_or_retry
235 * in mm/filemap.c.
236 */
237 goto retry;
238 }
Paul Mundt7433ab7702009-06-25 02:30:10 +0900239 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240
241 up_read(&mm->mmap_sem);
242 return;
243
Paul Mundt0f60bb22009-07-05 03:18:47 +0900244 /*
245 * Something tried to access memory that isn't in our memory map..
246 * Fix it, but check if it's kernel or user first..
247 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248bad_area:
249 up_read(&mm->mmap_sem);
250
Stuart Menefy99a596f2006-11-21 15:38:05 +0900251bad_area_nosemaphore:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252 if (user_mode(regs)) {
Stuart Menefyb5a1bcb2006-11-21 13:34:04 +0900253 info.si_signo = SIGSEGV;
254 info.si_errno = 0;
255 info.si_code = si_code;
256 info.si_addr = (void *) address;
257 force_sig_info(SIGSEGV, &info, tsk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258 return;
259 }
260
261no_context:
262 /* Are we prepared to handle this kernel fault? */
263 if (fixup_exception(regs))
264 return;
265
Magnus Damme7cc9a72008-02-07 20:18:21 +0900266 if (handle_trapped_io(regs, address))
267 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268/*
269 * Oops. The kernel tried to access some bad page. We'll have to
270 * terminate things with extreme prejudice.
271 *
272 */
Paul Mundt0630e452007-06-18 19:02:47 +0900273
274 bust_spinlocks(1);
275
276 if (oops_may_print()) {
Paul Mundtb62ad832008-01-10 14:07:03 +0900277 unsigned long page;
Paul Mundt0630e452007-06-18 19:02:47 +0900278
279 if (address < PAGE_SIZE)
280 printk(KERN_ALERT "Unable to handle kernel NULL "
281 "pointer dereference");
282 else
283 printk(KERN_ALERT "Unable to handle kernel paging "
284 "request");
285 printk(" at virtual address %08lx\n", address);
286 printk(KERN_ALERT "pc = %08lx\n", regs->pc);
287 page = (unsigned long)get_TTB();
288 if (page) {
Paul Mundt06f862c2007-08-01 16:39:51 +0900289 page = ((__typeof__(page) *)page)[address >> PGDIR_SHIFT];
Paul Mundt0630e452007-06-18 19:02:47 +0900290 printk(KERN_ALERT "*pde = %08lx\n", page);
291 if (page & _PAGE_PRESENT) {
292 page &= PAGE_MASK;
293 address &= 0x003ff000;
294 page = ((__typeof__(page) *)
295 __va(page))[address >>
296 PAGE_SHIFT];
297 printk(KERN_ALERT "*pte = %08lx\n", page);
298 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299 }
300 }
Paul Mundt0630e452007-06-18 19:02:47 +0900301
Linus Torvalds1da177e2005-04-16 15:20:36 -0700302 die("Oops", regs, writeaccess);
Paul Mundt0630e452007-06-18 19:02:47 +0900303 bust_spinlocks(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304 do_exit(SIGKILL);
305
306/*
307 * We ran out of memory, or some other thing happened to us that made
308 * us unable to handle the page fault gracefully.
309 */
310out_of_memory:
311 up_read(&mm->mmap_sem);
Nick Piggin6b6b18e2010-04-22 16:06:26 +0000312 if (!user_mode(regs))
313 goto no_context;
314 pagefault_out_of_memory();
315 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700316
317do_sigbus:
318 up_read(&mm->mmap_sem);
319
320 /*
321 * Send a sigbus, regardless of whether we were in kernel
322 * or user mode.
323 */
Stuart Menefyb5a1bcb2006-11-21 13:34:04 +0900324 info.si_signo = SIGBUS;
325 info.si_errno = 0;
326 info.si_code = BUS_ADRERR;
327 info.si_addr = (void *)address;
328 force_sig_info(SIGBUS, &info, tsk);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700329
330 /* Kernel mode? Handle exceptions or die */
331 if (!user_mode(regs))
332 goto no_context;
333}
Paul Mundtdb2e1fa2007-02-14 14:13:10 +0900334
Paul Mundtdb2e1fa2007-02-14 14:13:10 +0900335/*
336 * Called with interrupts disabled.
337 */
Paul Mundt112e5842009-08-15 02:49:40 +0900338asmlinkage int __kprobes
339handle_tlbmiss(struct pt_regs *regs, unsigned long writeaccess,
340 unsigned long address)
Paul Mundtdb2e1fa2007-02-14 14:13:10 +0900341{
342 pgd_t *pgd;
343 pud_t *pud;
344 pmd_t *pmd;
345 pte_t *pte;
346 pte_t entry;
Paul Mundt3d586952008-09-21 13:56:39 +0900347
Paul Mundtdb2e1fa2007-02-14 14:13:10 +0900348 /*
349 * We don't take page faults for P1, P2, and parts of P4, these
350 * are always mapped, whether it be due to legacy behaviour in
351 * 29-bit mode, or due to PMB configuration in 32-bit mode.
352 */
353 if (address >= P3SEG && address < P3_ADDR_MAX) {
354 pgd = pgd_offset_k(address);
Paul Mundtdb2e1fa2007-02-14 14:13:10 +0900355 } else {
Paul Mundt0f1a3942007-11-19 13:05:18 +0900356 if (unlikely(address >= TASK_SIZE || !current->mm))
Paul Mundt8010fbe2009-08-15 03:06:41 +0900357 return 1;
Paul Mundtdb2e1fa2007-02-14 14:13:10 +0900358
Paul Mundt0f1a3942007-11-19 13:05:18 +0900359 pgd = pgd_offset(current->mm, address);
Paul Mundtdb2e1fa2007-02-14 14:13:10 +0900360 }
361
362 pud = pud_offset(pgd, address);
363 if (pud_none_or_clear_bad(pud))
Paul Mundt8010fbe2009-08-15 03:06:41 +0900364 return 1;
Paul Mundtdb2e1fa2007-02-14 14:13:10 +0900365 pmd = pmd_offset(pud, address);
366 if (pmd_none_or_clear_bad(pmd))
Paul Mundt8010fbe2009-08-15 03:06:41 +0900367 return 1;
Paul Mundt0f1a3942007-11-19 13:05:18 +0900368 pte = pte_offset_kernel(pmd, address);
Paul Mundtdb2e1fa2007-02-14 14:13:10 +0900369 entry = *pte;
370 if (unlikely(pte_none(entry) || pte_not_present(entry)))
Paul Mundt8010fbe2009-08-15 03:06:41 +0900371 return 1;
Paul Mundtdb2e1fa2007-02-14 14:13:10 +0900372 if (unlikely(writeaccess && !pte_write(entry)))
Paul Mundt8010fbe2009-08-15 03:06:41 +0900373 return 1;
Paul Mundtdb2e1fa2007-02-14 14:13:10 +0900374
375 if (writeaccess)
376 entry = pte_mkdirty(entry);
377 entry = pte_mkyoung(entry);
378
Paul Mundt8010fbe2009-08-15 03:06:41 +0900379 set_pte(pte, entry);
380
Hideo Saitoa602cc02008-02-14 14:45:08 +0900381#if defined(CONFIG_CPU_SH4) && !defined(CONFIG_SMP)
382 /*
Paul Mundt8010fbe2009-08-15 03:06:41 +0900383 * SH-4 does not set MMUCR.RC to the corresponding TLB entry in
384 * the case of an initial page write exception, so we need to
385 * flush it in order to avoid potential TLB entry duplication.
Hideo Saitoa602cc02008-02-14 14:45:08 +0900386 */
Paul Mundt8010fbe2009-08-15 03:06:41 +0900387 if (writeaccess == 2)
388 local_flush_tlb_one(get_asid(), address & PAGE_MASK);
Hideo Saitoa602cc02008-02-14 14:45:08 +0900389#endif
390
Russell King4b3073e2009-12-18 16:40:18 +0000391 update_mmu_cache(NULL, address, pte);
Paul Mundt0f1a3942007-11-19 13:05:18 +0900392
Paul Mundt8010fbe2009-08-15 03:06:41 +0900393 return 0;
Paul Mundtdb2e1fa2007-02-14 14:13:10 +0900394}