blob: 3a94941578fab1420f00d71d8eb72a20cc1c1375 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/arch/x86-64/mm/fault.c
3 *
4 * Copyright (C) 1995 Linus Torvalds
5 * Copyright (C) 2001,2002 Andi Kleen, SuSE Labs.
6 */
7
Linus Torvalds1da177e2005-04-16 15:20:36 -07008#include <linux/signal.h>
9#include <linux/sched.h>
10#include <linux/kernel.h>
11#include <linux/errno.h>
12#include <linux/string.h>
13#include <linux/types.h>
14#include <linux/ptrace.h>
15#include <linux/mman.h>
16#include <linux/mm.h>
17#include <linux/smp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/interrupt.h>
19#include <linux/init.h>
20#include <linux/tty.h>
21#include <linux/vt_kern.h> /* For unblank_screen() */
22#include <linux/compiler.h>
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -070023#include <linux/vmalloc.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include <linux/module.h>
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -070025#include <linux/kprobes.h>
Andi Kleenab2bf0c2006-12-07 02:14:06 +010026#include <linux/uaccess.h>
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -070027#include <linux/kdebug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070028
29#include <asm/system.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070030#include <asm/pgalloc.h>
31#include <asm/smp.h>
32#include <asm/tlbflush.h>
33#include <asm/proto.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070034#include <asm-generic/sections.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070035
Andi Kleen66c58152006-01-11 22:44:09 +010036/* Page fault error code bits */
37#define PF_PROT (1<<0) /* or no page found */
38#define PF_WRITE (1<<1)
39#define PF_USER (1<<2)
40#define PF_RSVD (1<<3)
41#define PF_INSTR (1<<4)
42
Christoph Hellwig74a0b572007-10-16 01:24:07 -070043#ifdef CONFIG_KPROBES
44static inline int notify_page_fault(struct pt_regs *regs)
Anil S Keshavamurthy1bd858a2006-06-26 00:25:25 -070045{
Christoph Hellwig74a0b572007-10-16 01:24:07 -070046 int ret = 0;
Anil S Keshavamurthy1bd858a2006-06-26 00:25:25 -070047
Christoph Hellwig74a0b572007-10-16 01:24:07 -070048 /* kprobe_running() needs smp_processor_id() */
49 if (!user_mode(regs)) {
50 preempt_disable();
51 if (kprobe_running() && kprobe_fault_handler(regs, 14))
52 ret = 1;
53 preempt_enable();
54 }
Anil S Keshavamurthy1bd858a2006-06-26 00:25:25 -070055
Christoph Hellwig74a0b572007-10-16 01:24:07 -070056 return ret;
Anil S Keshavamurthy1bd858a2006-06-26 00:25:25 -070057}
Christoph Hellwig74a0b572007-10-16 01:24:07 -070058#else
59static inline int notify_page_fault(struct pt_regs *regs)
60{
61 return 0;
62}
63#endif
Anil S Keshavamurthy1bd858a2006-06-26 00:25:25 -070064
Linus Torvalds1da177e2005-04-16 15:20:36 -070065/* Sometimes the CPU reports invalid exceptions on prefetch.
66 Check that here and ignore.
67 Opcode checker based on code by Richard Brunner */
68static noinline int is_prefetch(struct pt_regs *regs, unsigned long addr,
69 unsigned long error_code)
70{
Andi Kleenab2bf0c2006-12-07 02:14:06 +010071 unsigned char *instr;
Linus Torvalds1da177e2005-04-16 15:20:36 -070072 int scan_more = 1;
73 int prefetch = 0;
Andi Kleenf1290ec2005-04-16 15:24:59 -070074 unsigned char *max_instr;
Linus Torvalds1da177e2005-04-16 15:20:36 -070075
76 /* If it was a exec fault ignore */
Andi Kleen66c58152006-01-11 22:44:09 +010077 if (error_code & PF_INSTR)
Linus Torvalds1da177e2005-04-16 15:20:36 -070078 return 0;
79
Andi Kleendd2994f2006-09-26 10:52:33 +020080 instr = (unsigned char __user *)convert_rip_to_linear(current, regs);
Andi Kleenf1290ec2005-04-16 15:24:59 -070081 max_instr = instr + 15;
Linus Torvalds1da177e2005-04-16 15:20:36 -070082
Vincent Hanquez76381fe2005-06-23 00:08:46 -070083 if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE)
Linus Torvalds1da177e2005-04-16 15:20:36 -070084 return 0;
85
86 while (scan_more && instr < max_instr) {
87 unsigned char opcode;
88 unsigned char instr_hi;
89 unsigned char instr_lo;
90
Andi Kleenab2bf0c2006-12-07 02:14:06 +010091 if (probe_kernel_address(instr, opcode))
Linus Torvalds1da177e2005-04-16 15:20:36 -070092 break;
93
94 instr_hi = opcode & 0xf0;
95 instr_lo = opcode & 0x0f;
96 instr++;
97
98 switch (instr_hi) {
99 case 0x20:
100 case 0x30:
101 /* Values 0x26,0x2E,0x36,0x3E are valid x86
102 prefixes. In long mode, the CPU will signal
103 invalid opcode if some of these prefixes are
104 present so we will never get here anyway */
105 scan_more = ((instr_lo & 7) == 0x6);
106 break;
107
108 case 0x40:
109 /* In AMD64 long mode, 0x40 to 0x4F are valid REX prefixes
110 Need to figure out under what instruction mode the
111 instruction was issued ... */
112 /* Could check the LDT for lm, but for now it's good
113 enough to assume that long mode only uses well known
114 segments or kernel. */
Vincent Hanquez76381fe2005-06-23 00:08:46 -0700115 scan_more = (!user_mode(regs)) || (regs->cs == __USER_CS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116 break;
117
118 case 0x60:
119 /* 0x64 thru 0x67 are valid prefixes in all modes. */
120 scan_more = (instr_lo & 0xC) == 0x4;
121 break;
122 case 0xF0:
123 /* 0xF0, 0xF2, and 0xF3 are valid prefixes in all modes. */
124 scan_more = !instr_lo || (instr_lo>>1) == 1;
125 break;
126 case 0x00:
127 /* Prefetch instruction is 0x0F0D or 0x0F18 */
128 scan_more = 0;
Andi Kleenab2bf0c2006-12-07 02:14:06 +0100129 if (probe_kernel_address(instr, opcode))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130 break;
131 prefetch = (instr_lo == 0xF) &&
132 (opcode == 0x0D || opcode == 0x18);
133 break;
134 default:
135 scan_more = 0;
136 break;
137 }
138 }
139 return prefetch;
140}
141
142static int bad_address(void *p)
143{
144 unsigned long dummy;
Andi Kleenab2bf0c2006-12-07 02:14:06 +0100145 return probe_kernel_address((unsigned long *)p, dummy);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146}
147
148void dump_pagetable(unsigned long address)
149{
150 pgd_t *pgd;
151 pud_t *pud;
152 pmd_t *pmd;
153 pte_t *pte;
154
Glauber de Oliveira Costaf51c9452007-07-22 11:12:29 +0200155 pgd = (pgd_t *)read_cr3();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156
157 pgd = __va((unsigned long)pgd & PHYSICAL_PAGE_MASK);
158 pgd += pgd_index(address);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159 if (bad_address(pgd)) goto bad;
Jan Beulichd646bce2006-02-03 21:51:47 +0100160 printk("PGD %lx ", pgd_val(*pgd));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161 if (!pgd_present(*pgd)) goto ret;
162
Andi Kleend2ae5b52006-06-26 13:57:56 +0200163 pud = pud_offset(pgd, address);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164 if (bad_address(pud)) goto bad;
165 printk("PUD %lx ", pud_val(*pud));
166 if (!pud_present(*pud)) goto ret;
167
168 pmd = pmd_offset(pud, address);
169 if (bad_address(pmd)) goto bad;
170 printk("PMD %lx ", pmd_val(*pmd));
Jan Beulichb1992df2007-10-19 20:35:03 +0200171 if (!pmd_present(*pmd) || pmd_large(*pmd)) goto ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172
173 pte = pte_offset_kernel(pmd, address);
174 if (bad_address(pte)) goto bad;
175 printk("PTE %lx", pte_val(*pte));
176ret:
177 printk("\n");
178 return;
179bad:
180 printk("BAD\n");
181}
182
183static const char errata93_warning[] =
184KERN_ERR "******* Your BIOS seems to not contain a fix for K8 errata #93\n"
185KERN_ERR "******* Working around it, but it may cause SEGVs or burn power.\n"
186KERN_ERR "******* Please consider a BIOS update.\n"
187KERN_ERR "******* Disabling USB legacy in the BIOS may also help.\n";
188
189/* Workaround for K8 erratum #93 & buggy BIOS.
190 BIOS SMM functions are required to use a specific workaround
191 to avoid corruption of the 64bit RIP register on C stepping K8.
192 A lot of BIOS that didn't get tested properly miss this.
193 The OS sees this as a page fault with the upper 32bits of RIP cleared.
194 Try to work around it here.
195 Note we only handle faults in kernel here. */
196
197static int is_errata93(struct pt_regs *regs, unsigned long address)
198{
199 static int warned;
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100200 if (address != regs->ip)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201 return 0;
202 if ((address >> 32) != 0)
203 return 0;
204 address |= 0xffffffffUL << 32;
205 if ((address >= (u64)_stext && address <= (u64)_etext) ||
206 (address >= MODULES_VADDR && address <= MODULES_END)) {
207 if (!warned) {
208 printk(errata93_warning);
209 warned = 1;
210 }
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100211 regs->ip = address;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212 return 1;
213 }
214 return 0;
215}
216
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
218 unsigned long error_code)
219{
Jan Beulich12091402005-09-12 18:49:24 +0200220 unsigned long flags = oops_begin();
Jan Beulich6e3f3612006-01-11 22:42:14 +0100221 struct task_struct *tsk;
Jan Beulich12091402005-09-12 18:49:24 +0200222
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223 printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
224 current->comm, address);
225 dump_pagetable(address);
Jan Beulich6e3f3612006-01-11 22:42:14 +0100226 tsk = current;
227 tsk->thread.cr2 = address;
228 tsk->thread.trap_no = 14;
229 tsk->thread.error_code = error_code;
Jan Beulich22f59912008-01-30 13:31:23 +0100230 if (__die("Bad pagetable", regs, error_code))
231 regs = NULL;
232 oops_end(flags, regs, SIGKILL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233}
234
235/*
Andi Kleenf95190b2006-01-11 22:44:00 +0100236 * Handle a fault on the vmalloc area
Andi Kleen3b9ba4d2005-05-16 21:53:31 -0700237 *
238 * This assumes no large pages in there.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239 */
240static int vmalloc_fault(unsigned long address)
241{
242 pgd_t *pgd, *pgd_ref;
243 pud_t *pud, *pud_ref;
244 pmd_t *pmd, *pmd_ref;
245 pte_t *pte, *pte_ref;
246
247 /* Copy kernel mappings over when needed. This can also
248 happen within a race in page table update. In the later
249 case just flush. */
250
251 pgd = pgd_offset(current->mm ?: &init_mm, address);
252 pgd_ref = pgd_offset_k(address);
253 if (pgd_none(*pgd_ref))
254 return -1;
255 if (pgd_none(*pgd))
256 set_pgd(pgd, *pgd_ref);
Jan Beulich8c914cb2006-03-25 16:29:40 +0100257 else
Dave McCracken46a82b22006-09-25 23:31:48 -0700258 BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259
260 /* Below here mismatches are bugs because these lower tables
261 are shared */
262
263 pud = pud_offset(pgd, address);
264 pud_ref = pud_offset(pgd_ref, address);
265 if (pud_none(*pud_ref))
266 return -1;
Dave McCracken46a82b22006-09-25 23:31:48 -0700267 if (pud_none(*pud) || pud_page_vaddr(*pud) != pud_page_vaddr(*pud_ref))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268 BUG();
269 pmd = pmd_offset(pud, address);
270 pmd_ref = pmd_offset(pud_ref, address);
271 if (pmd_none(*pmd_ref))
272 return -1;
273 if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
274 BUG();
275 pte_ref = pte_offset_kernel(pmd_ref, address);
276 if (!pte_present(*pte_ref))
277 return -1;
278 pte = pte_offset_kernel(pmd, address);
Andi Kleen3b9ba4d2005-05-16 21:53:31 -0700279 /* Don't use pte_page here, because the mappings can point
280 outside mem_map, and the NUMA hash lookup cannot handle
281 that. */
282 if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700283 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284 return 0;
285}
286
Masoud Asgharifard Sharbianiabd4f752007-07-22 11:12:28 +0200287int show_unhandled_signals = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288
289/*
290 * This routine handles page faults. It determines the address,
291 * and the problem, and then passes it off to one of the appropriate
292 * routines.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700293 */
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -0700294asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
295 unsigned long error_code)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296{
297 struct task_struct *tsk;
298 struct mm_struct *mm;
299 struct vm_area_struct * vma;
300 unsigned long address;
Nick Piggin83c54072007-07-19 01:47:05 -0700301 int write, fault;
Jan Beulich12091402005-09-12 18:49:24 +0200302 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303 siginfo_t info;
304
Peter Zijlstra143a5d32007-10-25 14:01:10 +0200305 /*
306 * We can fault from pretty much anywhere, with unknown IRQ state.
307 */
308 trace_hardirqs_fixup();
309
Arjan van de Vena9ba9a32006-03-25 16:30:10 +0100310 tsk = current;
311 mm = tsk->mm;
312 prefetchw(&mm->mmap_sem);
313
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314 /* get the address */
Glauber de Oliveira Costaf51c9452007-07-22 11:12:29 +0200315 address = read_cr2();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700316
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317 info.si_code = SEGV_MAPERR;
318
319
320 /*
321 * We fault-in kernel-space virtual memory on-demand. The
322 * 'reference' page table is init_mm.pgd.
323 *
324 * NOTE! We MUST NOT take any locks for this case. We may
325 * be in an interrupt or a critical region, and should
326 * only copy the information from the master page table,
327 * nothing more.
328 *
329 * This verifies that the fault happens in kernel space
330 * (error_code & 4) == 0, and that the fault was not a
Jan Beulich8b1bde92006-01-11 22:42:23 +0100331 * protection error (error_code & 9) == 0.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332 */
Suresh Siddha84929802005-06-21 17:14:32 -0700333 if (unlikely(address >= TASK_SIZE64)) {
Andi Kleenf95190b2006-01-11 22:44:00 +0100334 /*
335 * Don't check for the module range here: its PML4
336 * is always initialized because it's shared with the main
337 * kernel text. Only vmalloc may need PML4 syncups.
338 */
Andi Kleen66c58152006-01-11 22:44:09 +0100339 if (!(error_code & (PF_RSVD|PF_USER|PF_PROT)) &&
Andi Kleenf95190b2006-01-11 22:44:00 +0100340 ((address >= VMALLOC_START && address < VMALLOC_END))) {
Jan Beulich8c914cb2006-03-25 16:29:40 +0100341 if (vmalloc_fault(address) >= 0)
342 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343 }
Christoph Hellwig74a0b572007-10-16 01:24:07 -0700344 if (notify_page_fault(regs))
Jan Beulich8c914cb2006-03-25 16:29:40 +0100345 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346 /*
347 * Don't take the mm semaphore here. If we fixup a prefetch
348 * fault we could otherwise deadlock.
349 */
350 goto bad_area_nosemaphore;
351 }
352
Christoph Hellwig74a0b572007-10-16 01:24:07 -0700353 if (notify_page_fault(regs))
Jan Beulich8c914cb2006-03-25 16:29:40 +0100354 return;
355
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100356 if (likely(regs->flags & X86_EFLAGS_IF))
Jan Beulich8c914cb2006-03-25 16:29:40 +0100357 local_irq_enable();
358
Andi Kleen66c58152006-01-11 22:44:09 +0100359 if (unlikely(error_code & PF_RSVD))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360 pgtable_bad(address, regs, error_code);
361
362 /*
363 * If we're in an interrupt or have no user
364 * context, we must not take the fault..
365 */
366 if (unlikely(in_atomic() || !mm))
367 goto bad_area_nosemaphore;
368
Linus Torvaldsdbe3ed12007-09-19 11:37:14 -0700369 /*
370 * User-mode registers count as a user access even for any
371 * potential system fault or CPU buglet.
372 */
373 if (user_mode_vm(regs))
374 error_code |= PF_USER;
375
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376 again:
377 /* When running in the kernel we expect faults to occur only to
378 * addresses in user space. All other faults represent errors in the
Simon Arlott676b1852007-10-20 01:25:36 +0200379 * kernel and should generate an OOPS. Unfortunately, in the case of an
Adrian Bunk80f72282006-06-30 18:27:16 +0200380 * erroneous fault occurring in a code path which already holds mmap_sem
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381 * we will deadlock attempting to validate the fault against the
382 * address space. Luckily the kernel only validly references user
383 * space from well defined areas of code, which are listed in the
384 * exceptions table.
385 *
386 * As the vast majority of faults will be valid we will only perform
Simon Arlott676b1852007-10-20 01:25:36 +0200387 * the source reference check when there is a possibility of a deadlock.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388 * Attempt to lock the address space, if we cannot we then validate the
389 * source. If this is invalid we can skip the address space check,
390 * thus avoiding the deadlock.
391 */
392 if (!down_read_trylock(&mm->mmap_sem)) {
Andi Kleen66c58152006-01-11 22:44:09 +0100393 if ((error_code & PF_USER) == 0 &&
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100394 !search_exception_tables(regs->ip))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395 goto bad_area_nosemaphore;
396 down_read(&mm->mmap_sem);
397 }
398
399 vma = find_vma(mm, address);
400 if (!vma)
401 goto bad_area;
402 if (likely(vma->vm_start <= address))
403 goto good_area;
404 if (!(vma->vm_flags & VM_GROWSDOWN))
405 goto bad_area;
406 if (error_code & 4) {
Chuck Ebbert03fdc2c2006-06-26 13:59:50 +0200407 /* Allow userspace just enough access below the stack pointer
408 * to let the 'enter' instruction work.
409 */
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100410 if (address + 65536 + 32 * sizeof(unsigned long) < regs->sp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411 goto bad_area;
412 }
413 if (expand_stack(vma, address))
414 goto bad_area;
415/*
416 * Ok, we have a good vm_area for this memory access, so
417 * we can handle it..
418 */
419good_area:
420 info.si_code = SEGV_ACCERR;
421 write = 0;
Andi Kleen66c58152006-01-11 22:44:09 +0100422 switch (error_code & (PF_PROT|PF_WRITE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423 default: /* 3: write, present */
424 /* fall through */
Andi Kleen66c58152006-01-11 22:44:09 +0100425 case PF_WRITE: /* write, not present */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426 if (!(vma->vm_flags & VM_WRITE))
427 goto bad_area;
428 write++;
429 break;
Andi Kleen66c58152006-01-11 22:44:09 +0100430 case PF_PROT: /* read, present */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700431 goto bad_area;
Andi Kleen66c58152006-01-11 22:44:09 +0100432 case 0: /* read, not present */
Jason Barondf67b3d2006-09-29 01:58:58 -0700433 if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434 goto bad_area;
435 }
436
437 /*
438 * If for any reason at all we couldn't handle the fault,
439 * make sure we exit gracefully rather than endlessly redo
440 * the fault.
441 */
Nick Piggin83c54072007-07-19 01:47:05 -0700442 fault = handle_mm_fault(mm, vma, address, write);
443 if (unlikely(fault & VM_FAULT_ERROR)) {
444 if (fault & VM_FAULT_OOM)
445 goto out_of_memory;
446 else if (fault & VM_FAULT_SIGBUS)
447 goto do_sigbus;
448 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449 }
Nick Piggin83c54072007-07-19 01:47:05 -0700450 if (fault & VM_FAULT_MAJOR)
451 tsk->maj_flt++;
452 else
453 tsk->min_flt++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454 up_read(&mm->mmap_sem);
455 return;
456
457/*
458 * Something tried to access memory that isn't in our memory map..
459 * Fix it, but check if it's kernel or user first..
460 */
461bad_area:
462 up_read(&mm->mmap_sem);
463
464bad_area_nosemaphore:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465 /* User mode accesses just cause a SIGSEGV */
Andi Kleen66c58152006-01-11 22:44:09 +0100466 if (error_code & PF_USER) {
Steven Rostedte5e3c842007-06-06 23:34:04 -0400467
468 /*
469 * It's possible to have interrupts off here.
470 */
471 local_irq_enable();
472
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473 if (is_prefetch(regs, address, error_code))
474 return;
475
476 /* Work around K8 erratum #100 K8 in compat mode
477 occasionally jumps to illegal addresses >4GB. We
478 catch this here in the page fault handler because
479 these addresses are not reachable. Just detect this
480 case and return. Any code segment in LDT is
481 compatibility mode. */
482 if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) &&
483 (address >> 32))
484 return;
485
Masoud Asgharifard Sharbianiabd4f752007-07-22 11:12:28 +0200486 if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
487 printk_ratelimit()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488 printk(
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100489 "%s%s[%d]: segfault at %lx ip %lx sp %lx error %lx\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700490 tsk->pid > 1 ? KERN_INFO : KERN_EMERG,
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100491 tsk->comm, tsk->pid, address, regs->ip,
492 regs->sp, error_code);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700493 }
494
495 tsk->thread.cr2 = address;
496 /* Kernel addresses are always protection faults */
497 tsk->thread.error_code = error_code | (address >= TASK_SIZE);
498 tsk->thread.trap_no = 14;
499 info.si_signo = SIGSEGV;
500 info.si_errno = 0;
501 /* info.si_code has been set above */
502 info.si_addr = (void __user *)address;
503 force_sig_info(SIGSEGV, &info, tsk);
504 return;
505 }
506
507no_context:
508
509 /* Are we prepared to handle this kernel fault? */
Harvey Harrison85f2adf2008-01-30 13:31:42 +0100510 if (fixup_exception(regs)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700511 return;
512 }
513
514 /*
515 * Hall of shame of CPU/BIOS bugs.
516 */
517
518 if (is_prefetch(regs, address, error_code))
519 return;
520
521 if (is_errata93(regs, address))
522 return;
523
524/*
525 * Oops. The kernel tried to access some bad page. We'll have to
526 * terminate things with extreme prejudice.
527 */
528
Jan Beulich12091402005-09-12 18:49:24 +0200529 flags = oops_begin();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530
531 if (address < PAGE_SIZE)
532 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
533 else
534 printk(KERN_ALERT "Unable to handle kernel paging request");
535 printk(" at %016lx RIP: \n" KERN_ALERT,address);
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100536 printk_address(regs->ip);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537 dump_pagetable(address);
Jan Beulich6e3f3612006-01-11 22:42:14 +0100538 tsk->thread.cr2 = address;
539 tsk->thread.trap_no = 14;
540 tsk->thread.error_code = error_code;
Jan Beulich22f59912008-01-30 13:31:23 +0100541 if (__die("Oops", regs, error_code))
542 regs = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543 /* Executive summary in case the body of the oops scrolled away */
544 printk(KERN_EMERG "CR2: %016lx\n", address);
Jan Beulich22f59912008-01-30 13:31:23 +0100545 oops_end(flags, regs, SIGKILL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700546
547/*
548 * We ran out of memory, or some other thing happened to us that made
549 * us unable to handle the page fault gracefully.
550 */
551out_of_memory:
552 up_read(&mm->mmap_sem);
Serge E. Hallynb460cbc2007-10-18 23:39:52 -0700553 if (is_global_init(current)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554 yield();
555 goto again;
556 }
557 printk("VM: killing process %s\n", tsk->comm);
558 if (error_code & 4)
Will Schmidt021daae2007-07-21 17:11:17 +0200559 do_group_exit(SIGKILL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700560 goto no_context;
561
562do_sigbus:
563 up_read(&mm->mmap_sem);
564
565 /* Kernel mode? Handle exceptions or die */
Andi Kleen66c58152006-01-11 22:44:09 +0100566 if (!(error_code & PF_USER))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700567 goto no_context;
568
569 tsk->thread.cr2 = address;
570 tsk->thread.error_code = error_code;
571 tsk->thread.trap_no = 14;
572 info.si_signo = SIGBUS;
573 info.si_errno = 0;
574 info.si_code = BUS_ADRERR;
575 info.si_addr = (void __user *)address;
576 force_sig_info(SIGBUS, &info, tsk);
577 return;
578}
Andi Kleen9e43e1b2005-11-05 17:25:54 +0100579
Jan Beulich8c914cb2006-03-25 16:29:40 +0100580DEFINE_SPINLOCK(pgd_lock);
Christoph Lameter2bff7382007-05-02 19:27:10 +0200581LIST_HEAD(pgd_list);
Jan Beulich8c914cb2006-03-25 16:29:40 +0100582
583void vmalloc_sync_all(void)
584{
585 /* Note that races in the updates of insync and start aren't
586 problematic:
587 insync can only get set bits added, and updates to start are only
588 improving performance (without affecting correctness if undone). */
589 static DECLARE_BITMAP(insync, PTRS_PER_PGD);
590 static unsigned long start = VMALLOC_START & PGDIR_MASK;
591 unsigned long address;
592
593 for (address = start; address <= VMALLOC_END; address += PGDIR_SIZE) {
594 if (!test_bit(pgd_index(address), insync)) {
595 const pgd_t *pgd_ref = pgd_offset_k(address);
596 struct page *page;
597
598 if (pgd_none(*pgd_ref))
599 continue;
600 spin_lock(&pgd_lock);
Christoph Lameter2bff7382007-05-02 19:27:10 +0200601 list_for_each_entry(page, &pgd_list, lru) {
Jan Beulich8c914cb2006-03-25 16:29:40 +0100602 pgd_t *pgd;
603 pgd = (pgd_t *)page_address(page) + pgd_index(address);
604 if (pgd_none(*pgd))
605 set_pgd(pgd, *pgd_ref);
606 else
Dave McCracken46a82b22006-09-25 23:31:48 -0700607 BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
Jan Beulich8c914cb2006-03-25 16:29:40 +0100608 }
609 spin_unlock(&pgd_lock);
610 set_bit(pgd_index(address), insync);
611 }
612 if (address == start)
613 start = address + PGDIR_SIZE;
614 }
615 /* Check that there is no need to do the same for the modules area. */
616 BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));
617 BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
618 (__START_KERNEL & PGDIR_MASK)));
619}