blob: d0812e155f1d60da04c614bee8b97e2e33869876 [file] [log] [blame]
H. Peter Anvin1965aae2008-10-22 22:26:29 -07001#ifndef _ASM_X86_PGTABLE_H
2#define _ASM_X86_PGTABLE_H
Jeremy Fitzhardinge6c386652008-01-30 13:32:55 +01003
Ingo Molnarc47c1b12009-02-09 11:57:45 +01004#include <asm/page.h>
5
Jeremy Fitzhardinge8d19c992009-02-08 18:46:18 -08006#include <asm/pgtable_types.h>
Suresh Siddhab2bc2732008-09-23 14:00:36 -07007
venkatesh.pallipadi@intel.com8a7b12f2008-12-18 11:41:31 -08008/*
9 * Macro to mark a page protection value as UC-
10 */
11#define pgprot_noncached(prot) \
12 ((boot_cpu_data.x86 > 3) \
13 ? (__pgprot(pgprot_val(prot) | _PAGE_CACHE_UC_MINUS)) \
14 : (prot))
15
Jeremy Fitzhardinge46141392008-01-30 13:32:56 +010016#ifndef __ASSEMBLY__
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +010017
Jeremy Fitzhardinge46141392008-01-30 13:32:56 +010018/*
Jeremy Fitzhardinge8405b122008-01-30 13:32:58 +010019 * ZERO_PAGE is a global shared page that is always zero: used
20 * for zero-mapped memory areas etc..
21 */
Joe Perches3cbaeaf2008-03-23 01:03:12 -070022extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
Jeremy Fitzhardinge8405b122008-01-30 13:32:58 +010023#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page))
24
Jeremy Fitzhardingee3ed9102008-01-30 13:34:11 +010025extern spinlock_t pgd_lock;
26extern struct list_head pgd_list;
Jeremy Fitzhardinge8405b122008-01-30 13:32:58 +010027
Jeremy Fitzhardinge54321d92009-02-11 10:20:05 -080028#ifdef CONFIG_PARAVIRT
29#include <asm/paravirt.h>
30#else /* !CONFIG_PARAVIRT */
31#define set_pte(ptep, pte) native_set_pte(ptep, pte)
32#define set_pte_at(mm, addr, ptep, pte) native_set_pte_at(mm, addr, ptep, pte)
33
34#define set_pte_present(mm, addr, ptep, pte) \
35 native_set_pte_present(mm, addr, ptep, pte)
36#define set_pte_atomic(ptep, pte) \
37 native_set_pte_atomic(ptep, pte)
38
39#define set_pmd(pmdp, pmd) native_set_pmd(pmdp, pmd)
40
41#ifndef __PAGETABLE_PUD_FOLDED
42#define set_pgd(pgdp, pgd) native_set_pgd(pgdp, pgd)
43#define pgd_clear(pgd) native_pgd_clear(pgd)
44#endif
45
46#ifndef set_pud
47# define set_pud(pudp, pud) native_set_pud(pudp, pud)
48#endif
49
50#ifndef __PAGETABLE_PMD_FOLDED
51#define pud_clear(pud) native_pud_clear(pud)
52#endif
53
54#define pte_clear(mm, addr, ptep) native_pte_clear(mm, addr, ptep)
55#define pmd_clear(pmd) native_pmd_clear(pmd)
56
57#define pte_update(mm, addr, ptep) do { } while (0)
58#define pte_update_defer(mm, addr, ptep) do { } while (0)
59
60static inline void __init paravirt_pagetable_setup_start(pgd_t *base)
61{
62 native_pagetable_setup_start(base);
63}
64
65static inline void __init paravirt_pagetable_setup_done(pgd_t *base)
66{
67 native_pagetable_setup_done(base);
68}
69
70#define pgd_val(x) native_pgd_val(x)
71#define __pgd(x) native_make_pgd(x)
72
73#ifndef __PAGETABLE_PUD_FOLDED
74#define pud_val(x) native_pud_val(x)
75#define __pud(x) native_make_pud(x)
76#endif
77
78#ifndef __PAGETABLE_PMD_FOLDED
79#define pmd_val(x) native_pmd_val(x)
80#define __pmd(x) native_make_pmd(x)
81#endif
82
83#define pte_val(x) native_pte_val(x)
84#define __pte(x) native_make_pte(x)
85
86#endif /* CONFIG_PARAVIRT */
87
Jeremy Fitzhardinge8405b122008-01-30 13:32:58 +010088/*
Jeremy Fitzhardinge46141392008-01-30 13:32:56 +010089 * The following only work if pte_present() is true.
90 * Undefined behaviour if not..
91 */
Joe Perches3cbaeaf2008-03-23 01:03:12 -070092static inline int pte_dirty(pte_t pte)
93{
Jeremy Fitzhardingea15af1c2008-05-26 23:31:06 +010094 return pte_flags(pte) & _PAGE_DIRTY;
Jeremy Fitzhardinge46141392008-01-30 13:32:56 +010095}
96
Joe Perches3cbaeaf2008-03-23 01:03:12 -070097static inline int pte_young(pte_t pte)
98{
Jeremy Fitzhardingea15af1c2008-05-26 23:31:06 +010099 return pte_flags(pte) & _PAGE_ACCESSED;
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700100}
101
102static inline int pte_write(pte_t pte)
103{
Jeremy Fitzhardingea15af1c2008-05-26 23:31:06 +0100104 return pte_flags(pte) & _PAGE_RW;
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700105}
106
107static inline int pte_file(pte_t pte)
108{
Jeremy Fitzhardingea15af1c2008-05-26 23:31:06 +0100109 return pte_flags(pte) & _PAGE_FILE;
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700110}
111
112static inline int pte_huge(pte_t pte)
113{
Jeremy Fitzhardingea15af1c2008-05-26 23:31:06 +0100114 return pte_flags(pte) & _PAGE_PSE;
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700115}
116
117static inline int pte_global(pte_t pte)
118{
Jeremy Fitzhardingea15af1c2008-05-26 23:31:06 +0100119 return pte_flags(pte) & _PAGE_GLOBAL;
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700120}
121
122static inline int pte_exec(pte_t pte)
123{
Jeremy Fitzhardingea15af1c2008-05-26 23:31:06 +0100124 return !(pte_flags(pte) & _PAGE_NX);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700125}
126
Nick Piggin7e675132008-04-28 02:13:00 -0700127static inline int pte_special(pte_t pte)
128{
Jan Beulich606ee442008-09-17 16:48:17 +0100129 return pte_flags(pte) & _PAGE_SPECIAL;
Nick Piggin7e675132008-04-28 02:13:00 -0700130}
131
Hugh Dickins91030ca2008-09-09 16:42:45 +0100132static inline unsigned long pte_pfn(pte_t pte)
133{
134 return (pte_val(pte) & PTE_PFN_MASK) >> PAGE_SHIFT;
135}
136
137#define pte_page(pte) pfn_to_page(pte_pfn(pte))
138
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700139static inline int pmd_large(pmd_t pte)
140{
Jeremy Fitzhardinge18a7a1992009-02-05 11:31:16 -0800141 return (pmd_flags(pte) & (_PAGE_PSE | _PAGE_PRESENT)) ==
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700142 (_PAGE_PSE | _PAGE_PRESENT);
143}
144
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800145static inline pte_t pte_set_flags(pte_t pte, pteval_t set)
146{
147 pteval_t v = native_pte_val(pte);
148
149 return native_make_pte(v | set);
150}
151
152static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear)
153{
154 pteval_t v = native_pte_val(pte);
155
156 return native_make_pte(v & ~clear);
157}
158
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700159static inline pte_t pte_mkclean(pte_t pte)
160{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800161 return pte_clear_flags(pte, _PAGE_DIRTY);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700162}
163
164static inline pte_t pte_mkold(pte_t pte)
165{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800166 return pte_clear_flags(pte, _PAGE_ACCESSED);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700167}
168
169static inline pte_t pte_wrprotect(pte_t pte)
170{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800171 return pte_clear_flags(pte, _PAGE_RW);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700172}
173
174static inline pte_t pte_mkexec(pte_t pte)
175{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800176 return pte_clear_flags(pte, _PAGE_NX);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700177}
178
179static inline pte_t pte_mkdirty(pte_t pte)
180{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800181 return pte_set_flags(pte, _PAGE_DIRTY);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700182}
183
184static inline pte_t pte_mkyoung(pte_t pte)
185{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800186 return pte_set_flags(pte, _PAGE_ACCESSED);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700187}
188
189static inline pte_t pte_mkwrite(pte_t pte)
190{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800191 return pte_set_flags(pte, _PAGE_RW);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700192}
193
194static inline pte_t pte_mkhuge(pte_t pte)
195{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800196 return pte_set_flags(pte, _PAGE_PSE);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700197}
198
199static inline pte_t pte_clrhuge(pte_t pte)
200{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800201 return pte_clear_flags(pte, _PAGE_PSE);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700202}
203
204static inline pte_t pte_mkglobal(pte_t pte)
205{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800206 return pte_set_flags(pte, _PAGE_GLOBAL);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700207}
208
209static inline pte_t pte_clrglobal(pte_t pte)
210{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800211 return pte_clear_flags(pte, _PAGE_GLOBAL);
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700212}
Jeremy Fitzhardinge46141392008-01-30 13:32:56 +0100213
Nick Piggin7e675132008-04-28 02:13:00 -0700214static inline pte_t pte_mkspecial(pte_t pte)
215{
Jeremy Fitzhardinge65228692009-01-22 14:24:22 -0800216 return pte_set_flags(pte, _PAGE_SPECIAL);
Nick Piggin7e675132008-04-28 02:13:00 -0700217}
218
Jeremy Fitzhardingeb5348162009-02-04 18:33:38 -0800219/*
220 * Mask out unsupported bits in a present pgprot. Non-present pgprots
221 * can use those bits for other purposes, so leave them be.
222 */
223static inline pgprotval_t massage_pgprot(pgprot_t pgprot)
224{
225 pgprotval_t protval = pgprot_val(pgprot);
226
227 if (protval & _PAGE_PRESENT)
228 protval &= __supported_pte_mask;
229
230 return protval;
231}
232
Jeremy Fitzhardinge6fdc05d2008-01-30 13:32:57 +0100233static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot)
234{
Jeremy Fitzhardingeb5348162009-02-04 18:33:38 -0800235 return __pte(((phys_addr_t)page_nr << PAGE_SHIFT) |
236 massage_pgprot(pgprot));
Jeremy Fitzhardinge6fdc05d2008-01-30 13:32:57 +0100237}
238
239static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
240{
Jeremy Fitzhardingeb5348162009-02-04 18:33:38 -0800241 return __pmd(((phys_addr_t)page_nr << PAGE_SHIFT) |
242 massage_pgprot(pgprot));
Jeremy Fitzhardinge6fdc05d2008-01-30 13:32:57 +0100243}
244
Ingo Molnar38472312008-01-30 13:32:57 +0100245static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
246{
247 pteval_t val = pte_val(pte);
248
249 /*
250 * Chop off the NX bit (if present), and add the NX portion of
251 * the newprot (if present):
252 */
Venki Pallipadi1c12c4c2008-05-14 16:05:51 -0700253 val &= _PAGE_CHG_MASK;
Jeremy Fitzhardingeb5348162009-02-04 18:33:38 -0800254 val |= massage_pgprot(newprot) & ~_PAGE_CHG_MASK;
Ingo Molnar38472312008-01-30 13:32:57 +0100255
256 return __pte(val);
257}
258
Venki Pallipadi1c12c4c2008-05-14 16:05:51 -0700259/* mprotect needs to preserve PAT bits when updating vm_page_prot */
260#define pgprot_modify pgprot_modify
261static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
262{
263 pgprotval_t preservebits = pgprot_val(oldprot) & _PAGE_CHG_MASK;
264 pgprotval_t addbits = pgprot_val(newprot);
265 return __pgprot(preservebits | addbits);
266}
267
Jeremy Fitzhardinge77be1fa2008-07-21 22:59:56 -0700268#define pte_pgprot(x) __pgprot(pte_flags(x) & PTE_FLAGS_MASK)
Andi Kleenc6ca18e2008-01-30 13:33:51 +0100269
Jeremy Fitzhardingeb5348162009-02-04 18:33:38 -0800270#define canon_pgprot(p) __pgprot(massage_pgprot(p))
Andi Kleen1e8e23b2008-01-30 13:33:53 +0100271
venkatesh.pallipadi@intel.comafc7d202009-01-09 16:13:10 -0800272static inline int is_new_memtype_allowed(unsigned long flags,
273 unsigned long new_flags)
274{
275 /*
276 * Certain new memtypes are not allowed with certain
277 * requested memtype:
278 * - request is uncached, return cannot be write-back
279 * - request is write-combine, return cannot be write-back
280 */
281 if ((flags == _PAGE_CACHE_UC_MINUS &&
282 new_flags == _PAGE_CACHE_WB) ||
283 (flags == _PAGE_CACHE_WC &&
284 new_flags == _PAGE_CACHE_WB)) {
285 return 0;
286 }
287
288 return 1;
289}
290
Tejun Heo458a3e62009-02-24 11:57:21 +0900291pmd_t *populate_extra_pmd(unsigned long vaddr);
292pte_t *populate_extra_pte(unsigned long vaddr);
Jeremy Fitzhardinge46141392008-01-30 13:32:56 +0100293#endif /* __ASSEMBLY__ */
294
Thomas Gleixner96a388d2007-10-11 11:20:03 +0200295#ifdef CONFIG_X86_32
296# include "pgtable_32.h"
297#else
298# include "pgtable_64.h"
299#endif
Jeremy Fitzhardinge6c386652008-01-30 13:32:55 +0100300
Jeremy Fitzhardingeaca159d2009-02-05 11:30:54 -0800301#ifndef __ASSEMBLY__
Jeremy Fitzhardingef4769612009-02-05 11:30:55 -0800302#include <linux/mm_types.h>
Jeremy Fitzhardingeaca159d2009-02-05 11:30:54 -0800303
Jeremy Fitzhardingea034a012009-02-05 11:30:43 -0800304static inline int pte_none(pte_t pte)
305{
306 return !pte.pte;
307}
308
Jeremy Fitzhardinge8de01da2009-02-05 11:30:44 -0800309#define __HAVE_ARCH_PTE_SAME
310static inline int pte_same(pte_t a, pte_t b)
311{
312 return a.pte == b.pte;
313}
314
Jeremy Fitzhardinge7c683852009-02-05 11:30:45 -0800315static inline int pte_present(pte_t a)
316{
317 return pte_flags(a) & (_PAGE_PRESENT | _PAGE_PROTNONE);
318}
319
Jeremy Fitzhardinge649e8ef2009-02-05 11:30:50 -0800320static inline int pmd_present(pmd_t pmd)
321{
Jeremy Fitzhardinge18a7a1992009-02-05 11:31:16 -0800322 return pmd_flags(pmd) & _PAGE_PRESENT;
Jeremy Fitzhardinge649e8ef2009-02-05 11:30:50 -0800323}
324
Jeremy Fitzhardinge4fea8012009-02-05 11:30:51 -0800325static inline int pmd_none(pmd_t pmd)
326{
327 /* Only check low word on 32-bit platforms, since it might be
328 out of sync with upper half. */
Jeremy Fitzhardinge26c8e312009-02-05 11:31:17 -0800329 return (unsigned long)native_pmd_val(pmd) == 0;
Jeremy Fitzhardinge4fea8012009-02-05 11:30:51 -0800330}
331
Jeremy Fitzhardinge3ffb3562009-02-05 11:30:59 -0800332static inline unsigned long pmd_page_vaddr(pmd_t pmd)
333{
334 return (unsigned long)__va(pmd_val(pmd) & PTE_PFN_MASK);
335}
336
Ingo Molnare5f7f2022009-02-09 11:42:57 +0100337/*
338 * Currently stuck as a macro due to indirect forward reference to
339 * linux/mmzone.h's __section_mem_map_addr() definition:
340 */
341#define pmd_page(pmd) pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)
Jeremy Fitzhardinge20063ca2009-02-05 11:31:00 -0800342
Jeremy Fitzhardingee24d7ee2009-02-05 11:31:01 -0800343/*
344 * the pmd page can be thought of an array like this: pmd_t[PTRS_PER_PMD]
345 *
346 * this macro returns the index of the entry in the pmd page which would
347 * control the given virtual address
348 */
349static inline unsigned pmd_index(unsigned long address)
350{
351 return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1);
352}
353
Jeremy Fitzhardinge97e28172009-02-05 11:31:05 -0800354/*
355 * Conversion functions: convert a page and protection to a page entry,
356 * and a page entry and page directory to the page they refer to.
357 *
358 * (Currently stuck as a macro because of indirect forward reference
359 * to linux/mm.h:page_to_nid())
360 */
361#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot))
362
Jeremy Fitzhardinge346309c2009-02-05 11:31:06 -0800363/*
364 * the pte page can be thought of an array like this: pte_t[PTRS_PER_PTE]
365 *
366 * this function returns the index of the entry in the pte page which would
367 * control the given virtual address
368 */
369static inline unsigned pte_index(unsigned long address)
370{
371 return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
372}
373
Jeremy Fitzhardinge3fbc2442009-02-05 11:31:07 -0800374static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address)
375{
376 return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address);
377}
378
Jeremy Fitzhardinge99510232009-02-05 11:31:11 -0800379static inline int pmd_bad(pmd_t pmd)
380{
Jeremy Fitzhardinge18a7a1992009-02-05 11:31:16 -0800381 return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE;
Jeremy Fitzhardinge99510232009-02-05 11:31:11 -0800382}
383
Jeremy Fitzhardingecc290ca2009-02-05 11:31:12 -0800384static inline unsigned long pages_to_mb(unsigned long npg)
385{
386 return npg >> (20 - PAGE_SHIFT);
387}
388
Jeremy Fitzhardinge6cf71502009-02-05 11:31:15 -0800389#define io_remap_pfn_range(vma, vaddr, pfn, size, prot) \
390 remap_pfn_range(vma, vaddr, pfn, size, prot)
391
Jeremy Fitzhardinge5ba7c912009-02-05 11:30:48 -0800392#if PAGETABLE_LEVELS > 2
Jeremy Fitzhardingedeb79cf2009-02-05 11:31:13 -0800393static inline int pud_none(pud_t pud)
394{
Jeremy Fitzhardinge26c8e312009-02-05 11:31:17 -0800395 return native_pud_val(pud) == 0;
Jeremy Fitzhardingedeb79cf2009-02-05 11:31:13 -0800396}
397
Jeremy Fitzhardinge5ba7c912009-02-05 11:30:48 -0800398static inline int pud_present(pud_t pud)
399{
Jeremy Fitzhardinge18a7a1992009-02-05 11:31:16 -0800400 return pud_flags(pud) & _PAGE_PRESENT;
Jeremy Fitzhardinge5ba7c912009-02-05 11:30:48 -0800401}
Jeremy Fitzhardinge6fff47e2009-02-05 11:30:53 -0800402
403static inline unsigned long pud_page_vaddr(pud_t pud)
404{
405 return (unsigned long)__va((unsigned long)pud_val(pud) & PTE_PFN_MASK);
406}
Jeremy Fitzhardingef4769612009-02-05 11:30:55 -0800407
Ingo Molnare5f7f2022009-02-09 11:42:57 +0100408/*
409 * Currently stuck as a macro due to indirect forward reference to
410 * linux/mmzone.h's __section_mem_map_addr() definition:
411 */
412#define pud_page(pud) pfn_to_page(pud_val(pud) >> PAGE_SHIFT)
Jeremy Fitzhardinge01ade20d5a2009-02-05 11:31:02 -0800413
414/* Find an entry in the second-level page table.. */
415static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
416{
417 return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(address);
418}
Jeremy Fitzhardinge3180fba2009-02-05 11:31:04 -0800419
420static inline unsigned long pmd_pfn(pmd_t pmd)
421{
422 return (pmd_val(pmd) & PTE_PFN_MASK) >> PAGE_SHIFT;
423}
Jeremy Fitzhardinge3f6cbef2009-02-05 11:31:08 -0800424
425static inline int pud_large(pud_t pud)
426{
Jeremy Fitzhardingee2f5bda2009-02-09 00:09:52 -0800427 return (pud_val(pud) & (_PAGE_PSE | _PAGE_PRESENT)) ==
Jeremy Fitzhardinge3f6cbef2009-02-05 11:31:08 -0800428 (_PAGE_PSE | _PAGE_PRESENT);
429}
Jeremy Fitzhardingea61bb292009-02-05 11:31:10 -0800430
431static inline int pud_bad(pud_t pud)
432{
Jeremy Fitzhardinge18a7a1992009-02-05 11:31:16 -0800433 return (pud_flags(pud) & ~(_KERNPG_TABLE | _PAGE_USER)) != 0;
Jeremy Fitzhardingea61bb292009-02-05 11:31:10 -0800434}
Jeremy Fitzhardingee2f5bda2009-02-09 00:09:52 -0800435#else
436static inline int pud_large(pud_t pud)
437{
438 return 0;
439}
Jeremy Fitzhardinge5ba7c912009-02-05 11:30:48 -0800440#endif /* PAGETABLE_LEVELS > 2 */
441
Jeremy Fitzhardinge9f38d7e2009-02-05 11:30:49 -0800442#if PAGETABLE_LEVELS > 3
443static inline int pgd_present(pgd_t pgd)
444{
Jeremy Fitzhardinge18a7a1992009-02-05 11:31:16 -0800445 return pgd_flags(pgd) & _PAGE_PRESENT;
Jeremy Fitzhardinge9f38d7e2009-02-05 11:30:49 -0800446}
Jeremy Fitzhardingec5f040b2009-02-05 11:30:52 -0800447
448static inline unsigned long pgd_page_vaddr(pgd_t pgd)
449{
450 return (unsigned long)__va((unsigned long)pgd_val(pgd) & PTE_PFN_MASK);
451}
Jeremy Fitzhardinge777cba12009-02-05 11:30:56 -0800452
Ingo Molnare5f7f2022009-02-09 11:42:57 +0100453/*
454 * Currently stuck as a macro due to indirect forward reference to
455 * linux/mmzone.h's __section_mem_map_addr() definition:
456 */
457#define pgd_page(pgd) pfn_to_page(pgd_val(pgd) >> PAGE_SHIFT)
Jeremy Fitzhardinge7cfb8102009-02-05 11:30:57 -0800458
459/* to find an entry in a page-table-directory. */
460static inline unsigned pud_index(unsigned long address)
461{
462 return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1);
463}
Jeremy Fitzhardinge3d081b12009-02-05 11:30:58 -0800464
465static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address)
466{
467 return (pud_t *)pgd_page_vaddr(*pgd) + pud_index(address);
468}
Jeremy Fitzhardinge30f10312009-02-05 11:31:09 -0800469
470static inline int pgd_bad(pgd_t pgd)
471{
Jeremy Fitzhardinge18a7a1992009-02-05 11:31:16 -0800472 return (pgd_flags(pgd) & ~_PAGE_USER) != _KERNPG_TABLE;
Jeremy Fitzhardinge30f10312009-02-05 11:31:09 -0800473}
Jeremy Fitzhardinge7325cc22009-02-05 11:31:14 -0800474
475static inline int pgd_none(pgd_t pgd)
476{
Jeremy Fitzhardinge26c8e312009-02-05 11:31:17 -0800477 return !native_pgd_val(pgd);
Jeremy Fitzhardinge7325cc22009-02-05 11:31:14 -0800478}
Jeremy Fitzhardinge9f38d7e2009-02-05 11:30:49 -0800479#endif /* PAGETABLE_LEVELS > 3 */
480
Thomas Gleixner96a388d2007-10-11 11:20:03 +0200481#endif /* __ASSEMBLY__ */
482
Jeremy Fitzhardingefb15a9b2008-06-25 00:19:06 -0400483/*
484 * the pgd page can be thought of an array like this: pgd_t[PTRS_PER_PGD]
485 *
486 * this macro returns the index of the entry in the pgd page which would
487 * control the given virtual address
488 */
489#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
490
491/*
492 * pgd_offset() returns a (pgd_t *)
493 * pgd_index() is used get the offset into the pgd page's array of pgd_t's;
494 */
495#define pgd_offset(mm, address) ((mm)->pgd + pgd_index((address)))
496/*
497 * a shortcut which implies the use of the kernel's pgd, instead
498 * of a process's
499 */
500#define pgd_offset_k(address) pgd_offset(&init_mm, (address))
501
502
Jeremy Fitzhardinge68db0652008-03-17 16:37:13 -0700503#define KERNEL_PGD_BOUNDARY pgd_index(PAGE_OFFSET)
504#define KERNEL_PGD_PTRS (PTRS_PER_PGD - KERNEL_PGD_BOUNDARY)
505
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100506#ifndef __ASSEMBLY__
507
Jeremy Fitzhardinge48916452008-01-30 13:32:58 +0100508/* local pte updates need not use xchg for locking */
509static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep)
510{
511 pte_t res = *ptep;
512
513 /* Pure native function needs no input for mm, addr */
514 native_pte_clear(NULL, 0, ptep);
515 return res;
516}
517
518static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr,
519 pte_t *ptep , pte_t pte)
520{
521 native_set_pte(ptep, pte);
522}
523
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100524#ifndef CONFIG_PARAVIRT
525/*
526 * Rules for using pte_update - it must be called after any PTE update which
527 * has not been done using the set_pte / clear_pte interfaces. It is used by
528 * shadow mode hypervisors to resynchronize the shadow page tables. Kernel PTE
529 * updates should either be sets, clears, or set_pte_atomic for P->P
530 * transitions, which means this hook should only be called for user PTEs.
531 * This hook implies a P->P protection or access change has taken place, which
532 * requires a subsequent TLB flush. The notification can optionally be delayed
533 * until the TLB flush event by using the pte_update_defer form of the
534 * interface, but care must be taken to assure that the flush happens while
535 * still holding the same page table lock so that the shadow and primary pages
536 * do not become out of sync on SMP.
537 */
538#define pte_update(mm, addr, ptep) do { } while (0)
539#define pte_update_defer(mm, addr, ptep) do { } while (0)
540#endif
541
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100542/*
543 * We only update the dirty/accessed state if we set
544 * the dirty bit by hand in the kernel, since the hardware
545 * will do the accessed bit for us, and we don't want to
546 * race with other CPU's that might be updating the dirty
547 * bit at the same time.
548 */
Jeremy Fitzhardingebea41802008-06-25 00:18:57 -0400549struct vm_area_struct;
550
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100551#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
Jeremy Fitzhardingeee5aa8d2008-03-17 16:37:03 -0700552extern int ptep_set_access_flags(struct vm_area_struct *vma,
553 unsigned long address, pte_t *ptep,
554 pte_t entry, int dirty);
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100555
556#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
Jeremy Fitzhardingef9fbf1a2008-03-17 16:37:04 -0700557extern int ptep_test_and_clear_young(struct vm_area_struct *vma,
558 unsigned long addr, pte_t *ptep);
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100559
560#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
Jeremy Fitzhardingec20311e2008-03-17 16:37:05 -0700561extern int ptep_clear_flush_young(struct vm_area_struct *vma,
562 unsigned long address, pte_t *ptep);
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100563
564#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700565static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
566 pte_t *ptep)
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100567{
568 pte_t pte = native_ptep_get_and_clear(ptep);
569 pte_update(mm, addr, ptep);
570 return pte;
571}
572
573#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700574static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
575 unsigned long addr, pte_t *ptep,
576 int full)
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100577{
578 pte_t pte;
579 if (full) {
580 /*
581 * Full address destruction in progress; paravirt does not
582 * care about updates and native needs no locking
583 */
584 pte = native_local_ptep_get_and_clear(ptep);
585 } else {
586 pte = ptep_get_and_clear(mm, addr, ptep);
587 }
588 return pte;
589}
590
591#define __HAVE_ARCH_PTEP_SET_WRPROTECT
Joe Perches3cbaeaf2008-03-23 01:03:12 -0700592static inline void ptep_set_wrprotect(struct mm_struct *mm,
593 unsigned long addr, pte_t *ptep)
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100594{
Jeremy Fitzhardinged8d89822008-01-30 13:32:58 +0100595 clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte);
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100596 pte_update(mm, addr, ptep);
597}
598
Jeremy Fitzhardinge85958b42008-03-17 16:37:14 -0700599/*
600 * clone_pgd_range(pgd_t *dst, pgd_t *src, int count);
601 *
602 * dst - pointer to pgd range anwhere on a pgd page
603 * src - ""
604 * count - the number of pgds to copy.
605 *
606 * dst and src can be on the same page, but the range must not overlap,
607 * and must not cross a page boundary.
608 */
609static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count)
610{
611 memcpy(dst, src, count * sizeof(pgd_t));
612}
613
614
Jeremy Fitzhardinge195466d2008-01-30 13:32:58 +0100615#include <asm-generic/pgtable.h>
616#endif /* __ASSEMBLY__ */
617
H. Peter Anvin1965aae2008-10-22 22:26:29 -0700618#endif /* _ASM_X86_PGTABLE_H */