diff options
Diffstat (limited to 'arch/powerpc/include/asm/pgalloc-64.h')
| -rw-r--r-- | arch/powerpc/include/asm/pgalloc-64.h | 158 |
1 files changed, 129 insertions, 29 deletions
diff --git a/arch/powerpc/include/asm/pgalloc-64.h b/arch/powerpc/include/asm/pgalloc-64.h index 292725cec2e..91acb12bac9 100644 --- a/arch/powerpc/include/asm/pgalloc-64.h +++ b/arch/powerpc/include/asm/pgalloc-64.h @@ -35,7 +35,10 @@ struct vmemmap_backing { #define MAX_PGTABLE_INDEX_SIZE 0xf extern struct kmem_cache *pgtable_cache[]; -#define PGT_CACHE(shift) (pgtable_cache[(shift)-1]) +#define PGT_CACHE(shift) ({ \ + BUG_ON(!(shift)); \ + pgtable_cache[(shift) - 1]; \ + }) static inline pgd_t *pgd_alloc(struct mm_struct *mm) { @@ -72,8 +75,100 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) #define pmd_populate_kernel(mm, pmd, pte) pmd_set(pmd, (unsigned long)(pte)) #define pmd_pgtable(pmd) pmd_page(pmd) +static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, + unsigned long address) +{ + return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO); +} + +static inline pgtable_t pte_alloc_one(struct mm_struct *mm, + unsigned long address) +{ + struct page *page; + pte_t *pte; + + pte = pte_alloc_one_kernel(mm, address); + if (!pte) + return NULL; + page = virt_to_page(pte); + pgtable_page_ctor(page); + return page; +} + +static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) +{ + free_page((unsigned long)pte); +} + +static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage) +{ + pgtable_page_dtor(ptepage); + __free_page(ptepage); +} + +static inline void pgtable_free(void *table, unsigned index_size) +{ + if (!index_size) + free_page((unsigned long)table); + else { + BUG_ON(index_size > MAX_PGTABLE_INDEX_SIZE); + kmem_cache_free(PGT_CACHE(index_size), table); + } +} + +#ifdef CONFIG_SMP +static inline void pgtable_free_tlb(struct mmu_gather *tlb, + void *table, int shift) +{ + unsigned long pgf = (unsigned long)table; + BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE); + pgf |= shift; + tlb_remove_table(tlb, (void *)pgf); +} + +static inline void __tlb_remove_table(void *_table) +{ + void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE); + unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE; + + pgtable_free(table, shift); +} +#else /* !CONFIG_SMP */ +static inline void pgtable_free_tlb(struct mmu_gather *tlb, + void *table, int shift) +{ + pgtable_free(table, shift); +} +#endif /* CONFIG_SMP */ + +static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table, + unsigned long address) +{ + struct page *page = page_address(table); + + tlb_flush_pgtable(tlb, address); + pgtable_page_dtor(page); + pgtable_free_tlb(tlb, page, 0); +} + +#else /* if CONFIG_PPC_64K_PAGES */ +/* + * we support 16 fragments per PTE page. + */ +#define PTE_FRAG_NR 16 +/* + * We use a 2K PTE page fragment and another 2K for storing + * real_pte_t hash index + */ +#define PTE_FRAG_SIZE_SHIFT 12 +#define PTE_FRAG_SIZE (2 * PTRS_PER_PTE * sizeof(pte_t)) -#else /* CONFIG_PPC_64K_PAGES */ +extern pte_t *page_table_alloc(struct mm_struct *, unsigned long, int); +extern void page_table_free(struct mm_struct *, unsigned long *, int); +extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift); +#ifdef CONFIG_SMP +extern void __tlb_remove_table(void *_table); +#endif #define pud_populate(mm, pud, pmd) pud_set(pud, (unsigned long)pmd) @@ -83,51 +178,56 @@ static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pmd_set(pmd, (unsigned long)pte); } -#define pmd_populate(mm, pmd, pte_page) \ - pmd_populate_kernel(mm, pmd, page_address(pte_page)) -#define pmd_pgtable(pmd) pmd_page(pmd) - -#endif /* CONFIG_PPC_64K_PAGES */ - -static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) +static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, + pgtable_t pte_page) { - return kmem_cache_alloc(PGT_CACHE(PMD_INDEX_SIZE), - GFP_KERNEL|__GFP_REPEAT); + pmd_set(pmd, (unsigned long)pte_page); } -static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) +static inline pgtable_t pmd_pgtable(pmd_t pmd) { - kmem_cache_free(PGT_CACHE(PMD_INDEX_SIZE), pmd); + return (pgtable_t)(pmd_val(pmd) & -sizeof(pte_t)*PTRS_PER_PTE); } static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { - return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO); + return (pte_t *)page_table_alloc(mm, address, 1); } static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { - struct page *page; - pte_t *pte; + return (pgtable_t)page_table_alloc(mm, address, 0); +} - pte = pte_alloc_one_kernel(mm, address); - if (!pte) - return NULL; - page = virt_to_page(pte); - pgtable_page_ctor(page); - return page; +static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) +{ + page_table_free(mm, (unsigned long *)pte, 1); } -static inline void pgtable_free(void *table, unsigned index_size) +static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage) { - if (!index_size) - free_page((unsigned long)table); - else { - BUG_ON(index_size > MAX_PGTABLE_INDEX_SIZE); - kmem_cache_free(PGT_CACHE(index_size), table); - } + page_table_free(mm, (unsigned long *)ptepage, 0); +} + +static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table, + unsigned long address) +{ + tlb_flush_pgtable(tlb, address); + pgtable_free_tlb(tlb, table, 0); +} +#endif /* CONFIG_PPC_64K_PAGES */ + +static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) +{ + return kmem_cache_alloc(PGT_CACHE(PMD_INDEX_SIZE), + GFP_KERNEL|__GFP_REPEAT); +} + +static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) +{ + kmem_cache_free(PGT_CACHE(PMD_INDEX_SIZE), pmd); } #define __pmd_free_tlb(tlb, pmd, addr) \ |