aboutsummaryrefslogtreecommitdiff
path: root/arch/mips/mm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/mips/mm')
-rw-r--r--arch/mips/mm/Makefile1
-rw-r--r--arch/mips/mm/c-octeon.c5
-rw-r--r--arch/mips/mm/c-r3k.c1
-rw-r--r--arch/mips/mm/c-r4k.c13
-rw-r--r--arch/mips/mm/c-tx39.c1
-rw-r--r--arch/mips/mm/dma-default.c23
-rw-r--r--arch/mips/mm/extable.c6
-rw-r--r--arch/mips/mm/fault.c3
-rw-r--r--arch/mips/mm/highmem.c1
-rw-r--r--arch/mips/mm/hugetlbpage.c100
-rw-r--r--arch/mips/mm/init.c1
-rw-r--r--arch/mips/mm/page.c1
-rw-r--r--arch/mips/mm/tlb-r3k.c1
-rw-r--r--arch/mips/mm/tlb-r4k.c44
-rw-r--r--arch/mips/mm/tlb-r8k.c1
-rw-r--r--arch/mips/mm/tlbex.c283
16 files changed, 421 insertions, 64 deletions
diff --git a/arch/mips/mm/Makefile b/arch/mips/mm/Makefile
index d7ec9552229..f0e43559970 100644
--- a/arch/mips/mm/Makefile
+++ b/arch/mips/mm/Makefile
@@ -8,6 +8,7 @@ obj-y += cache.o dma-default.o extable.o fault.o \
obj-$(CONFIG_32BIT) += ioremap.o pgtable-32.o
obj-$(CONFIG_64BIT) += pgtable-64.o
obj-$(CONFIG_HIGHMEM) += highmem.o
+obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
obj-$(CONFIG_CPU_LOONGSON2) += c-r4k.o cex-gen.o tlb-r4k.o
obj-$(CONFIG_CPU_MIPS32) += c-r4k.o cex-gen.o tlb-r4k.o
diff --git a/arch/mips/mm/c-octeon.c b/arch/mips/mm/c-octeon.c
index 44d01a0a849..10ab69f7183 100644
--- a/arch/mips/mm/c-octeon.c
+++ b/arch/mips/mm/c-octeon.c
@@ -8,6 +8,7 @@
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/smp.h>
#include <linux/mm.h>
#include <linux/bitops.h>
#include <linux/cpu.h>
@@ -288,7 +289,7 @@ static void cache_parity_error_octeon(int non_recoverable)
}
/**
- * Called when the the exception is not recoverable
+ * Called when the the exception is recoverable
*/
asmlinkage void cache_parity_error_octeon_recoverable(void)
@@ -297,7 +298,7 @@ asmlinkage void cache_parity_error_octeon_recoverable(void)
}
/**
- * Called when the the exception is recoverable
+ * Called when the the exception is not recoverable
*/
asmlinkage void cache_parity_error_octeon_non_recoverable(void)
diff --git a/arch/mips/mm/c-r3k.c b/arch/mips/mm/c-r3k.c
index 5500c20c79a..54e5f7b9f44 100644
--- a/arch/mips/mm/c-r3k.c
+++ b/arch/mips/mm/c-r3k.c
@@ -12,6 +12,7 @@
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/smp.h>
#include <linux/mm.h>
#include <asm/page.h>
diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c
index 171951d2305..6721ee2b1e8 100644
--- a/arch/mips/mm/c-r4k.c
+++ b/arch/mips/mm/c-r4k.c
@@ -13,6 +13,7 @@
#include <linux/kernel.h>
#include <linux/linkage.h>
#include <linux/sched.h>
+#include <linux/smp.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/bitops.h>
@@ -100,6 +101,12 @@ static inline void r4k_blast_dcache_page_dc32(unsigned long addr)
blast_dcache32_page(addr);
}
+static inline void r4k_blast_dcache_page_dc64(unsigned long addr)
+{
+ R4600_HIT_CACHEOP_WAR_IMPL;
+ blast_dcache64_page(addr);
+}
+
static void __cpuinit r4k_blast_dcache_page_setup(void)
{
unsigned long dc_lsize = cpu_dcache_line_size();
@@ -110,6 +117,8 @@ static void __cpuinit r4k_blast_dcache_page_setup(void)
r4k_blast_dcache_page = blast_dcache16_page;
else if (dc_lsize == 32)
r4k_blast_dcache_page = r4k_blast_dcache_page_dc32;
+ else if (dc_lsize == 64)
+ r4k_blast_dcache_page = r4k_blast_dcache_page_dc64;
}
static void (* r4k_blast_dcache_page_indexed)(unsigned long addr);
@@ -124,6 +133,8 @@ static void __cpuinit r4k_blast_dcache_page_indexed_setup(void)
r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed;
else if (dc_lsize == 32)
r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed;
+ else if (dc_lsize == 64)
+ r4k_blast_dcache_page_indexed = blast_dcache64_page_indexed;
}
static void (* r4k_blast_dcache)(void);
@@ -138,6 +149,8 @@ static void __cpuinit r4k_blast_dcache_setup(void)
r4k_blast_dcache = blast_dcache16;
else if (dc_lsize == 32)
r4k_blast_dcache = blast_dcache32;
+ else if (dc_lsize == 64)
+ r4k_blast_dcache = blast_dcache64;
}
/* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */
diff --git a/arch/mips/mm/c-tx39.c b/arch/mips/mm/c-tx39.c
index f7c8f9ce39c..6515b441871 100644
--- a/arch/mips/mm/c-tx39.c
+++ b/arch/mips/mm/c-tx39.c
@@ -11,6 +11,7 @@
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/smp.h>
#include <linux/mm.h>
#include <asm/cacheops.h>
diff --git a/arch/mips/mm/dma-default.c b/arch/mips/mm/dma-default.c
index 4fdb7f5216b..7e48e76148a 100644
--- a/arch/mips/mm/dma-default.c
+++ b/arch/mips/mm/dma-default.c
@@ -20,9 +20,10 @@
#include <dma-coherence.h>
-static inline unsigned long dma_addr_to_virt(dma_addr_t dma_addr)
+static inline unsigned long dma_addr_to_virt(struct device *dev,
+ dma_addr_t dma_addr)
{
- unsigned long addr = plat_dma_addr_to_phys(dma_addr);
+ unsigned long addr = plat_dma_addr_to_phys(dev, dma_addr);
return (unsigned long)phys_to_virt(addr);
}
@@ -111,7 +112,7 @@ EXPORT_SYMBOL(dma_alloc_coherent);
void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr,
dma_addr_t dma_handle)
{
- plat_unmap_dma_mem(dev, dma_handle);
+ plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
free_pages((unsigned long) vaddr, get_order(size));
}
@@ -122,7 +123,7 @@ void dma_free_coherent(struct device *dev, size_t size, void *vaddr,
{
unsigned long addr = (unsigned long) vaddr;
- plat_unmap_dma_mem(dev, dma_handle);
+ plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
if (!plat_device_is_coherent(dev))
addr = CAC_ADDR(addr);
@@ -170,10 +171,10 @@ void dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
enum dma_data_direction direction)
{
if (cpu_is_noncoherent_r10000(dev))
- __dma_sync(dma_addr_to_virt(dma_addr), size,
+ __dma_sync(dma_addr_to_virt(dev, dma_addr), size,
direction);
- plat_unmap_dma_mem(dev, dma_addr);
+ plat_unmap_dma_mem(dev, dma_addr, size, direction);
}
EXPORT_SYMBOL(dma_unmap_single);
@@ -232,7 +233,7 @@ void dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nhwentries,
if (addr)
__dma_sync(addr, sg->length, direction);
}
- plat_unmap_dma_mem(dev, sg->dma_address);
+ plat_unmap_dma_mem(dev, sg->dma_address, sg->length, direction);
}
}
@@ -246,7 +247,7 @@ void dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
if (cpu_is_noncoherent_r10000(dev)) {
unsigned long addr;
- addr = dma_addr_to_virt(dma_handle);
+ addr = dma_addr_to_virt(dev, dma_handle);
__dma_sync(addr, size, direction);
}
}
@@ -262,7 +263,7 @@ void dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle,
if (!plat_device_is_coherent(dev)) {
unsigned long addr;
- addr = dma_addr_to_virt(dma_handle);
+ addr = dma_addr_to_virt(dev, dma_handle);
__dma_sync(addr, size, direction);
}
}
@@ -277,7 +278,7 @@ void dma_sync_single_range_for_cpu(struct device *dev, dma_addr_t dma_handle,
if (cpu_is_noncoherent_r10000(dev)) {
unsigned long addr;
- addr = dma_addr_to_virt(dma_handle);
+ addr = dma_addr_to_virt(dev, dma_handle);
__dma_sync(addr + offset, size, direction);
}
}
@@ -293,7 +294,7 @@ void dma_sync_single_range_for_device(struct device *dev, dma_addr_t dma_handle,
if (!plat_device_is_coherent(dev)) {
unsigned long addr;
- addr = dma_addr_to_virt(dma_handle);
+ addr = dma_addr_to_virt(dev, dma_handle);
__dma_sync(addr + offset, size, direction);
}
}
diff --git a/arch/mips/mm/extable.c b/arch/mips/mm/extable.c
index 297fb9f390d..9d25d2ba4b9 100644
--- a/arch/mips/mm/extable.c
+++ b/arch/mips/mm/extable.c
@@ -1,5 +1,9 @@
/*
- * linux/arch/mips/mm/extable.c
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 1997, 99, 2001 - 2004 Ralf Baechle <ralf@linux-mips.org>
*/
#include <linux/module.h>
#include <linux/spinlock.h>
diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c
index 55767ad9f00..f956ecbb813 100644
--- a/arch/mips/mm/fault.c
+++ b/arch/mips/mm/fault.c
@@ -102,7 +102,7 @@ good_area:
* make sure we exit gracefully rather than endlessly redo
* the fault.
*/
- fault = handle_mm_fault(mm, vma, address, write);
+ fault = handle_mm_fault(mm, vma, address, write ? FAULT_FLAG_WRITE : 0);
if (unlikely(fault & VM_FAULT_ERROR)) {
if (fault & VM_FAULT_OOM)
goto out_of_memory;
@@ -171,6 +171,7 @@ out_of_memory:
* We ran out of memory, call the OOM killer, and return the userspace
* (which will retry the fault, or kill us if we got oom-killed).
*/
+ up_read(&mm->mmap_sem);
pagefault_out_of_memory();
return;
diff --git a/arch/mips/mm/highmem.c b/arch/mips/mm/highmem.c
index 2b1309b2580..e274fda329f 100644
--- a/arch/mips/mm/highmem.c
+++ b/arch/mips/mm/highmem.c
@@ -1,5 +1,6 @@
#include <linux/module.h>
#include <linux/highmem.h>
+#include <linux/smp.h>
#include <asm/fixmap.h>
#include <asm/tlbflush.h>
diff --git a/arch/mips/mm/hugetlbpage.c b/arch/mips/mm/hugetlbpage.c
new file mode 100644
index 00000000000..8c2834f5919
--- /dev/null
+++ b/arch/mips/mm/hugetlbpage.c
@@ -0,0 +1,100 @@
+/*
+ * MIPS Huge TLB Page Support for Kernel.
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
+ * Copyright 2005, Embedded Alley Solutions, Inc.
+ * Matt Porter <mporter@embeddedalley.com>
+ * Copyright (C) 2008, 2009 Cavium Networks, Inc.
+ */
+
+#include <linux/init.h>
+#include <linux/fs.h>
+#include <linux/mm.h>
+#include <linux/hugetlb.h>
+#include <linux/pagemap.h>
+#include <linux/slab.h>
+#include <linux/err.h>
+#include <linux/sysctl.h>
+#include <asm/mman.h>
+#include <asm/tlb.h>
+#include <asm/tlbflush.h>
+
+pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr,
+ unsigned long sz)
+{
+ pgd_t *pgd;
+ pud_t *pud;
+ pte_t *pte = NULL;
+
+ pgd = pgd_offset(mm, addr);
+ pud = pud_alloc(mm, pgd, addr);
+ if (pud)
+ pte = (pte_t *)pmd_alloc(mm, pud, addr);
+
+ return pte;
+}
+
+pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
+{
+ pgd_t *pgd;
+ pud_t *pud;
+ pmd_t *pmd = NULL;
+
+ pgd = pgd_offset(mm, addr);
+ if (pgd_present(*pgd)) {
+ pud = pud_offset(pgd, addr);
+ if (pud_present(*pud))
+ pmd = pmd_offset(pud, addr);
+ }
+ return (pte_t *) pmd;
+}
+
+int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
+{
+ return 0;
+}
+
+/*
+ * This function checks for proper alignment of input addr and len parameters.
+ */
+int is_aligned_hugepage_range(unsigned long addr, unsigned long len)
+{
+ if (len & ~HPAGE_MASK)
+ return -EINVAL;
+ if (addr & ~HPAGE_MASK)
+ return -EINVAL;
+ return 0;
+}
+
+struct page *
+follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
+{
+ return ERR_PTR(-EINVAL);
+}
+
+int pmd_huge(pmd_t pmd)
+{
+ return (pmd_val(pmd) & _PAGE_HUGE) != 0;
+}
+
+int pud_huge(pud_t pud)
+{
+ return (pud_val(pud) & _PAGE_HUGE) != 0;
+}
+
+struct page *
+follow_huge_pmd(struct mm_struct *mm, unsigned long address,
+ pmd_t *pmd, int write)
+{
+ struct page *page;
+
+ page = pte_page(*(pte_t *)pmd);
+ if (page)
+ page += ((address & ~HPAGE_MASK) >> PAGE_SHIFT);
+ return page;
+}
+
diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
index c5511294a9e..0e820508ff2 100644
--- a/arch/mips/mm/init.c
+++ b/arch/mips/mm/init.c
@@ -13,6 +13,7 @@
#include <linux/module.h>
#include <linux/signal.h>
#include <linux/sched.h>
+#include <linux/smp.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
diff --git a/arch/mips/mm/page.c b/arch/mips/mm/page.c
index 48060c635ac..f5c73754d66 100644
--- a/arch/mips/mm/page.c
+++ b/arch/mips/mm/page.c
@@ -10,6 +10,7 @@
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/smp.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/proc_fs.h>
diff --git a/arch/mips/mm/tlb-r3k.c b/arch/mips/mm/tlb-r3k.c
index 1c0048a6f5c..0f5ab236ab6 100644
--- a/arch/mips/mm/tlb-r3k.c
+++ b/arch/mips/mm/tlb-r3k.c
@@ -13,6 +13,7 @@
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/smp.h>
#include <linux/mm.h>
#include <asm/page.h>
diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c
index 892be426787..cee502caf39 100644
--- a/arch/mips/mm/tlb-r4k.c
+++ b/arch/mips/mm/tlb-r4k.c
@@ -10,7 +10,9 @@
*/
#include <linux/init.h>
#include <linux/sched.h>
+#include <linux/smp.h>
#include <linux/mm.h>
+#include <linux/hugetlb.h>
#include <asm/cpu.h>
#include <asm/bootinfo.h>
@@ -295,21 +297,41 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
pudp = pud_offset(pgdp, address);
pmdp = pmd_offset(pudp, address);
idx = read_c0_index();
- ptep = pte_offset_map(pmdp, address);
+#ifdef CONFIG_HUGETLB_PAGE
+ /* this could be a huge page */
+ if (pmd_huge(*pmdp)) {
+ unsigned long lo;
+ write_c0_pagemask(PM_HUGE_MASK);
+ ptep = (pte_t *)pmdp;
+ lo = pte_val(*ptep) >> 6;
+ write_c0_entrylo0(lo);
+ write_c0_entrylo1(lo + (HPAGE_SIZE >> 7));
+
+ mtc0_tlbw_hazard();
+ if (idx < 0)
+ tlb_write_random();
+ else
+ tlb_write_indexed();
+ write_c0_pagemask(PM_DEFAULT_MASK);
+ } else
+#endif
+ {
+ ptep = pte_offset_map(pmdp, address);
#if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32)
- write_c0_entrylo0(ptep->pte_high);
- ptep++;
- write_c0_entrylo1(ptep->pte_high);
+ write_c0_entrylo0(ptep->pte_high);
+ ptep++;
+ write_c0_entrylo1(ptep->pte_high);
#else
- write_c0_entrylo0(pte_val(*ptep++) >> 6);
- write_c0_entrylo1(pte_val(*ptep) >> 6);
+ write_c0_entrylo0(pte_val(*ptep++) >> 6);
+ write_c0_entrylo1(pte_val(*ptep) >> 6);
#endif
- mtc0_tlbw_hazard();
- if (idx < 0)
- tlb_write_random();
- else
- tlb_write_indexed();
+ mtc0_tlbw_hazard();
+ if (idx < 0)
+ tlb_write_random();
+ else
+ tlb_write_indexed();
+ }
tlbw_use_hazard();
FLUSH_ITLB_VM(vma);
EXIT_CRITICAL(flags);
diff --git a/arch/mips/mm/tlb-r8k.c b/arch/mips/mm/tlb-r8k.c
index 4ec95cc2df2..2b82f23df1a 100644
--- a/arch/mips/mm/tlb-r8k.c
+++ b/arch/mips/mm/tlb-r8k.c
@@ -10,6 +10,7 @@
*/
#include <linux/init.h>
#include <linux/sched.h>
+#include <linux/smp.h>
#include <linux/mm.h>
#include <asm/cpu.h>
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
index 0615b62efd6..9a17bf8395d 100644
--- a/arch/mips/mm/tlbex.c
+++ b/arch/mips/mm/tlbex.c
@@ -6,8 +6,9 @@
* Synthesize TLB refill handlers at runtime.
*
* Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer
- * Copyright (C) 2005, 2007 Maciej W. Rozycki
+ * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki
* Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
+ * Copyright (C) 2008, 2009 Cavium Networks, Inc.
*
* ... and the days got worse and worse and now you see
* I've gone completly out of my mind.
@@ -19,8 +20,10 @@
* (Condolences to Napoleon XIV)
*/
+#include <linux/bug.h>
#include <linux/kernel.h>
#include <linux/types.h>
+#include <linux/smp.h>
#include <linux/string.h>
#include <linux/init.h>
@@ -82,6 +85,9 @@ enum label_id {
label_nopage_tlbm,
label_smp_pgtable_change,
label_r3000_write_probe_fail,
+#ifdef CONFIG_HUGETLB_PAGE
+ label_tlb_huge_update,
+#endif
};
UASM_L_LA(_second_part)
@@ -98,6 +104,9 @@ UASM_L_LA(_nopage_tlbs)
UASM_L_LA(_nopage_tlbm)
UASM_L_LA(_smp_pgtable_change)
UASM_L_LA(_r3000_write_probe_fail)
+#ifdef CONFIG_HUGETLB_PAGE
+UASM_L_LA(_tlb_huge_update)
+#endif
/*
* For debug purposes.
@@ -125,6 +134,7 @@ static inline void dump_handler(const u32 *handler, int count)
#define C0_TCBIND 2, 2
#define C0_ENTRYLO1 3, 0
#define C0_CONTEXT 4, 0
+#define C0_PAGEMASK 5, 0
#define C0_BADVADDR 8, 0
#define C0_ENTRYHI 10, 0
#define C0_EPC 14, 0
@@ -258,7 +268,8 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
}
if (cpu_has_mips_r2) {
- uasm_i_ehb(p);
+ if (cpu_has_mips_r2_exec_hazard)
+ uasm_i_ehb(p);
tlbw(p);
return;
}
@@ -310,7 +321,6 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
case CPU_BCM3302:
case CPU_BCM4710:
case CPU_LOONGSON2:
- case CPU_CAVIUM_OCTEON:
case CPU_R5500:
if (m4kc_tlbp_war())
uasm_i_nop(p);
@@ -382,6 +392,98 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
}
}
+#ifdef CONFIG_HUGETLB_PAGE
+static __cpuinit void build_huge_tlb_write_entry(u32 **p,
+ struct uasm_label **l,
+ struct uasm_reloc **r,
+ unsigned int tmp,
+ enum tlb_write_entry wmode)
+{
+ /* Set huge page tlb entry size */
+ uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16);
+ uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff);
+ uasm_i_mtc0(p, tmp, C0_PAGEMASK);
+
+ build_tlb_write_entry(p, l, r, wmode);
+
+ /* Reset default page size */
+ if (PM_DEFAULT_MASK >> 16) {
+ uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
+ uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff);
+ uasm_il_b(p, r, label_leave);
+ uasm_i_mtc0(p, tmp, C0_PAGEMASK);
+ } else if (PM_DEFAULT_MASK) {
+ uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK);
+ uasm_il_b(p, r, label_leave);
+ uasm_i_mtc0(p, tmp, C0_PAGEMASK);
+ } else {
+ uasm_il_b(p, r, label_leave);
+ uasm_i_mtc0(p, 0, C0_PAGEMASK);
+ }
+}
+
+/*
+ * Check if Huge PTE is present, if so then jump to LABEL.
+ */
+static void __cpuinit
+build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp,
+ unsigned int pmd, int lid)
+{
+ UASM_i_LW(p, tmp, 0, pmd);
+ uasm_i_andi(p, tmp, tmp, _PAGE_HUGE);
+ uasm_il_bnez(p, r, tmp, lid);
+}
+
+static __cpuinit void build_huge_update_entries(u32 **p,
+ unsigned int pte,
+ unsigned int tmp)
+{
+ int small_sequence;
+
+ /*
+ * A huge PTE describes an area the size of the
+ * configured huge page size. This is twice the
+ * of the large TLB entry size we intend to use.
+ * A TLB entry half the size of the configured
+ * huge page size is configured into entrylo0
+ * and entrylo1 to cover the contiguous huge PTE
+ * address space.
+ */
+ small_sequence = (HPAGE_SIZE >> 7) < 0x10000;
+
+ /* We can clobber tmp. It isn't used after this.*/
+ if (!small_sequence)
+ uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16));
+
+ UASM_i_SRL(p, pte, pte, 6); /* convert to entrylo */
+ uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* load it */
+ /* convert to entrylo1 */
+ if (small_sequence)
+ UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7);
+ else
+ UASM_i_ADDU(p, pte, pte, tmp);
+
+ uasm_i_mtc0(p, pte, C0_ENTRYLO1); /* load it */
+}
+
+static __cpuinit void build_huge_handler_tail(u32 **p,
+ struct uasm_reloc **r,
+ struct uasm_label **l,
+ unsigned int pte,
+ unsigned int ptr)
+{
+#ifdef CONFIG_SMP
+ UASM_i_SC(p, pte, 0, ptr);
+ uasm_il_beqz(p, r, pte, label_tlb_huge_update);
+ UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */
+#else
+ UASM_i_SW(p, pte, 0, ptr);
+#endif
+ build_huge_update_entries(p, pte, ptr);
+ build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed);
+}
+#endif /* CONFIG_HUGETLB_PAGE */
+
#ifdef CONFIG_64BIT
/*
* TMP and PTR are scratch.
@@ -649,6 +751,14 @@ static void __cpuinit build_update_entries(u32 **p, unsigned int tmp,
#endif
}
+/*
+ * For a 64-bit kernel, we are using the 64-bit XTLB refill exception
+ * because EXL == 0. If we wrap, we can also use the 32 instruction
+ * slots before the XTLB refill exception handler which belong to the
+ * unused TLB refill exception.
+ */
+#define MIPS64_REFILL_INSNS 32
+
static void __cpuinit build_r4000_tlb_refill_handler(void)
{
u32 *p = tlb_handler;
@@ -680,12 +790,23 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
#endif
+#ifdef CONFIG_HUGETLB_PAGE
+ build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update);
+#endif
+
build_get_ptep(&p, K0, K1);
build_update_entries(&p, K0, K1);
build_tlb_write_entry(&p, &l, &r, tlb_random);
uasm_l_leave(&l, p);
uasm_i_eret(&p); /* return from trap */
+#ifdef CONFIG_HUGETLB_PAGE
+ uasm_l_tlb_huge_update(&l, p);
+ UASM_i_LW(&p, K0, 0, K1);
+ build_huge_update_entries(&p, K0, K1);
+ build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random);
+#endif
+
#ifdef CONFIG_64BIT
build_get_pgd_vmalloc64(&p, &l, &r, K0, K1);
#endif
@@ -702,9 +823,10 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
if ((p - tlb_handler) > 64)
panic("TLB refill handler space exceeded");
#else
- if (((p - tlb_handler) > 63)
- || (((p - tlb_handler) > 61)
- && uasm_insn_has_bdelay(relocs, tlb_handler + 29)))
+ if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1)
+ || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3)
+ && uasm_insn_has_bdelay(relocs,
+ tlb_handler + MIPS64_REFILL_INSNS - 3)))
panic("TLB refill handler space exceeded");
#endif
@@ -717,39 +839,74 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
uasm_copy_handler(relocs, labels, tlb_handler, p, f);
final_len = p - tlb_handler;
#else /* CONFIG_64BIT */
- f = final_handler + 32;
- if ((p - tlb_handler) <= 32) {
+ f = final_handler + MIPS64_REFILL_INSNS;
+ if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) {
/* Just copy the handler. */
uasm_copy_handler(relocs, labels, tlb_handler, p, f);
final_len = p - tlb_handler;
} else {
- u32 *split = tlb_handler + 30;
+#if defined(CONFIG_HUGETLB_PAGE)
+ const enum label_id ls = label_tlb_huge_update;
+#elif defined(MODULE_START)
+ const enum label_id ls = label_module_alloc;
+#else
+ const enum label_id ls = label_vmalloc;
+#endif
+ u32 *split;
+ int ov = 0;
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++)
+ ;
+ BUG_ON(i == ARRAY_SIZE(labels));
+ split = labels[i].addr;
/*
- * Find the split point.
+ * See if we have overflown one way or the other.
*/
- if (uasm_insn_has_bdelay(relocs, split - 1))
- split--;
-
+ if (split > tlb_handler + MIPS64_REFILL_INSNS ||
+ split < p - MIPS64_REFILL_INSNS)
+ ov = 1;
+
+ if (ov) {
+ /*
+ * Split two instructions before the end. One
+ * for the branch and one for the instruction
+ * in the delay slot.
+ */
+ split = tlb_handler + MIPS64_REFILL_INSNS - 2;
+
+ /*
+ * If the branch would fall in a delay slot,
+ * we must back up an additional instruction
+ * so that it is no longer in a delay slot.
+ */
+ if (uasm_insn_has_bdelay(relocs, split - 1))
+ split--;
+ }
/* Copy first part of the handler. */
uasm_copy_handler(relocs, labels, tlb_handler, split, f);
f += split - tlb_handler;
- /* Insert branch. */
- uasm_l_split(&l, final_handler);
- uasm_il_b(&f, &r, label_split);
- if (uasm_insn_has_bdelay(relocs, split))
- uasm_i_nop(&f);
- else {
- uasm_copy_handler(relocs, labels, split, split + 1, f);
- uasm_move_labels(labels, f, f + 1, -1);
- f++;
- split++;
+ if (ov) {
+ /* Insert branch. */
+ uasm_l_split(&l, final_handler);
+ uasm_il_b(&f, &r, label_split);
+ if (uasm_insn_has_bdelay(relocs, split))
+ uasm_i_nop(&f);
+ else {
+ uasm_copy_handler(relocs, labels,
+ split, split + 1, f);
+ uasm_move_labels(labels, f, f + 1, -1);
+ f++;
+ split++;
+ }
}
/* Copy the rest of the handler. */
uasm_copy_handler(relocs, labels, split, p, final_handler);
- final_len = (f - (final_handler + 32)) + (p - split);
+ final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) +
+ (p - split);
}
#endif /* CONFIG_64BIT */
@@ -782,7 +939,7 @@ u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned;
u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned;
static void __cpuinit
-iPTE_LW(u32 **p, struct uasm_label **l, unsigned int pte, unsigned int ptr)
+iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr)
{
#ifdef CONFIG_SMP
# ifdef CONFIG_64BIT_PHYS_ADDR
@@ -862,13 +1019,13 @@ iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr,
* with it's original value.
*/
static void __cpuinit
-build_pte_present(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
+build_pte_present(u32 **p, struct uasm_reloc **r,
unsigned int pte, unsigned int ptr, enum label_id lid)
{
uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
uasm_il_bnez(p, r, pte, lid);
- iPTE_LW(p, l, pte, ptr);
+ iPTE_LW(p, pte, ptr);
}
/* Make PTE valid, store result in PTR. */
@@ -886,13 +1043,13 @@ build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte,
* restore PTE with value from PTR when done.
*/
static void __cpuinit
-build_pte_writable(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
+build_pte_writable(u32 **p, struct uasm_reloc **r,
unsigned int pte, unsigned int ptr, enum label_id lid)
{
uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
uasm_il_bnez(p, r, pte, lid);
- iPTE_LW(p, l, pte, ptr);
+ iPTE_LW(p, pte, ptr);
}
/* Make PTE writable, update software status bits as well, then store
@@ -913,12 +1070,12 @@ build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte,
* restore PTE with value from PTR when done.
*/
static void __cpuinit
-build_pte_modifiable(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
+build_pte_modifiable(u32 **p, struct uasm_reloc **r,
unsigned int pte, unsigned int ptr, enum label_id lid)
{
uasm_i_andi(p, pte, pte, _PAGE_WRITE);
uasm_il_beqz(p, r, pte, lid);
- iPTE_LW(p, l, pte, ptr);
+ iPTE_LW(p, pte, ptr);
}
/*
@@ -994,7 +1151,7 @@ static void __cpuinit build_r3000_tlb_load_handler(void)
memset(relocs, 0, sizeof(relocs));
build_r3000_tlbchange_handler_head(&p, K0, K1);
- build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl);
+ build_pte_present(&p, &r, K0, K1, label_nopage_tlbl);
uasm_i_nop(&p); /* load delay */
build_make_valid(&p, &r, K0, K1);
build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
@@ -1024,7 +1181,7 @@ static void __cpuinit build_r3000_tlb_store_handler(void)
memset(relocs, 0, sizeof(relocs));
build_r3000_tlbchange_handler_head(&p, K0, K1);
- build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs);
+ build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs);
uasm_i_nop(&p); /* load delay */
build_make_write(&p, &r, K0, K1);
build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
@@ -1054,7 +1211,7 @@ static void __cpuinit build_r3000_tlb_modify_handler(void)
memset(relocs, 0, sizeof(relocs));
build_r3000_tlbchange_handler_head(&p, K0, K1);
- build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm);
+ build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm);
uasm_i_nop(&p); /* load delay */
build_make_write(&p, &r, K0, K1);
build_r3000_pte_reload_tlbwi(&p, K0, K1);
@@ -1087,6 +1244,15 @@ build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l,
build_get_pgde32(p, pte, ptr); /* get pgd in ptr */
#endif
+#ifdef CONFIG_HUGETLB_PAGE
+ /*
+ * For huge tlb entries, pmd doesn't contain an address but
+ * instead contains the tlb pte. Check the PAGE_HUGE bit and
+ * see if we need to jump to huge tlb processing.
+ */
+ build_is_huge_pte(p, r, pte, ptr, label_tlb_huge_update);
+#endif
+
UASM_i_MFC0(p, pte, C0_BADVADDR);
UASM_i_LW(p, ptr, 0, ptr);
UASM_i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2);
@@ -1096,7 +1262,7 @@ build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l,
#ifdef CONFIG_SMP
uasm_l_smp_pgtable_change(l, *p);
#endif
- iPTE_LW(p, l, pte, ptr); /* get even pte */
+ iPTE_LW(p, pte, ptr); /* get even pte */
if (!m4kc_tlbp_war())
build_tlb_probe_entry(p);
}
@@ -1138,12 +1304,25 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
}
build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
- build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl);
+ build_pte_present(&p, &r, K0, K1, label_nopage_tlbl);
if (m4kc_tlbp_war())
build_tlb_probe_entry(&p);
build_make_valid(&p, &r, K0, K1);
build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
+#ifdef CONFIG_HUGETLB_PAGE
+ /*
+ * This is the entry point when build_r4000_tlbchange_handler_head
+ * spots a huge page.
+ */
+ uasm_l_tlb_huge_update(&l, p);
+ iPTE_LW(&p, K0, K1);
+ build_pte_present(&p, &r, K0, K1, label_nopage_tlbl);
+ build_tlb_probe_entry(&p);
+ uasm_i_ori(&p, K0, K0, (_PAGE_ACCESSED | _PAGE_VALID));
+ build_huge_handler_tail(&p, &r, &l, K0, K1);
+#endif
+
uasm_l_nopage_tlbl(&l, p);
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
uasm_i_nop(&p);
@@ -1169,12 +1348,26 @@ static void __cpuinit build_r4000_tlb_store_handler(void)
memset(relocs, 0, sizeof(relocs));
build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
- build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs);
+ build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs);
if (m4kc_tlbp_war())
build_tlb_probe_entry(&p);
build_make_write(&p, &r, K0, K1);
build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
+#ifdef CONFIG_HUGETLB_PAGE
+ /*
+ * This is the entry point when
+ * build_r4000_tlbchange_handler_head spots a huge page.
+ */
+ uasm_l_tlb_huge_update(&l, p);
+ iPTE_LW(&p, K0, K1);
+ build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs);
+ build_tlb_probe_entry(&p);
+ uasm_i_ori(&p, K0, K0,
+ _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
+ build_huge_handler_tail(&p, &r, &l, K0, K1);
+#endif
+
uasm_l_nopage_tlbs(&l, p);
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
uasm_i_nop(&p);
@@ -1200,13 +1393,27 @@ static void __cpuinit build_r4000_tlb_modify_handler(void)
memset(relocs, 0, sizeof(relocs));
build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
- build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm);
+ build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm);
if (m4kc_tlbp_war())
build_tlb_probe_entry(&p);
/* Present and writable bits set, set accessed and dirty bits. */
build_make_write(&p, &r, K0, K1);
build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
+#ifdef CONFIG_HUGETLB_PAGE
+ /*
+ * This is the entry point when
+ * build_r4000_tlbchange_handler_head spots a huge page.
+ */
+ uasm_l_tlb_huge_update(&l, p);
+ iPTE_LW(&p, K0, K1);
+ build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm);
+ build_tlb_probe_entry(&p);
+ uasm_i_ori(&p, K0, K0,
+ _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
+ build_huge_handler_tail(&p, &r, &l, K0, K1);
+#endif
+
uasm_l_nopage_tlbm(&l, p);
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
uasm_i_nop(&p);