From ab7ccd91004b55c29910c999f51bc40880646810 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Mon, 2 Jun 2014 11:47:23 +0100 Subject: arm64: ptrace: change fs when passing kernel pointer to regset code Our compat PTRACE_POKEUSR implementation simply passes the user data to regset_copy_from_user after some simple range checking. Unfortunately, the data in question has already been copied to the kernel stack by this point, so the subsequent access_ok check fails and the ptrace request returns -EFAULT. This causes problems tracing fork() with older versions of strace. This patch briefly changes the fs to KERNEL_DS, so that the access_ok check passes even with a kernel address. Signed-off-by: Will Deacon Cc: Signed-off-by: Catalin Marinas (cherry picked from commit c168870704bcde6bb63d05f7882b620dd3985a46) Signed-off-by: Mark Brown --- arch/arm64/kernel/ptrace.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 5341534b6d04..85536688f753 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -872,6 +872,7 @@ static int compat_ptrace_write_user(struct task_struct *tsk, compat_ulong_t off, compat_ulong_t val) { int ret; + mm_segment_t old_fs = get_fs(); if (off & 3 || off >= COMPAT_USER_SZ) return -EIO; @@ -879,10 +880,13 @@ static int compat_ptrace_write_user(struct task_struct *tsk, compat_ulong_t off, if (off >= sizeof(compat_elf_gregset_t)) return 0; + set_fs(KERNEL_DS); ret = copy_regset_from_user(tsk, &user_aarch32_view, REGSET_COMPAT_GPR, off, sizeof(compat_ulong_t), &val); + set_fs(old_fs); + return ret; } -- cgit v1.2.3 From dd7d4626b70d246242c57c4a9ffaecb961570406 Mon Sep 17 00:00:00 2001 From: Victor Kamensky Date: Tue, 3 Jun 2014 19:21:30 +0100 Subject: arm64: ptrace: fix empty registers set in prstatus of aarch32 process core Currently core file of aarch32 process prstatus note has empty registers set. As result aarch32 core files create by V8 kernel are not very useful. It happens because compat_gpr_get and compat_gpr_set functions can copy registers values to/from either kbuf or ubuf. ELF core file collection function fill_thread_core_info calls compat_gpr_get with kbuf set and ubuf set to 0. But current compat_gpr_get and compat_gpr_set function handle copy to/from only ubuf case. Fix is to handle kbuf and ubuf as two separate cases in similar way as other functions like user_regset_copyout, user_regset_copyin do. Signed-off-by: Victor Kamensky Acked-by: Will Deacon Cc: stable@vger.kernel.org Signed-off-by: Catalin Marinas (cherry picked from commit 2227901a0230d8fde81ba9c602d649839390f56b) Signed-off-by: Mark Brown Conflicts: arch/arm64/kernel/ptrace.c --- arch/arm64/kernel/ptrace.c | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 85536688f753..9212dc4bcd09 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -672,12 +672,16 @@ static int compat_gpr_get(struct task_struct *target, reg = (void *)&task_pt_regs(target)->regs[idx]; } - ret = copy_to_user(ubuf, reg, sizeof(compat_ulong_t)); + if (kbuf) { + memcpy(kbuf, ®, sizeof(reg)); + kbuf += sizeof(reg); + } else { + ret = copy_to_user(ubuf, ®, sizeof(reg)); + if (ret) + break; - if (ret) - break; - else - ubuf += sizeof(compat_ulong_t); + ubuf += sizeof(reg); + } } return ret; @@ -705,7 +709,18 @@ static int compat_gpr_set(struct task_struct *target, for (i = 0; i < num_regs; ++i) { unsigned int idx = start + i; - void *reg; + compat_ulong_t reg; + + if (kbuf) { + memcpy(®, kbuf, sizeof(reg)); + kbuf += sizeof(reg); + } else { + ret = copy_from_user(®, ubuf, sizeof(reg)); + if (ret) + return ret; + + ubuf += sizeof(reg); + } switch (idx) { case 15: -- cgit v1.2.3 From 91eba7f91f33524f3b9a68d9a752be1dbb49bc90 Mon Sep 17 00:00:00 2001 From: Jungseok Lee Date: Mon, 12 May 2014 10:40:30 +0100 Subject: arm64: Use pr_* instead of printk This patch fixed the following checkpatch complaint as using pr_* instead of printk. WARNING: printk() should include KERN_ facility level Signed-off-by: Jungseok Lee Reviewed-by: Sungjinn Chung Acked-by: Kukjin Kim Signed-off-by: Catalin Marinas (cherry picked from commit ac7b406c1a9d50ddbf5e5cbce8ca4d68d36ac2db) Signed-off-by: Mark Brown --- arch/arm64/kernel/traps.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 7ffadddb645d..268ce966afd6 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -156,7 +156,7 @@ static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) frame.pc = thread_saved_pc(tsk); } - printk("Call trace:\n"); + pr_emerg("Call trace:\n"); while (1) { unsigned long where = frame.pc; int ret; @@ -328,17 +328,17 @@ asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr) void __pte_error(const char *file, int line, unsigned long val) { - printk("%s:%d: bad pte %016lx.\n", file, line, val); + pr_crit("%s:%d: bad pte %016lx.\n", file, line, val); } void __pmd_error(const char *file, int line, unsigned long val) { - printk("%s:%d: bad pmd %016lx.\n", file, line, val); + pr_crit("%s:%d: bad pmd %016lx.\n", file, line, val); } void __pgd_error(const char *file, int line, unsigned long val) { - printk("%s:%d: bad pgd %016lx.\n", file, line, val); + pr_crit("%s:%d: bad pgd %016lx.\n", file, line, val); } void __init trap_init(void) -- cgit v1.2.3 From 95c91bdd8eafdc74337049c45d74c903b7dac49c Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 9 Jul 2014 19:22:12 +0100 Subject: arm64: vdso: move to _install_special_mapping and remove arch_vma_name _install_special_mapping replaces install_special_mapping and removes the need to detect special VMA in arch_vma_name. This patch moves the vdso and compat vectors page code over to the new API. Cc: Andy Lutomirski Signed-off-by: Will Deacon Signed-off-by: Catalin Marinas (cherry picked from commit 2fea7f6c98f5957e539eb8aa0ce849729b900342) Signed-off-by: Mark Brown --- arch/arm64/kernel/vdso.c | 80 +++++++++++++++++++++--------------------------- 1 file changed, 35 insertions(+), 45 deletions(-) diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c index 40ba5dea2ed7..ad8dddbffef2 100644 --- a/arch/arm64/kernel/vdso.c +++ b/arch/arm64/kernel/vdso.c @@ -85,22 +85,29 @@ int aarch32_setup_vectors_page(struct linux_binprm *bprm, int uses_interp) { struct mm_struct *mm = current->mm; unsigned long addr = AARCH32_VECTORS_BASE; - int ret; + static struct vm_special_mapping spec = { + .name = "[vectors]", + .pages = vectors_page, + + }; + void *ret; down_write(&mm->mmap_sem); current->mm->context.vdso = (void *)addr; /* Map vectors page at the high address. */ - ret = install_special_mapping(mm, addr, PAGE_SIZE, - VM_READ|VM_EXEC|VM_MAYREAD|VM_MAYEXEC, - vectors_page); + ret = _install_special_mapping(mm, addr, PAGE_SIZE, + VM_READ|VM_EXEC|VM_MAYREAD|VM_MAYEXEC, + &spec); up_write(&mm->mmap_sem); - return ret; + return PTR_ERR_OR_ZERO(ret); } #endif /* CONFIG_COMPAT */ +static struct vm_special_mapping vdso_spec[2]; + static int __init vdso_init(void) { int i; @@ -127,6 +134,17 @@ static int __init vdso_init(void) /* Grab the vDSO data page. */ vdso_pagelist[i] = virt_to_page(vdso_data); + /* Populate the special mapping structures */ + vdso_spec[0] = (struct vm_special_mapping) { + .name = "[vdso]", + .pages = vdso_pagelist, + }; + + vdso_spec[1] = (struct vm_special_mapping) { + .name = "[vvar]", + .pages = vdso_pagelist + vdso_pages, + }; + return 0; } arch_initcall(vdso_init); @@ -136,7 +154,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, { struct mm_struct *mm = current->mm; unsigned long vdso_base, vdso_text_len, vdso_mapping_len; - int ret; + void *ret; vdso_text_len = vdso_pages << PAGE_SHIFT; /* Be sure to map the data page */ @@ -145,23 +163,23 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, down_write(&mm->mmap_sem); vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0); if (IS_ERR_VALUE(vdso_base)) { - ret = vdso_base; + ret = ERR_PTR(vdso_base); goto up_fail; } mm->context.vdso = (void *)vdso_base; - ret = install_special_mapping(mm, vdso_base, vdso_text_len, - VM_READ|VM_EXEC| - VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, - vdso_pagelist); - if (ret) + ret = _install_special_mapping(mm, vdso_base, vdso_text_len, + VM_READ|VM_EXEC| + VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, + &vdso_spec[0]); + if (IS_ERR(ret)) goto up_fail; vdso_base += vdso_text_len; - ret = install_special_mapping(mm, vdso_base, PAGE_SIZE, - VM_READ|VM_MAYREAD, - vdso_pagelist + vdso_pages); - if (ret) + ret = _install_special_mapping(mm, vdso_base, PAGE_SIZE, + VM_READ|VM_MAYREAD, + &vdso_spec[1]); + if (IS_ERR(ret)) goto up_fail; up_write(&mm->mmap_sem); @@ -170,35 +188,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, up_fail: mm->context.vdso = NULL; up_write(&mm->mmap_sem); - return ret; -} - -const char *arch_vma_name(struct vm_area_struct *vma) -{ - unsigned long vdso_text; - - if (!vma->vm_mm) - return NULL; - - vdso_text = (unsigned long)vma->vm_mm->context.vdso; - - /* - * We can re-use the vdso pointer in mm_context_t for identifying - * the vectors page for compat applications. The vDSO will always - * sit above TASK_UNMAPPED_BASE and so we don't need to worry about - * it conflicting with the vectors base. - */ - if (vma->vm_start == vdso_text) { -#ifdef CONFIG_COMPAT - if (vma->vm_start == AARCH32_VECTORS_BASE) - return "[vectors]"; -#endif - return "[vdso]"; - } else if (vma->vm_start == (vdso_text + (vdso_pages << PAGE_SHIFT))) { - return "[vvar]"; - } - - return NULL; + return PTR_ERR(ret); } /* -- cgit v1.2.3 From 82a95d0521cb6258559d18dca736da8272ba05a7 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 9 Jul 2014 19:22:13 +0100 Subject: arm64: vdso: move data page before code pages Andy pointed out that binutils generates additional sections in the vdso image (e.g. section string table) which, if our .text section gets big enough, could cross a page boundary and end up screwing up the location where the kernel expects to put the data page. This patch solves the issue in the same manner as x86_32, by moving the data page before the code pages. Cc: Andy Lutomirski Signed-off-by: Will Deacon Signed-off-by: Catalin Marinas (cherry picked from commit 601255ae3c98fdeeee3a8bb4696425e4f868b4f1) Signed-off-by: Mark Brown --- arch/arm64/kernel/vdso.c | 34 +++++++++++++++++----------------- arch/arm64/kernel/vdso/vdso.lds.S | 4 +--- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c index ad8dddbffef2..b8289f08795f 100644 --- a/arch/arm64/kernel/vdso.c +++ b/arch/arm64/kernel/vdso.c @@ -118,8 +118,8 @@ static int __init vdso_init(void) } vdso_pages = (&vdso_end - &vdso_start) >> PAGE_SHIFT; - pr_info("vdso: %ld pages (%ld code, %ld data) at base %p\n", - vdso_pages + 1, vdso_pages, 1L, &vdso_start); + pr_info("vdso: %ld pages (%ld code @ %p, %ld data @ %p)\n", + vdso_pages + 1, vdso_pages, &vdso_start, 1L, vdso_data); /* Allocate the vDSO pagelist, plus a page for the data. */ vdso_pagelist = kcalloc(vdso_pages + 1, sizeof(struct page *), @@ -127,22 +127,22 @@ static int __init vdso_init(void) if (vdso_pagelist == NULL) return -ENOMEM; + /* Grab the vDSO data page. */ + vdso_pagelist[0] = virt_to_page(vdso_data); + /* Grab the vDSO code pages. */ for (i = 0; i < vdso_pages; i++) - vdso_pagelist[i] = virt_to_page(&vdso_start + i * PAGE_SIZE); - - /* Grab the vDSO data page. */ - vdso_pagelist[i] = virt_to_page(vdso_data); + vdso_pagelist[i + 1] = virt_to_page(&vdso_start + i * PAGE_SIZE); /* Populate the special mapping structures */ vdso_spec[0] = (struct vm_special_mapping) { - .name = "[vdso]", + .name = "[vvar]", .pages = vdso_pagelist, }; vdso_spec[1] = (struct vm_special_mapping) { - .name = "[vvar]", - .pages = vdso_pagelist + vdso_pages, + .name = "[vdso]", + .pages = &vdso_pagelist[1], }; return 0; @@ -166,22 +166,22 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, ret = ERR_PTR(vdso_base); goto up_fail; } - mm->context.vdso = (void *)vdso_base; - - ret = _install_special_mapping(mm, vdso_base, vdso_text_len, - VM_READ|VM_EXEC| - VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, + ret = _install_special_mapping(mm, vdso_base, PAGE_SIZE, + VM_READ|VM_MAYREAD, &vdso_spec[0]); if (IS_ERR(ret)) goto up_fail; - vdso_base += vdso_text_len; - ret = _install_special_mapping(mm, vdso_base, PAGE_SIZE, - VM_READ|VM_MAYREAD, + vdso_base += PAGE_SIZE; + mm->context.vdso = (void *)vdso_base; + ret = _install_special_mapping(mm, vdso_base, vdso_text_len, + VM_READ|VM_EXEC| + VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, &vdso_spec[1]); if (IS_ERR(ret)) goto up_fail; + up_write(&mm->mmap_sem); return 0; diff --git a/arch/arm64/kernel/vdso/vdso.lds.S b/arch/arm64/kernel/vdso/vdso.lds.S index 8154b8d1c826..beca249bc2f3 100644 --- a/arch/arm64/kernel/vdso/vdso.lds.S +++ b/arch/arm64/kernel/vdso/vdso.lds.S @@ -28,6 +28,7 @@ OUTPUT_ARCH(aarch64) SECTIONS { + PROVIDE(_vdso_data = . - PAGE_SIZE); . = VDSO_LBASE + SIZEOF_HEADERS; .hash : { *(.hash) } :text @@ -57,9 +58,6 @@ SECTIONS _end = .; PROVIDE(end = .); - . = ALIGN(PAGE_SIZE); - PROVIDE(_vdso_data = .); - /DISCARD/ : { *(.note.GNU-stack) *(.data .data.* .gnu.linkonce.d.* .sdata*) -- cgit v1.2.3 From c106ad82ccd063b34a89fbc5c4a44c9dfd53c930 Mon Sep 17 00:00:00 2001 From: Lorenzo Pieralisi Date: Thu, 17 Jul 2014 18:19:18 +0100 Subject: arm64: kernel: enable PSCI cpu operations on UP systems PSCI CPU operations have to be enabled on UP kernels so that calls like eg cpu_suspend can be made functional on UP too. This patch reworks the PSCI CPU operations so that they can be enabled on UP systems. Acked-by: Mark Rutland Signed-off-by: Lorenzo Pieralisi Signed-off-by: Catalin Marinas (cherry picked from commit 756854d9b99a735f86bc3b86df5c19be12e8746e) Signed-off-by: Mark Brown --- arch/arm64/kernel/cpu_ops.c | 2 +- arch/arm64/kernel/psci.c | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kernel/cpu_ops.c b/arch/arm64/kernel/cpu_ops.c index 04efea8fe4bc..24fb449bc6aa 100644 --- a/arch/arm64/kernel/cpu_ops.c +++ b/arch/arm64/kernel/cpu_ops.c @@ -30,8 +30,8 @@ const struct cpu_operations *cpu_ops[NR_CPUS]; static const struct cpu_operations *supported_cpu_ops[] __initconst = { #ifdef CONFIG_SMP &smp_spin_table_ops, - &cpu_psci_ops, #endif + &cpu_psci_ops, NULL, }; diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c index 9e9798f91172..a623c44a6bc4 100644 --- a/arch/arm64/kernel/psci.c +++ b/arch/arm64/kernel/psci.c @@ -434,9 +434,11 @@ static int cpu_psci_cpu_kill(unsigned int cpu) return 0; } #endif +#endif const struct cpu_operations cpu_psci_ops = { .name = "psci", +#ifdef CONFIG_SMP .cpu_init = cpu_psci_cpu_init, .cpu_prepare = cpu_psci_cpu_prepare, .cpu_boot = cpu_psci_cpu_boot, @@ -445,6 +447,6 @@ const struct cpu_operations cpu_psci_ops = { .cpu_die = cpu_psci_cpu_die, .cpu_kill = cpu_psci_cpu_kill, #endif +#endif }; -#endif -- cgit v1.2.3 From dfbf350f1b30e0d9bf68bce97191603bfd2ce714 Mon Sep 17 00:00:00 2001 From: Lorenzo Pieralisi Date: Thu, 17 Jul 2014 18:19:19 +0100 Subject: arm64: kernel: add __init marker to PSCI init functions PSCI init functions must be marked as __init so that they are freed by the kernel upon boot. This patch marks the PSCI init functions as such since they need not be persistent in the kernel address space after the kernel has booted. Signed-off-by: Lorenzo Pieralisi Signed-off-by: Catalin Marinas (cherry picked from commit b9e97ef93c630404f305350d88d09391d1a55648) Signed-off-by: Mark Brown --- arch/arm64/kernel/psci.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c index a623c44a6bc4..553954771a67 100644 --- a/arch/arm64/kernel/psci.c +++ b/arch/arm64/kernel/psci.c @@ -235,7 +235,7 @@ static void psci_sys_poweroff(void) * PSCI Function IDs for v0.2+ are well defined so use * standard values. */ -static int psci_0_2_init(struct device_node *np) +static int __init psci_0_2_init(struct device_node *np) { int err, ver; @@ -296,7 +296,7 @@ out_put_node: /* * PSCI < v0.2 get PSCI Function IDs via DT. */ -static int psci_0_1_init(struct device_node *np) +static int __init psci_0_1_init(struct device_node *np) { u32 id; int err; -- cgit v1.2.3 From 29cdbe3b974687b51f1016489455e65751a7fc40 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 16 Jul 2014 16:32:43 +0100 Subject: arm64: add MIDR_EL1 field accessors The MIDR_EL1 register is composed of a number of bitfields, and uses of the fields has so far involved open-coding of the shifts and masks required. This patch adds shifts and masks for each of the MIDR_EL1 subfields, and also provides accessors built atop of these. Existing uses within cputype.h are updated to use these accessors. The read_cpuid_part_number macro is modified to return the extracted bitfield rather than returning the value in-place with all other fields (including revision) masked out, to better match the other accessors. As the value is only used in comparison with the *_CPU_PART_* macros which are similarly updated, and these values are never exposed to userspace, this change should not affect any functionality. Signed-off-by: Mark Rutland Acked-by: Will Deacon Reviewed-by: Will Deacon Reviewed-by: Catalin Marinas Signed-off-by: Catalin Marinas (cherry picked from commit 89c4a306e7631bcb71cc537c8a029172af6047fe) Signed-off-by: Mark Brown --- arch/arm64/include/asm/cputype.h | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 343f7f737970..632298c2d951 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -36,15 +36,34 @@ __val; \ }) +#define MIDR_REVISION_MASK 0xf +#define MIDR_REVISION(midr) ((midr) & MIDR_REVISION_MASK) +#define MIDR_PARTNUM_SHIFT 4 +#define MIDR_PARTNUM_MASK (0xfff << MIDR_PARTNUM_SHIFT) +#define MIDR_PARTNUM(midr) \ + (((midr) & MIDR_PARTNUM_MASK) >> MIDR_PARTNUM_SHIFT) +#define MIDR_ARCHITECTURE_SHIFT 16 +#define MIDR_ARCHITECTURE_MASK (0xf << MIDR_ARCHITECTURE_SHIFT) +#define MIDR_ARCHITECTURE(midr) \ + (((midr) & MIDR_ARCHITECTURE_MASK) >> MIDR_ARCHITECTURE_SHIFT) +#define MIDR_VARIANT_SHIFT 20 +#define MIDR_VARIANT_MASK (0xf << MIDR_VARIANT_SHIFT) +#define MIDR_VARIANT(midr) \ + (((midr) & MIDR_VARIANT_MASK) >> MIDR_VARIANT_SHIFT) +#define MIDR_IMPLEMENTOR_SHIFT 24 +#define MIDR_IMPLEMENTOR_MASK (0xff << MIDR_IMPLEMENTOR_SHIFT) +#define MIDR_IMPLEMENTOR(midr) \ + (((midr) & MIDR_IMPLEMENTOR_MASK) >> MIDR_IMPLEMENTOR_SHIFT) + #define ARM_CPU_IMP_ARM 0x41 #define ARM_CPU_IMP_APM 0x50 -#define ARM_CPU_PART_AEM_V8 0xD0F0 -#define ARM_CPU_PART_FOUNDATION 0xD000 -#define ARM_CPU_PART_CORTEX_A53 0xD030 -#define ARM_CPU_PART_CORTEX_A57 0xD070 +#define ARM_CPU_PART_AEM_V8 0xD0F +#define ARM_CPU_PART_FOUNDATION 0xD00 +#define ARM_CPU_PART_CORTEX_A57 0xD07 +#define ARM_CPU_PART_CORTEX_A53 0xD03 -#define APM_CPU_PART_POTENZA 0x0000 +#define APM_CPU_PART_POTENZA 0x000 #ifndef __ASSEMBLY__ @@ -65,12 +84,12 @@ static inline u64 __attribute_const__ read_cpuid_mpidr(void) static inline unsigned int __attribute_const__ read_cpuid_implementor(void) { - return (read_cpuid_id() & 0xFF000000) >> 24; + return MIDR_IMPLEMENTOR(read_cpuid_id()); } static inline unsigned int __attribute_const__ read_cpuid_part_number(void) { - return (read_cpuid_id() & 0xFFF0); + return MIDR_PARTNUM(read_cpuid_id()); } static inline u32 __attribute_const__ read_cpuid_cachetype(void) -- cgit v1.2.3 From 34c668387658fabfb18c67abdc7f194009954d7e Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Fri, 18 Jul 2014 11:54:37 +0100 Subject: arm64: Create non-empty ZONE_DMA when DRAM starts above 4GB ZONE_DMA is created to allow 32-bit only devices to access memory in the absence of an IOMMU. On systems where the memory starts above 4GB, it is expected that some devices have a DMA offset hardwired to be able to access the bottom of the memory. Linux currently supports DT bindings for the DMA offsets but they are not (easily) available early during boot. This patch tries to guess a DMA offset and assumes that ZONE_DMA corresponds to the 32-bit mask above the start of DRAM. Fixes: 2d5a5612bc (arm64: Limit the CMA buffer to 32-bit if ZONE_DMA) Signed-off-by: Catalin Marinas Reported-by: Mark Salter Tested-by: Mark Salter Tested-by: Anup Patel (cherry picked from commit d50314a6b0702c630c35b88148c1acb76d2e4ede) Signed-off-by: Mark Brown Conflicts: arch/arm64/mm/init.c --- arch/arm64/mm/init.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 81bdd29df3a4..6757d02d6976 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -68,6 +68,17 @@ static int __init early_initrd(char *p) } early_param("initrd", early_initrd); +/* + * Return the maximum physical address for ZONE_DMA (DMA_BIT_MASK(32)). It + * currently assumes that for memory starting above 4G, 32-bit devices will + * use a DMA offset. + */ +static phys_addr_t max_zone_dma_phys(void) +{ + phys_addr_t offset = memblock_start_of_DRAM() & GENMASK_ULL(63, 32); + return min(offset + (1ULL << 32), memblock_end_of_DRAM()); +} + static void __init zone_sizes_init(unsigned long min, unsigned long max) { struct memblock_region *reg; @@ -78,9 +89,7 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max) /* 4GB maximum for 32-bit only capable devices */ if (IS_ENABLED(CONFIG_ZONE_DMA)) { - unsigned long max_dma_phys = - (unsigned long)dma_to_phys(NULL, DMA_BIT_MASK(32) + 1); - max_dma = max(min, min(max, max_dma_phys >> PAGE_SHIFT)); + max_dma = PFN_DOWN(max_zone_dma_phys()); zone_size[ZONE_DMA] = max_dma - min; } zone_size[ZONE_NORMAL] = max - max_dma; @@ -171,7 +180,11 @@ void __init arm64_memblock_init(void) } early_init_fdt_scan_reserved_mem(); - dma_contiguous_reserve(0); + + /* 4GB maximum for 32-bit only capable devices */ + if (IS_ENABLED(CONFIG_ZONE_DMA)) + dma_phys_limit = max_zone_dma_phys(); + dma_contiguous_reserve(dma_phys_limit); memblock_allow_resize(); memblock_dump_all(); -- cgit v1.2.3 From a8ba6a1bc823c07982767189e03a40e203dd98c9 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Wed, 16 Jul 2014 12:07:17 +0100 Subject: arm64: Do not initialise the fixmap page tables in head.S The early_ioremap_init() function already handles fixmap pte initialisation, so upgrade this to cover all of pud/pmd/pte and remove one page from swapper_pg_dir. Signed-off-by: Catalin Marinas Tested-by: Jungseok Lee (cherry picked from commit 7edd88ad7e59c2b7b49da0e00f251884fb785d4f) Signed-off-by: Mark Brown --- arch/arm64/include/asm/page.h | 8 ++++---- arch/arm64/kernel/head.S | 7 ------- arch/arm64/mm/ioremap.c | 26 ++++++++++++++++++-------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h index a6331e6a92b5..e84ca637af6b 100644 --- a/arch/arm64/include/asm/page.h +++ b/arch/arm64/include/asm/page.h @@ -33,11 +33,11 @@ /* * The idmap and swapper page tables need some space reserved in the kernel - * image. The idmap only requires a pgd and a next level table to (section) map - * the kernel, while the swapper also maps the FDT and requires an additional - * table to map an early UART. See __create_page_tables for more information. + * image. Both require a pgd and a next level table to (section) map the + * kernel. The the swapper also maps the FDT (see __create_page_tables for + * more information). */ -#define SWAPPER_DIR_SIZE (3 * PAGE_SIZE) +#define SWAPPER_DIR_SIZE (2 * PAGE_SIZE) #define IDMAP_DIR_SIZE (2 * PAGE_SIZE) #ifndef __ASSEMBLY__ diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index f49bb0f5a7f8..7218fefc6ac9 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -455,13 +455,6 @@ __create_page_tables: sub x6, x6, #1 // inclusive range create_block_map x0, x7, x3, x5, x6 1: - /* - * Create the pgd entry for the fixed mappings. - */ - ldr x5, =FIXADDR_TOP // Fixed mapping virtual address - add x0, x26, #2 * PAGE_SIZE // section table address - create_pgd_entry x26, x0, x5, x6, x7 - /* * Since the page tables have been populated with non-cacheable * accesses (MMU disabled), invalidate the idmap and swapper page diff --git a/arch/arm64/mm/ioremap.c b/arch/arm64/mm/ioremap.c index 00d315ae1de9..cf25fa9aa68f 100644 --- a/arch/arm64/mm/ioremap.c +++ b/arch/arm64/mm/ioremap.c @@ -98,19 +98,25 @@ void __iomem *ioremap_cache(phys_addr_t phys_addr, size_t size) } EXPORT_SYMBOL(ioremap_cache); -#ifndef CONFIG_ARM64_64K_PAGES static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss; +#ifndef CONFIG_ARM64_64K_PAGES +static pte_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss; #endif -static inline pmd_t * __init early_ioremap_pmd(unsigned long addr) +static inline pud_t * __init early_ioremap_pud(unsigned long addr) { pgd_t *pgd; - pud_t *pud; pgd = pgd_offset_k(addr); BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd)); - pud = pud_offset(pgd, addr); + return pud_offset(pgd, addr); +} + +static inline pmd_t * __init early_ioremap_pmd(unsigned long addr) +{ + pud_t *pud = early_ioremap_pud(addr); + BUG_ON(pud_none(*pud) || pud_bad(*pud)); return pmd_offset(pud, addr); @@ -127,13 +133,17 @@ static inline pte_t * __init early_ioremap_pte(unsigned long addr) void __init early_ioremap_init(void) { + pgd_t *pgd; + pud_t *pud; pmd_t *pmd; + unsigned long addr = fix_to_virt(FIX_BTMAP_BEGIN); - pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)); -#ifndef CONFIG_ARM64_64K_PAGES - /* need to populate pmd for 4k pagesize only */ + pgd = pgd_offset_k(addr); + pud = pud_offset(pgd, addr); + pud_populate(&init_mm, pud, bm_pmd); + pmd = pmd_offset(pud, addr); pmd_populate_kernel(&init_mm, pmd, bm_pte); -#endif + /* * The boot-ioremap range spans multiple pmds, for which * we are not prepared: -- cgit v1.2.3 From fb8acafd9091e9864ecc3f62fb4281db5cd52a7c Mon Sep 17 00:00:00 2001 From: Mark Salter Date: Thu, 24 Jul 2014 15:56:15 +0100 Subject: arm64: fix soft lockup due to large tlb flush range Under certain loads, this soft lockup has been observed: BUG: soft lockup - CPU#2 stuck for 22s! [ip6tables:1016] Modules linked in: ip6t_rpfilter ip6t_REJECT cfg80211 rfkill xt_conntrack ebtable_nat ebtable_broute bridge stp llc ebtable_filter ebtables ip6table_nat nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6 ip6table_mangle ip6table_security ip6table_raw ip6table_filter ip6_tables iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle iptable_security iptable_raw vfat fat efivarfs xfs libcrc32c CPU: 2 PID: 1016 Comm: ip6tables Not tainted 3.13.0-0.rc7.30.sa2.aarch64 #1 task: fffffe03e81d1400 ti: fffffe03f01f8000 task.ti: fffffe03f01f8000 PC is at __cpu_flush_kern_tlb_range+0xc/0x40 LR is at __purge_vmap_area_lazy+0x28c/0x3ac pc : [] lr : [] pstate: 80000145 sp : fffffe03f01fbb70 x29: fffffe03f01fbb70 x28: fffffe03f01f8000 x27: fffffe0000b19000 x26: 00000000000000d0 x25: 000000000000001c x24: fffffe03f01fbc50 x23: fffffe03f01fbc58 x22: fffffe03f01fbc10 x21: fffffe0000b2a3f8 x20: 0000000000000802 x19: fffffe0000b2a3c8 x18: 000003fffdf52710 x17: 000003ff9d8bb910 x16: fffffe000050fbfc x15: 0000000000005735 x14: 000003ff9d7e1a5c x13: 0000000000000000 x12: 000003ff9d7e1a5c x11: 0000000000000007 x10: fffffe0000c09af0 x9 : fffffe0000ad1000 x8 : 000000000000005c x7 : fffffe03e8624000 x6 : 0000000000000000 x5 : 0000000000000000 x4 : 0000000000000000 x3 : fffffe0000c09cc8 x2 : 0000000000000000 x1 : 000fffffdfffca80 x0 : 000fffffcd742150 The __cpu_flush_kern_tlb_range() function looks like: ENTRY(__cpu_flush_kern_tlb_range) dsb sy lsr x0, x0, #12 lsr x1, x1, #12 1: tlbi vaae1is, x0 add x0, x0, #1 cmp x0, x1 b.lo 1b dsb sy isb ret ENDPROC(__cpu_flush_kern_tlb_range) The above soft lockup shows the PC at tlbi insn with: x0 = 0x000fffffcd742150 x1 = 0x000fffffdfffca80 So __cpu_flush_kern_tlb_range has 0x128ba930 tlbi flushes left after it has already been looping for 23 seconds!. Looking up one frame at __purge_vmap_area_lazy(), there is: ... list_for_each_entry_rcu(va, &vmap_area_list, list) { if (va->flags & VM_LAZY_FREE) { if (va->va_start < *start) *start = va->va_start; if (va->va_end > *end) *end = va->va_end; nr += (va->va_end - va->va_start) >> PAGE_SHIFT; list_add_tail(&va->purge_list, &valist); va->flags |= VM_LAZY_FREEING; va->flags &= ~VM_LAZY_FREE; } } ... if (nr || force_flush) flush_tlb_kernel_range(*start, *end); So if two areas are being freed, the range passed to flush_tlb_kernel_range() may be as large as the vmalloc space. For arm64, this is ~240GB for 4k pagesize and ~2TB for 64kpage size. This patch works around this problem by adding a loop limit. If the range is larger than the limit, use flush_tlb_all() rather than flushing based on individual pages. The limit chosen is arbitrary as the TLB size is implementation specific and not accessible in an architected way. The aim of the arbitrary limit is to avoid soft lockup. Signed-off-by: Mark Salter [catalin.marinas@arm.com: commit log update] [catalin.marinas@arm.com: marginal optimisation] [catalin.marinas@arm.com: changed to MAX_TLB_RANGE and added comment] Signed-off-by: Catalin Marinas (cherry picked from commit 05ac65305437e8ef63d2d19cac704138a2a05aa5) Signed-off-by: Mark Brown --- arch/arm64/include/asm/tlbflush.h | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index 3796ea6bb734..73f0ce570fb3 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -98,8 +98,8 @@ static inline void flush_tlb_page(struct vm_area_struct *vma, dsb(ish); } -static inline void flush_tlb_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end) +static inline void __flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) { unsigned long asid = (unsigned long)ASID(vma->vm_mm) << 48; unsigned long addr; @@ -112,7 +112,7 @@ static inline void flush_tlb_range(struct vm_area_struct *vma, dsb(ish); } -static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end) +static inline void __flush_tlb_kernel_range(unsigned long start, unsigned long end) { unsigned long addr; start >>= 12; @@ -125,6 +125,29 @@ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end isb(); } +/* + * This is meant to avoid soft lock-ups on large TLB flushing ranges and not + * necessarily a performance improvement. + */ +#define MAX_TLB_RANGE (1024UL << PAGE_SHIFT) + +static inline void flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ + if ((end - start) <= MAX_TLB_RANGE) + __flush_tlb_range(vma, start, end); + else + flush_tlb_mm(vma->vm_mm); +} + +static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + if ((end - start) <= MAX_TLB_RANGE) + __flush_tlb_kernel_range(start, end); + else + flush_tlb_all(); +} + /* * On AArch64, the cache coherency is handled via the set_pte_at() function. */ -- cgit v1.2.3 From 6eb8fed568d2d39d198391012c62c0bd5f36b8ac Mon Sep 17 00:00:00 2001 From: Arun Chandran Date: Thu, 26 Jun 2014 15:16:03 +0530 Subject: arm64: vdso: fix build error when switching from LE to BE Building a kernel with CPU_BIG_ENDIAN fails if there are stale objects from a !CPU_BIG_ENDIAN build. Due to a missing FORCE prerequisite on an if_changed rule in the VDSO Makefile, we attempt to link a stale LE object into the new BE kernel. According to Documentation/kbuild/makefiles.txt, FORCE is required for if_changed rules and forgetting it is a common mistake, so fix it by 'Forcing' the build of vdso. This patch fixes build errors like these: arch/arm64/kernel/vdso/note.o: compiled for a little endian system and target is big endian failed to merge target specific data of file arch/arm64/kernel/vdso/note.o arch/arm64/kernel/vdso/sigreturn.o: compiled for a little endian system and target is big endian failed to merge target specific data of file arch/arm64/kernel/vdso/sigreturn.o Tested-by: Mark Rutland Signed-off-by: Arun Chandran Signed-off-by: Will Deacon (cherry picked from commit 1915e2ad1cf548217c963121e4076b3d44dd0169) Signed-off-by: Mark Brown --- arch/arm64/kernel/vdso/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kernel/vdso/Makefile b/arch/arm64/kernel/vdso/Makefile index 84b942612051..ff3bddea482d 100644 --- a/arch/arm64/kernel/vdso/Makefile +++ b/arch/arm64/kernel/vdso/Makefile @@ -43,7 +43,7 @@ $(obj)/vdso-offsets.h: $(obj)/vdso.so.dbg FORCE $(call if_changed,vdsosym) # Assembly rules for the .S files -$(obj-vdso): %.o: %.S +$(obj-vdso): %.o: %.S FORCE $(call if_changed_dep,vdsoas) # Actual build commands -- cgit v1.2.3 From 83f1bff42dfdbf3f4bc70a07f4304db54f89d27d Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 31 Jul 2014 11:36:08 +0100 Subject: arm64: don't call break hooks for BRK exceptions from EL0 Our break hooks are used to handle brk exceptions from kgdb (and potentially kprobes if that code ever resurfaces), so don't bother calling them if the BRK exception comes from userspace. This prevents userspace from trapping to a kdb shell on systems where kgdb is enabled and active. Cc: Reported-by: Omar Sandoval Signed-off-by: Will Deacon (cherry picked from commit c878e0cff5c5e56b216951cbe75f7a3dd500a736) Signed-off-by: Mark Brown Conflicts: arch/arm64/kernel/debug-monitors.c --- arch/arm64/kernel/debug-monitors.c | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c index fea84694fce4..e3b37ee9076a 100644 --- a/arch/arm64/kernel/debug-monitors.c +++ b/arch/arm64/kernel/debug-monitors.c @@ -303,23 +303,20 @@ static int brk_handler(unsigned long addr, unsigned int esr, { siginfo_t info; - if (call_break_hook(regs, esr) == DBG_HOOK_HANDLED) - return 0; - - pr_warn("unexpected brk exception at %lx, esr=0x%x\n", - (long)instruction_pointer(regs), esr); + if (user_mode(regs)) { + info = (siginfo_t) { + .si_signo = SIGTRAP, + .si_errno = 0, + .si_code = TRAP_BRKPT, + .si_addr = (void __user *)instruction_pointer(regs), + }; - if (!user_mode(regs)) + force_sig_info(SIGTRAP, &info, current); + } else if (call_break_hook(regs, esr) != DBG_HOOK_HANDLED) { + pr_warning("Unexpected kernel BRK exception at EL1\n"); return -EFAULT; + } - info = (siginfo_t) { - .si_signo = SIGTRAP, - .si_errno = 0, - .si_code = TRAP_BRKPT, - .si_addr = (void __user *)instruction_pointer(regs), - }; - - force_sig_info(SIGTRAP, &info, current); return 0; } -- cgit v1.2.3 From 432d182423d53bc7f03d3a399cd57482c5ca47b0 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Fri, 13 Jun 2014 13:41:20 +0100 Subject: arm64: Limit the CMA buffer to 32-bit if ZONE_DMA When the CMA buffer is allocated, it is too early to know whether devices will require ZONE_DMA memory. This patch limits the CMA buffer to (DMA_BIT_MASK(32) + 1) if CONFIG_ZONE_DMA is enabled. In addition, it computes the dma_to_phys(DMA_BIT_MASK(32)) before the increment (no current functional change). Signed-off-by: Catalin Marinas (cherry picked from commit 2d5a5612bceda8edd25b29f363c4e2c6cda28bab) Signed-off-by: Mark Brown Conflicts: arch/arm64/mm/init.c --- arch/arm64/mm/init.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 6757d02d6976..d0f86b74fce6 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -144,6 +144,7 @@ static void arm64_memory_present(void) void __init arm64_memblock_init(void) { u64 *reserve_map, base, size; + phys_addr_t dma_phys_limit = 0; /* * Register the kernel text, kernel data, initrd, and initial -- cgit v1.2.3 From c86df325d594f33eff078fe6b6014f1994f9bcb7 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 13 Aug 2014 18:53:03 +0100 Subject: arm64: align randomized TEXT_OFFSET on 4 kB boundary When booting via UEFI, the kernel Image is loaded at a 4 kB boundary and the embedded EFI stub is executed in place. The EFI stub relocates the Image to reside TEXT_OFFSET bytes above a 2 MB boundary, and jumps into the kernel proper. In AArch64, PC relative symbol references are emitted using adrp/add or adrp/ldr pairs, where the offset into a 4 kB page is resolved using a separate :lo12: relocation. This implicitly assumes that the code will always be executed at the same relative offset with respect to a 4 kB boundary, or the references will point to the wrong address. This means we should link the kernel at a 4 kB aligned base address in order to remain compatible with the base address the UEFI loader uses when doing the initial load of Image. So update the code that generates TEXT_OFFSET to choose a multiple of 4 kB. At the same time, update the code so it chooses from the interval [0..2MB) as the author originally intended. Reviewed-by: Mark Rutland Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon (cherry picked from commit 4190312beb2acfb7bfb1bb971e24a759aa96b0e8) Signed-off-by: Mark Brown Conflicts: arch/arm64/Makefile arch/arm64/kernel/head.S --- arch/arm64/Makefile | 4 ++++ arch/arm64/kernel/head.S | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 7ab6b358cc35..a5004a5b7aa4 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -32,7 +32,11 @@ CHECKFLAGS += -D__aarch64__ head-y := arch/arm64/kernel/head.o # The byte offset of the kernel image in RAM from the start of RAM. +ifeq ($(CONFIG_ARM64_RANDOMIZE_TEXT_OFFSET), y) +TEXT_OFFSET := $(shell awk 'BEGIN {srand(); printf "0x%03x000\n", int(512 * rand())}') +else TEXT_OFFSET := 0x00080000 +endif export TEXT_OFFSET GZFLAGS diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 7218fefc6ac9..9129a917203e 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -37,8 +37,12 @@ #define KERNEL_RAM_VADDR (PAGE_OFFSET + TEXT_OFFSET) -#if (KERNEL_RAM_VADDR & 0xfffff) != 0x80000 -#error KERNEL_RAM_VADDR must start at 0xXXX80000 +#if (TEXT_OFFSET & 0xfff) != 0 +#error TEXT_OFFSET must be at least 4KB aligned +#elif (PAGE_OFFSET & 0x1fffff) != 0 +#error PAGE_OFFSET must be at least 2MB aligned +#elif TEXT_OFFSET > 0x1fffff +#error TEXT_OFFSET must be less than 2MB #endif .macro pgtbl, ttb0, ttb1, virt_to_phys -- cgit v1.2.3 From 7454a3fe586c6ae1272682e6188549916a47562f Mon Sep 17 00:00:00 2001 From: Ganapatrao Kulkarni Date: Thu, 14 Aug 2014 20:49:46 +0530 Subject: arm64: mm: update max pa bits to 48 Now that we support 48-bit physical addressing, update MAX_PHYSMEM_BITS accordingly. Acked-by: Catalin Marinas Signed-off-by: Ganapatrao Kulkarni Signed-off-by: Will Deacon (cherry picked from commit 07a15dd55a3d65f81b4b09eab293f4afc720b082) Signed-off-by: Mark Brown --- arch/arm64/include/asm/sparsemem.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/sparsemem.h b/arch/arm64/include/asm/sparsemem.h index 1be62bcb9d47..74a9d301819f 100644 --- a/arch/arm64/include/asm/sparsemem.h +++ b/arch/arm64/include/asm/sparsemem.h @@ -17,7 +17,7 @@ #define __ASM_SPARSEMEM_H #ifdef CONFIG_SPARSEMEM -#define MAX_PHYSMEM_BITS 40 +#define MAX_PHYSMEM_BITS 48 #define SECTION_SIZE_BITS 30 #endif -- cgit v1.2.3 From 660ba08887a7f4a0a1ac14f6d1693568fa2623fa Mon Sep 17 00:00:00 2001 From: Geoff Levand Date: Fri, 22 Aug 2014 20:49:16 +0100 Subject: arm64: Remove unused variable in head.S Remove an unused local variable from head.S. It seems this was never used even from the initial commit 9703d9d7f77ce129621f7d80a844822e2daa7008 (arm64: Kernel booting and initialisation), and is a left over from a previous implementation of __calc_phys_offset. Signed-off-by: Geoff Levand Signed-off-by: Will Deacon (cherry picked from commit 5843be2279d7a91ef48c20ac31715d1eb9607a84) Signed-off-by: Mark Brown --- arch/arm64/kernel/head.S | 4 ---- 1 file changed, 4 deletions(-) diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 9129a917203e..b7f76fff32a6 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -231,10 +231,6 @@ ENTRY(__boot_cpu_mode) .long 0 .popsection - .align 3 -2: .quad . - .quad PAGE_OFFSET - #ifdef CONFIG_SMP .align 3 1: .quad . -- cgit v1.2.3 From 2fee2fdbfb9e95f592c6fbfab263d8dc86d5772e Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 22 Aug 2014 14:13:24 +0100 Subject: arm64: ptrace: fix compat hardware watchpoint reporting I'm not sure what I was on when I wrote this, but when iterating over the hardware watchpoint array (hbp_watch_array), our index is off by ARM_MAX_BRP, so we walk off the end of our thread_struct... ... except, a dodgy condition in the loop means that it never executes at all (bp cannot be NULL). This patch fixes the code so that we remove the bp check and use the correct index for accessing the watchpoint structures. Cc: Signed-off-by: Will Deacon (cherry picked from commit 27d7ff273c2aad37b28f6ff0cab2cfa35b51e648) Signed-off-by: Mark Brown --- arch/arm64/include/asm/hw_breakpoint.h | 1 - arch/arm64/kernel/ptrace.c | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/hw_breakpoint.h b/arch/arm64/include/asm/hw_breakpoint.h index d064047612b1..52b484b6aa1a 100644 --- a/arch/arm64/include/asm/hw_breakpoint.h +++ b/arch/arm64/include/asm/hw_breakpoint.h @@ -79,7 +79,6 @@ static inline void decode_ctrl_reg(u32 reg, */ #define ARM_MAX_BRP 16 #define ARM_MAX_WRP 16 -#define ARM_MAX_HBP_SLOTS (ARM_MAX_BRP + ARM_MAX_WRP) /* Virtual debug register bases. */ #define AARCH64_DBG_REG_BVR 0 diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 71c6a623e226..f9177993d135 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -81,7 +81,8 @@ static void ptrace_hbptriggered(struct perf_event *bp, break; } } - for (i = ARM_MAX_BRP; i < ARM_MAX_HBP_SLOTS && !bp; ++i) { + + for (i = 0; i < ARM_MAX_WRP; ++i) { if (current->thread.debug.hbp_watch[i] == bp) { info.si_errno = -((i << 1) + 1); break; -- cgit v1.2.3 From adf5da773504328b2520ecf63f2098eaef982349 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 22 Aug 2014 14:20:24 +0100 Subject: arm64: ptrace: fix compat reg getter/setter return values copy_{to,from}_user return the number of bytes remaining on failure, not an error code. This patch returns -EFAULT when the copy operation didn't complete, rather than expose the number of bytes not copied directly to userspace. Signed-off-by: Will Deacon (cherry picked from commit 85487edd252fa04718dcd735bc0f41213bbb9546) Signed-off-by: Mark Brown --- arch/arm64/kernel/ptrace.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index f9177993d135..2b453624d5ad 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -656,8 +656,10 @@ static int compat_gpr_get(struct task_struct *target, kbuf += sizeof(reg); } else { ret = copy_to_user(ubuf, ®, sizeof(reg)); - if (ret) + if (ret) { + ret = -EFAULT; break; + } ubuf += sizeof(reg); } @@ -695,8 +697,10 @@ static int compat_gpr_set(struct task_struct *target, kbuf += sizeof(reg); } else { ret = copy_from_user(®, ubuf, sizeof(reg)); - if (ret) - return ret; + if (ret) { + ret = -EFAULT; + break; + } ubuf += sizeof(reg); } -- cgit v1.2.3 From b9725baa1ec2a8827096d3941a38c887cd07ffd9 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 29 Aug 2014 16:11:10 +0100 Subject: arm64: report correct stack pointer in KSTK_ESP for compat tasks The KSTK_ESP macro is used to determine the user stack pointer for a given task. In particular, this is used to to report the '[stack]' VMA in /proc/self/maps, which is used by Android to determine the stack location for children of the main thread. This patch fixes the macro to use user_stack_pointer instead of directly returning sp. This means that we report w13 instead of sp, since the former is used as the stack pointer when executing in AArch32 state. Cc: Reported-by: Serban Constantinescu Signed-off-by: Will Deacon (cherry picked from commit 3168a743461ecf86adf3e7dcfcd79271828fb263) Signed-off-by: Mark Brown --- arch/arm64/include/asm/processor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index db3112886968..3b7bb031f98f 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -132,7 +132,7 @@ extern struct task_struct *cpu_switch_to(struct task_struct *prev, ((struct pt_regs *)(THREAD_START_SP + task_stack_page(p)) - 1) #define KSTK_EIP(tsk) ((unsigned long)task_pt_regs(tsk)->pc) -#define KSTK_ESP(tsk) ((unsigned long)task_pt_regs(tsk)->sp) +#define KSTK_ESP(tsk) user_stack_pointer(task_pt_regs(tsk)) /* * Prefetching support -- cgit v1.2.3 From ecf06ef82a0be34db53a469c22fd2c95b0b80039 Mon Sep 17 00:00:00 2001 From: Sudeep Holla Date: Tue, 2 Sep 2014 11:35:24 +0100 Subject: arm64: use irq_set_affinity with force=false when migrating irqs The arm64 interrupt migration code on cpu offline calls irqchip.irq_set_affinity() with the argument force=true. Originally this argument had no effect because it was not used by any interrupt chip driver and there was no semantics defined. This changed with commit 01f8fa4f01d8 ("genirq: Allow forcing cpu affinity of interrupts") which made the force argument useful to route interrupts to not yet online cpus without checking the target cpu against the cpu online mask. The following commit ffde1de64012 ("irqchip: gic: Support forced affinity setting") implemented this for the GIC interrupt controller. As a consequence the cpu offline irq migration fails if CPU0 is offlined, because CPU0 is still set in the affinity mask and the validation against cpu online mask is skipped to the force argument being true. The following first_cpu(mask) selection always selects CPU0 as the target. Commit 601c942176d8("arm64: use cpu_online_mask when using forced irq_set_affinity") intended to fix the above mentioned issue but introduced another issue where affinity can be migrated to a wrong CPU due to unconditional copy of cpu_online_mask. As with for arm, solve the issue by calling irq_set_affinity() with force=false from the CPU offline irq migration code so the GIC driver validates the affinity mask against CPU online mask and therefore removes CPU0 from the possible target candidates. Also revert the changes done in the commit 601c942176d8 as it's no longer needed. Tested on Juno platform. Fixes: 601c942176d8("arm64: use cpu_online_mask when using forced irq_set_affinity") Signed-off-by: Sudeep Holla Acked-by: Mark Rutland Cc: Catalin Marinas Cc: Will Deacon Cc: # 3.10.x Signed-off-by: Will Deacon (cherry picked from commit 3d8afe3099ebc602848aa7f09235cce3a9a023ce) Signed-off-by: Mark Brown --- arch/arm64/kernel/irq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c index 473e5dbf8f39..dfa6e3e74fdd 100644 --- a/arch/arm64/kernel/irq.c +++ b/arch/arm64/kernel/irq.c @@ -105,7 +105,7 @@ static bool migrate_one_irq(struct irq_desc *desc) c = irq_data_get_irq_chip(d); if (!c->irq_set_affinity) pr_debug("IRQ%u: unable to set affinity\n", d->irq); - else if (c->irq_set_affinity(d, affinity, true) == IRQ_SET_MASK_OK && ret) + else if (c->irq_set_affinity(d, affinity, false) == IRQ_SET_MASK_OK && ret) cpumask_copy(d->affinity, affinity); return ret; -- cgit v1.2.3 From 6d11fa22acd8d5ddcdb80e2e520f3543e0fd6077 Mon Sep 17 00:00:00 2001 From: Arun Chandran Date: Mon, 18 Aug 2014 10:06:58 +0100 Subject: arm64: convert part of soft_restart() to assembly The current soft_restart() and setup_restart implementations incorrectly assume that compiler will not spill/fill values to/from stack. However this assumption seems to be wrong, revealed by the disassembly of the currently existing code (v3.16) built with Linaro GCC 4.9-2014.05. ffffffc000085224 : ffffffc000085224: a9be7bfd stp x29, x30, [sp,#-32]! ffffffc000085228: 910003fd mov x29, sp ffffffc00008522c: f9000fa0 str x0, [x29,#24] ffffffc000085230: 94003d21 bl ffffffc0000946b4 ffffffc000085234: 94003b33 bl ffffffc000093f00 ffffffc000085238: 94003dfa bl ffffffc000094a20 ffffffc00008523c: 94003b31 bl ffffffc000093f00 ffffffc000085240: b0003321 adrp x1, ffffffc0006ea000 ffffffc000085244: f9400fa0 ldr x0, [x29,#24] ----> spilled addr ffffffc000085248: f942fc22 ldr x2, [x1,#1528] ----> global memstart_addr ffffffc00008524c: f0000061 adrp x1, ffffffc000094000 <__inval_cache_range+0x40> ffffffc000085250: 91290021 add x1, x1, #0xa40 ffffffc000085254: 8b010041 add x1, x2, x1 ffffffc000085258: d2c00802 mov x2, #0x4000000000 // #274877906944 ffffffc00008525c: 8b020021 add x1, x1, x2 ffffffc000085260: d63f0020 blr x1 ... Here the compiler generates memory accesses after the cache is disabled, loading stale values for the spilled value and global variable. As we cannot control when the compiler will access memory we must rewrite the functions in assembly to stash values we need in registers prior to disabling the cache, avoiding the use of memory. Reviewed-by: Mark Rutland Signed-off-by: Arun Chandran Signed-off-by: Will Deacon (cherry picked from commit 5e051531447259e5df95c44bccb69979537c19e4) Signed-off-by: Mark Brown Conflicts: arch/arm64/include/asm/proc-fns.h --- arch/arm64/include/asm/proc-fns.h | 2 ++ arch/arm64/kernel/process.c | 30 ++---------------------------- arch/arm64/mm/proc.S | 15 +++++++++++++++ 3 files changed, 19 insertions(+), 28 deletions(-) diff --git a/arch/arm64/include/asm/proc-fns.h b/arch/arm64/include/asm/proc-fns.h index 7cdf466fd0c5..a496aa6ff7ef 100644 --- a/arch/arm64/include/asm/proc-fns.h +++ b/arch/arm64/include/asm/proc-fns.h @@ -31,6 +31,8 @@ extern void cpu_cache_off(void); extern void cpu_do_idle(void); extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm); extern void cpu_reset(unsigned long addr) __attribute__((noreturn)); +void cpu_soft_restart(phys_addr_t cpu_reset, + unsigned long addr) __attribute__((noreturn)); #include diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index feb01cf4b255..f38f224b04bb 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -56,36 +56,10 @@ unsigned long __stack_chk_guard __read_mostly; EXPORT_SYMBOL(__stack_chk_guard); #endif -static void setup_restart(void) -{ - /* - * Tell the mm system that we are going to reboot - - * we may need it to insert some 1:1 mappings so that - * soft boot works. - */ - setup_mm_for_reboot(); - - /* Clean and invalidate caches */ - flush_cache_all(); - - /* Turn D-cache off */ - cpu_cache_off(); - - /* Push out any further dirty data, and ensure cache is empty */ - flush_cache_all(); -} - void soft_restart(unsigned long addr) { - typedef void (*phys_reset_t)(unsigned long); - phys_reset_t phys_reset; - - setup_restart(); - - /* Switch to the identity mapping */ - phys_reset = (phys_reset_t)virt_to_phys(cpu_reset); - phys_reset(addr); - + setup_mm_for_reboot(); + cpu_soft_restart(virt_to_phys(cpu_reset), addr); /* Should never get here */ BUG(); } diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index e85bf0667bd7..cc65d201cd9d 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -69,6 +69,21 @@ ENTRY(cpu_reset) ret x0 ENDPROC(cpu_reset) +ENTRY(cpu_soft_restart) + /* Save address of cpu_reset() and reset address */ + mov x19, x0 + mov x20, x1 + + /* Turn D-cache off */ + bl cpu_cache_off + + /* Push out all dirty data, and ensure cache is empty */ + bl flush_cache_all + + mov x0, x20 + ret x19 +ENDPROC(cpu_soft_restart) + /* * cpu_do_idle() * -- cgit v1.2.3 From 73cf4139a89eb67351d5e63209d8f554ada0ae6e Mon Sep 17 00:00:00 2001 From: Laura Abbott Date: Tue, 19 Aug 2014 20:41:42 +0100 Subject: arm64: Introduce {set,clear}_pte_bit It's useful to be able to change individual bits in ptes at times. Introduce functions for this and update existing pte_mk* functions to use these primatives. Acked-by: Will Deacon Signed-off-by: Laura Abbott [will: added missing inline keyword for new header functions] Signed-off-by: Will Deacon (cherry picked from commit b6d4f2800b7bad654caf00654f4bff21594ef838) Signed-off-by: Mark Brown --- arch/arm64/include/asm/pgtable.h | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 7099e3b84778..d86b4d4df1f0 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -141,46 +141,51 @@ extern struct page *empty_zero_page; #define pte_valid_not_user(pte) \ ((pte_val(pte) & (PTE_VALID | PTE_USER)) == PTE_VALID) -static inline pte_t pte_wrprotect(pte_t pte) +static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot) { - pte_val(pte) &= ~PTE_WRITE; + pte_val(pte) &= ~pgprot_val(prot); return pte; } -static inline pte_t pte_mkwrite(pte_t pte) +static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot) { - pte_val(pte) |= PTE_WRITE; + pte_val(pte) |= pgprot_val(prot); return pte; } +static inline pte_t pte_wrprotect(pte_t pte) +{ + return clear_pte_bit(pte, __pgprot(PTE_WRITE)); +} + +static inline pte_t pte_mkwrite(pte_t pte) +{ + return set_pte_bit(pte, __pgprot(PTE_WRITE)); +} + static inline pte_t pte_mkclean(pte_t pte) { - pte_val(pte) &= ~PTE_DIRTY; - return pte; + return clear_pte_bit(pte, __pgprot(PTE_DIRTY)); } static inline pte_t pte_mkdirty(pte_t pte) { - pte_val(pte) |= PTE_DIRTY; - return pte; + return set_pte_bit(pte, __pgprot(PTE_DIRTY)); } static inline pte_t pte_mkold(pte_t pte) { - pte_val(pte) &= ~PTE_AF; - return pte; + return clear_pte_bit(pte, __pgprot(PTE_AF)); } static inline pte_t pte_mkyoung(pte_t pte) { - pte_val(pte) |= PTE_AF; - return pte; + return set_pte_bit(pte, __pgprot(PTE_AF)); } static inline pte_t pte_mkspecial(pte_t pte) { - pte_val(pte) |= PTE_SPECIAL; - return pte; + return set_pte_bit(pte, __pgprot(PTE_SPECIAL)); } static inline void set_pte(pte_t *ptep, pte_t pte) -- cgit v1.2.3 From da81aae1834686f559d6e06060d8f3c3f7cdf508 Mon Sep 17 00:00:00 2001 From: Laura Abbott Date: Tue, 19 Aug 2014 20:41:43 +0100 Subject: arm64: Add CONFIG_DEBUG_SET_MODULE_RONX support In a similar fashion to other architecture, add the infrastructure and Kconfig to enable DEBUG_SET_MODULE_RONX support. When enabled, module ranges will be marked read-only/no-execute as appropriate. Signed-off-by: Laura Abbott [will: fixed off-by-one in module end check] Signed-off-by: Will Deacon (cherry picked from commit 11d91a770f1fff44dafdf88d6089a3451f99c9b6) Signed-off-by: Mark Brown Conflicts: arch/arm64/Kconfig.debug --- arch/arm64/Kconfig.debug | 11 +++++ arch/arm64/include/asm/cacheflush.h | 4 ++ arch/arm64/mm/Makefile | 2 +- arch/arm64/mm/pageattr.c | 96 +++++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/mm/pageattr.c diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index e1b0c4601b3e..bb55717c8dad 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -44,4 +44,15 @@ config PID_IN_CONTEXTIDR instructions during context switch. Say Y here only if you are planning to use hardware trace tools with this kernel. +config DEBUG_SET_MODULE_RONX + bool "Set loadable kernel module data as NX and text as RO" + depends on MODULES + help + This option helps catch unintended modifications to loadable + kernel module's text and read-only data. It also prevents execution + of module data. Such protection may interfere with run-time code + patching and dynamic kernel tracing - and they might also protect + against certain classes of kernel exploits. + If in doubt, say "N". + endmenu diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h index f2defe1c380c..689b6379188c 100644 --- a/arch/arm64/include/asm/cacheflush.h +++ b/arch/arm64/include/asm/cacheflush.h @@ -148,4 +148,8 @@ static inline void flush_cache_vunmap(unsigned long start, unsigned long end) { } +int set_memory_ro(unsigned long addr, int numpages); +int set_memory_rw(unsigned long addr, int numpages); +int set_memory_x(unsigned long addr, int numpages); +int set_memory_nx(unsigned long addr, int numpages); #endif diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile index 3ecb56c624d3..c56179ed2c09 100644 --- a/arch/arm64/mm/Makefile +++ b/arch/arm64/mm/Makefile @@ -1,5 +1,5 @@ obj-y := dma-mapping.o extable.o fault.o init.o \ cache.o copypage.o flush.o \ ioremap.o mmap.o pgd.o mmu.o \ - context.o proc.o + context.o proc.o pageattr.o obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c new file mode 100644 index 000000000000..75e744e4cec5 --- /dev/null +++ b/arch/arm64/mm/pageattr.c @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2014, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#include +#include +#include +#include + +#include +#include + +struct page_change_data { + pgprot_t set_mask; + pgprot_t clear_mask; +}; + +static int change_page_range(pte_t *ptep, pgtable_t token, unsigned long addr, + void *data) +{ + struct page_change_data *cdata = data; + pte_t pte = *ptep; + + pte = clear_pte_bit(pte, cdata->clear_mask); + pte = set_pte_bit(pte, cdata->set_mask); + + set_pte(ptep, pte); + return 0; +} + +static int change_memory_common(unsigned long addr, int numpages, + pgprot_t set_mask, pgprot_t clear_mask) +{ + unsigned long start = addr; + unsigned long size = PAGE_SIZE*numpages; + unsigned long end = start + size; + int ret; + struct page_change_data data; + + if (!IS_ALIGNED(addr, PAGE_SIZE)) { + addr &= PAGE_MASK; + WARN_ON_ONCE(1); + } + + if (!is_module_address(start) || !is_module_address(end - 1)) + return -EINVAL; + + data.set_mask = set_mask; + data.clear_mask = clear_mask; + + ret = apply_to_page_range(&init_mm, start, size, change_page_range, + &data); + + flush_tlb_kernel_range(start, end); + return ret; +} + +int set_memory_ro(unsigned long addr, int numpages) +{ + return change_memory_common(addr, numpages, + __pgprot(PTE_RDONLY), + __pgprot(PTE_WRITE)); +} +EXPORT_SYMBOL_GPL(set_memory_ro); + +int set_memory_rw(unsigned long addr, int numpages) +{ + return change_memory_common(addr, numpages, + __pgprot(PTE_WRITE), + __pgprot(PTE_RDONLY)); +} +EXPORT_SYMBOL_GPL(set_memory_rw); + +int set_memory_nx(unsigned long addr, int numpages) +{ + return change_memory_common(addr, numpages, + __pgprot(PTE_PXN), + __pgprot(0)); +} +EXPORT_SYMBOL_GPL(set_memory_nx); + +int set_memory_x(unsigned long addr, int numpages) +{ + return change_memory_common(addr, numpages, + __pgprot(0), + __pgprot(PTE_PXN)); +} +EXPORT_SYMBOL_GPL(set_memory_x); -- cgit v1.2.3 From 98f12c0a80ae19316c1b32f610d094aaae2c5fc0 Mon Sep 17 00:00:00 2001 From: Behan Webster Date: Wed, 27 Aug 2014 05:29:29 +0100 Subject: arm64: LLVMLinux: Add current_stack_pointer() for arm64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Define a global named register for current_stack_pointer. The use of this new variable guarantees that both gcc and clang can access this register in C code. Signed-off-by: Behan Webster Reviewed-by: Jan-Simon Möller Reviewed-by: Mark Charlebois Reviewed-by: Olof Johansson Acked-by: Will Deacon Signed-off-by: Will Deacon (cherry picked from commit 3337a10e0d0cbc9225cefc23aa7a604b698367ed) Signed-off-by: Mark Brown --- arch/arm64/include/asm/thread_info.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h index 23a3c4791d86..5093806522e1 100644 --- a/arch/arm64/include/asm/thread_info.h +++ b/arch/arm64/include/asm/thread_info.h @@ -68,6 +68,11 @@ struct thread_info { #define init_thread_info (init_thread_union.thread_info) #define init_stack (init_thread_union.stack) +/* + * how to get the current stack pointer from C + */ +register unsigned long current_stack_pointer asm ("sp"); + /* * how to get the thread information struct from C */ -- cgit v1.2.3 From 9b2ff971ff9b5d88712078c6ebafb1d85e1469d7 Mon Sep 17 00:00:00 2001 From: Behan Webster Date: Wed, 27 Aug 2014 05:29:30 +0100 Subject: arm64: LLVMLinux: Use current_stack_pointer in save_stack_trace_tsk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the global current_stack_pointer to get the value of the stack pointer. This change supports being able to compile the kernel with both gcc and clang. Signed-off-by: Behan Webster Signed-off-by: Mark Charlebois Reviewed-by: Jan-Simon Möller Reviewed-by: Olof Johansson Acked-by: Will Deacon Signed-off-by: Will Deacon (cherry picked from commit bb28cec4ea2f5151c08e061c6de825a8c853bbd6) Signed-off-by: Mark Brown --- arch/arm64/kernel/stacktrace.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c index 38f0558f0c0a..6f44bf97e4cf 100644 --- a/arch/arm64/kernel/stacktrace.c +++ b/arch/arm64/kernel/stacktrace.c @@ -111,10 +111,9 @@ void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) frame.sp = thread_saved_sp(tsk); frame.pc = thread_saved_pc(tsk); } else { - register unsigned long current_sp asm("sp"); data.no_sched_functions = 0; frame.fp = (unsigned long)__builtin_frame_address(0); - frame.sp = current_sp; + frame.sp = current_stack_pointer; frame.pc = (unsigned long)save_stack_trace_tsk; } -- cgit v1.2.3 From 51c50680dbf8a763bd787e0f7e3fff3df7036997 Mon Sep 17 00:00:00 2001 From: Behan Webster Date: Wed, 27 Aug 2014 05:29:31 +0100 Subject: arm64: LLVMLinux: Calculate current_thread_info from current_stack_pointer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the global current_stack_pointer to get the value of the stack pointer. This change supports being able to compile the kernel with both gcc and clang. Signed-off-by: Behan Webster Signed-off-by: Mark Charlebois Reviewed-by: Jan-Simon Möller Reviewed-by: Olof Johansson Acked-by: Will Deacon Signed-off-by: Will Deacon (cherry picked from commit 786248705ecf5290f26534e8eef62ba6dd63b806) Signed-off-by: Mark Brown --- arch/arm64/include/asm/thread_info.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h index 5093806522e1..d971013f73c1 100644 --- a/arch/arm64/include/asm/thread_info.h +++ b/arch/arm64/include/asm/thread_info.h @@ -80,8 +80,8 @@ static inline struct thread_info *current_thread_info(void) __attribute_const__; static inline struct thread_info *current_thread_info(void) { - register unsigned long sp asm ("sp"); - return (struct thread_info *)(sp & ~(THREAD_SIZE - 1)); + return (struct thread_info *) + (current_stack_pointer & ~(THREAD_SIZE - 1)); } #define thread_saved_pc(tsk) \ -- cgit v1.2.3 From 29769603ea7b06e98f229b87bc0a67058cea394d Mon Sep 17 00:00:00 2001 From: Behan Webster Date: Wed, 27 Aug 2014 05:29:32 +0100 Subject: arm64: LLVMLinux: Use current_stack_pointer in kernel/traps.c Use the global current_stack_pointer to get the value of the stack pointer. This change supports being able to compile the kernel with both gcc and clang. Signed-off-by: Behan Webster Signed-off-by: Mark Charlebois Reviewed-by: Olof Johansson Acked-by: Will Deacon Signed-off-by: Will Deacon (cherry picked from commit 2128df143d840a20e12818290eb6e40b95cc4ac0) Signed-off-by: Mark Brown --- arch/arm64/kernel/traps.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 268ce966afd6..12cd34ff35d0 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -132,7 +132,6 @@ static void dump_instr(const char *lvl, struct pt_regs *regs) static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) { struct stackframe frame; - const register unsigned long current_sp asm ("sp"); pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk); @@ -145,7 +144,7 @@ static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) frame.pc = regs->pc; } else if (tsk == current) { frame.fp = (unsigned long)__builtin_frame_address(0); - frame.sp = current_sp; + frame.sp = current_stack_pointer; frame.pc = (unsigned long)dump_backtrace; } else { /* -- cgit v1.2.3 From 0fab209aac128615dd2a8e88188073f89fba6dab Mon Sep 17 00:00:00 2001 From: Mark Charlebois Date: Wed, 27 Aug 2014 05:29:33 +0100 Subject: arm64: LLVMLinux: Use global stack register variable for aarch64 To support both Clang and GCC, use the global stack register variable vs a local register variable. Author: Mark Charlebois Signed-off-by: Mark Charlebois Signed-off-by: Behan Webster Signed-off-by: Will Deacon (cherry picked from commit 34ccf8f455f1ae7761810a74308f82daca67ced1) Signed-off-by: Mark Brown --- arch/arm64/include/asm/percpu.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h index 453a179469a3..5279e5733386 100644 --- a/arch/arm64/include/asm/percpu.h +++ b/arch/arm64/include/asm/percpu.h @@ -26,13 +26,13 @@ static inline void set_my_cpu_offset(unsigned long off) static inline unsigned long __my_cpu_offset(void) { unsigned long off; - register unsigned long *sp asm ("sp"); /* * We want to allow caching the value, so avoid using volatile and * instead use a fake stack read to hazard against barrier(). */ - asm("mrs %0, tpidr_el1" : "=r" (off) : "Q" (*sp)); + asm("mrs %0, tpidr_el1" : "=r" (off) : + "Q" (*(const unsigned long *)current_stack_pointer)); return off; } -- cgit v1.2.3 From d74b6d355a010ed2d70da9455f92167f8f5bbd60 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Fri, 23 Jan 2015 20:15:15 +0000 Subject: Revert "arm64: vdso: move data page before code pages" This reverts commit 82a95d0521cb6258559d18dca736da8272ba05a7. Signed-off-by: Mark Brown --- arch/arm64/kernel/vdso.c | 34 +++++++++++++++++----------------- arch/arm64/kernel/vdso/vdso.lds.S | 4 +++- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c index b8289f08795f..ad8dddbffef2 100644 --- a/arch/arm64/kernel/vdso.c +++ b/arch/arm64/kernel/vdso.c @@ -118,8 +118,8 @@ static int __init vdso_init(void) } vdso_pages = (&vdso_end - &vdso_start) >> PAGE_SHIFT; - pr_info("vdso: %ld pages (%ld code @ %p, %ld data @ %p)\n", - vdso_pages + 1, vdso_pages, &vdso_start, 1L, vdso_data); + pr_info("vdso: %ld pages (%ld code, %ld data) at base %p\n", + vdso_pages + 1, vdso_pages, 1L, &vdso_start); /* Allocate the vDSO pagelist, plus a page for the data. */ vdso_pagelist = kcalloc(vdso_pages + 1, sizeof(struct page *), @@ -127,22 +127,22 @@ static int __init vdso_init(void) if (vdso_pagelist == NULL) return -ENOMEM; - /* Grab the vDSO data page. */ - vdso_pagelist[0] = virt_to_page(vdso_data); - /* Grab the vDSO code pages. */ for (i = 0; i < vdso_pages; i++) - vdso_pagelist[i + 1] = virt_to_page(&vdso_start + i * PAGE_SIZE); + vdso_pagelist[i] = virt_to_page(&vdso_start + i * PAGE_SIZE); + + /* Grab the vDSO data page. */ + vdso_pagelist[i] = virt_to_page(vdso_data); /* Populate the special mapping structures */ vdso_spec[0] = (struct vm_special_mapping) { - .name = "[vvar]", + .name = "[vdso]", .pages = vdso_pagelist, }; vdso_spec[1] = (struct vm_special_mapping) { - .name = "[vdso]", - .pages = &vdso_pagelist[1], + .name = "[vvar]", + .pages = vdso_pagelist + vdso_pages, }; return 0; @@ -166,21 +166,21 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, ret = ERR_PTR(vdso_base); goto up_fail; } - ret = _install_special_mapping(mm, vdso_base, PAGE_SIZE, - VM_READ|VM_MAYREAD, - &vdso_spec[0]); - if (IS_ERR(ret)) - goto up_fail; - - vdso_base += PAGE_SIZE; mm->context.vdso = (void *)vdso_base; + ret = _install_special_mapping(mm, vdso_base, vdso_text_len, VM_READ|VM_EXEC| VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, - &vdso_spec[1]); + &vdso_spec[0]); if (IS_ERR(ret)) goto up_fail; + vdso_base += vdso_text_len; + ret = _install_special_mapping(mm, vdso_base, PAGE_SIZE, + VM_READ|VM_MAYREAD, + &vdso_spec[1]); + if (IS_ERR(ret)) + goto up_fail; up_write(&mm->mmap_sem); return 0; diff --git a/arch/arm64/kernel/vdso/vdso.lds.S b/arch/arm64/kernel/vdso/vdso.lds.S index beca249bc2f3..8154b8d1c826 100644 --- a/arch/arm64/kernel/vdso/vdso.lds.S +++ b/arch/arm64/kernel/vdso/vdso.lds.S @@ -28,7 +28,6 @@ OUTPUT_ARCH(aarch64) SECTIONS { - PROVIDE(_vdso_data = . - PAGE_SIZE); . = VDSO_LBASE + SIZEOF_HEADERS; .hash : { *(.hash) } :text @@ -58,6 +57,9 @@ SECTIONS _end = .; PROVIDE(end = .); + . = ALIGN(PAGE_SIZE); + PROVIDE(_vdso_data = .); + /DISCARD/ : { *(.note.GNU-stack) *(.data .data.* .gnu.linkonce.d.* .sdata*) -- cgit v1.2.3 From b749ecc6dac6514265c4a07f3d283c214377fcc5 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Fri, 23 Jan 2015 20:15:32 +0000 Subject: Revert "arm64: vdso: move to _install_special_mapping and remove arch_vma_name" This reverts commit 95c91bdd8eafdc74337049c45d74c903b7dac49c. Signed-off-by: Mark Brown --- arch/arm64/kernel/vdso.c | 80 +++++++++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 35 deletions(-) diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c index ad8dddbffef2..40ba5dea2ed7 100644 --- a/arch/arm64/kernel/vdso.c +++ b/arch/arm64/kernel/vdso.c @@ -85,29 +85,22 @@ int aarch32_setup_vectors_page(struct linux_binprm *bprm, int uses_interp) { struct mm_struct *mm = current->mm; unsigned long addr = AARCH32_VECTORS_BASE; - static struct vm_special_mapping spec = { - .name = "[vectors]", - .pages = vectors_page, - - }; - void *ret; + int ret; down_write(&mm->mmap_sem); current->mm->context.vdso = (void *)addr; /* Map vectors page at the high address. */ - ret = _install_special_mapping(mm, addr, PAGE_SIZE, - VM_READ|VM_EXEC|VM_MAYREAD|VM_MAYEXEC, - &spec); + ret = install_special_mapping(mm, addr, PAGE_SIZE, + VM_READ|VM_EXEC|VM_MAYREAD|VM_MAYEXEC, + vectors_page); up_write(&mm->mmap_sem); - return PTR_ERR_OR_ZERO(ret); + return ret; } #endif /* CONFIG_COMPAT */ -static struct vm_special_mapping vdso_spec[2]; - static int __init vdso_init(void) { int i; @@ -134,17 +127,6 @@ static int __init vdso_init(void) /* Grab the vDSO data page. */ vdso_pagelist[i] = virt_to_page(vdso_data); - /* Populate the special mapping structures */ - vdso_spec[0] = (struct vm_special_mapping) { - .name = "[vdso]", - .pages = vdso_pagelist, - }; - - vdso_spec[1] = (struct vm_special_mapping) { - .name = "[vvar]", - .pages = vdso_pagelist + vdso_pages, - }; - return 0; } arch_initcall(vdso_init); @@ -154,7 +136,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, { struct mm_struct *mm = current->mm; unsigned long vdso_base, vdso_text_len, vdso_mapping_len; - void *ret; + int ret; vdso_text_len = vdso_pages << PAGE_SHIFT; /* Be sure to map the data page */ @@ -163,23 +145,23 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, down_write(&mm->mmap_sem); vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0); if (IS_ERR_VALUE(vdso_base)) { - ret = ERR_PTR(vdso_base); + ret = vdso_base; goto up_fail; } mm->context.vdso = (void *)vdso_base; - ret = _install_special_mapping(mm, vdso_base, vdso_text_len, - VM_READ|VM_EXEC| - VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, - &vdso_spec[0]); - if (IS_ERR(ret)) + ret = install_special_mapping(mm, vdso_base, vdso_text_len, + VM_READ|VM_EXEC| + VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, + vdso_pagelist); + if (ret) goto up_fail; vdso_base += vdso_text_len; - ret = _install_special_mapping(mm, vdso_base, PAGE_SIZE, - VM_READ|VM_MAYREAD, - &vdso_spec[1]); - if (IS_ERR(ret)) + ret = install_special_mapping(mm, vdso_base, PAGE_SIZE, + VM_READ|VM_MAYREAD, + vdso_pagelist + vdso_pages); + if (ret) goto up_fail; up_write(&mm->mmap_sem); @@ -188,7 +170,35 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, up_fail: mm->context.vdso = NULL; up_write(&mm->mmap_sem); - return PTR_ERR(ret); + return ret; +} + +const char *arch_vma_name(struct vm_area_struct *vma) +{ + unsigned long vdso_text; + + if (!vma->vm_mm) + return NULL; + + vdso_text = (unsigned long)vma->vm_mm->context.vdso; + + /* + * We can re-use the vdso pointer in mm_context_t for identifying + * the vectors page for compat applications. The vDSO will always + * sit above TASK_UNMAPPED_BASE and so we don't need to worry about + * it conflicting with the vectors base. + */ + if (vma->vm_start == vdso_text) { +#ifdef CONFIG_COMPAT + if (vma->vm_start == AARCH32_VECTORS_BASE) + return "[vectors]"; +#endif + return "[vdso]"; + } else if (vma->vm_start == (vdso_text + (vdso_pages << PAGE_SHIFT))) { + return "[vvar]"; + } + + return NULL; } /* -- cgit v1.2.3 From 9698ac7c353d6f22100c9fa24a830cae6a466cb4 Mon Sep 17 00:00:00 2001 From: Laura Abbott Date: Thu, 9 Oct 2014 15:26:44 -0700 Subject: arm64: add atomic pool for non-coherent and CMA allocations Neither CMA nor noncoherent allocations support atomic allocations. Add a dedicated atomic pool to support this. Reviewed-by: Catalin Marinas Signed-off-by: Laura Abbott Cc: Arnd Bergmann Cc: David Riley Cc: Olof Johansson Cc: Ritesh Harjain Cc: Russell King Cc: Thierry Reding Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds (cherry picked from commit d4932f9e81ae7a7bf3c3967e48373909b9c98ee5) Signed-off-by: Mark Brown --- arch/arm64/Kconfig | 1 + arch/arm64/mm/dma-mapping.c | 164 +++++++++++++++++++++++++++++++++++++++----- drivers/base/dma-mapping.c | 2 +- 3 files changed, 147 insertions(+), 20 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 4142cab9d4ff..d185ea209ef1 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -13,6 +13,7 @@ config ARM64 select CLONE_BACKWARDS select COMMON_CLK select DCACHE_WORD_ACCESS + select GENERIC_ALLOCATOR select GENERIC_CLOCKEVENTS select GENERIC_EARLY_IOREMAP select GENERIC_IOMAP diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index f39a55d58918..eeb1cf3ff299 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -43,6 +44,54 @@ static pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot, return prot; } +static struct gen_pool *atomic_pool; + +#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K +static size_t atomic_pool_size = DEFAULT_DMA_COHERENT_POOL_SIZE; + +static int __init early_coherent_pool(char *p) +{ + atomic_pool_size = memparse(p, &p); + return 0; +} +early_param("coherent_pool", early_coherent_pool); + +static void *__alloc_from_pool(size_t size, struct page **ret_page) +{ + unsigned long val; + void *ptr = NULL; + + if (!atomic_pool) { + WARN(1, "coherent pool not initialised!\n"); + return NULL; + } + + val = gen_pool_alloc(atomic_pool, size); + if (val) { + phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val); + + *ret_page = phys_to_page(phys); + ptr = (void *)val; + } + + return ptr; +} + +static bool __in_atomic_pool(void *start, size_t size) +{ + return addr_in_gen_pool(atomic_pool, (unsigned long)start, size); +} + +static int __free_from_pool(void *start, size_t size) +{ + if (!__in_atomic_pool(start, size)) + return 0; + + gen_pool_free(atomic_pool, (unsigned long)start, size); + + return 1; +} + static void *__dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flags, struct dma_attrs *attrs) @@ -50,7 +99,7 @@ static void *__dma_alloc_coherent(struct device *dev, size_t size, if (IS_ENABLED(CONFIG_ZONE_DMA) && dev->coherent_dma_mask <= DMA_BIT_MASK(32)) flags |= GFP_DMA; - if (IS_ENABLED(CONFIG_DMA_CMA)) { + if (IS_ENABLED(CONFIG_DMA_CMA) && (flags & __GFP_WAIT)) { struct page *page; size = PAGE_ALIGN(size); @@ -70,50 +119,54 @@ static void __dma_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, struct dma_attrs *attrs) { + bool freed; + phys_addr_t paddr = dma_to_phys(dev, dma_handle); + if (dev == NULL) { WARN_ONCE(1, "Use an actual device structure for DMA allocation\n"); return; } - if (IS_ENABLED(CONFIG_DMA_CMA)) { - phys_addr_t paddr = dma_to_phys(dev, dma_handle); - - dma_release_from_contiguous(dev, + freed = dma_release_from_contiguous(dev, phys_to_page(paddr), size >> PAGE_SHIFT); - } else { + if (!freed) swiotlb_free_coherent(dev, size, vaddr, dma_handle); - } } static void *__dma_alloc_noncoherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flags, struct dma_attrs *attrs) { - struct page *page, **map; + struct page *page; void *ptr, *coherent_ptr; - int order, i; size = PAGE_ALIGN(size); - order = get_order(size); + + if (!(flags & __GFP_WAIT)) { + struct page *page = NULL; + void *addr = __alloc_from_pool(size, &page); + + if (addr) + *dma_handle = phys_to_dma(dev, page_to_phys(page)); + + return addr; + + } ptr = __dma_alloc_coherent(dev, size, dma_handle, flags, attrs); if (!ptr) goto no_mem; - map = kmalloc(sizeof(struct page *) << order, flags & ~GFP_DMA); - if (!map) - goto no_map; /* remove any dirty cache lines on the kernel alias */ __dma_flush_range(ptr, ptr + size); /* create a coherent mapping */ page = virt_to_page(ptr); - for (i = 0; i < (size >> PAGE_SHIFT); i++) - map[i] = page + i; - coherent_ptr = vmap(map, size >> PAGE_SHIFT, VM_MAP, - __get_dma_pgprot(attrs, __pgprot(PROT_NORMAL_NC), false)); - kfree(map); + coherent_ptr = dma_common_contiguous_remap(page, size, VM_USERMAP, + __get_dma_pgprot(attrs, + __pgprot(PROT_NORMAL_NC), false), + NULL); if (!coherent_ptr) goto no_map; @@ -132,6 +185,8 @@ static void __dma_free_noncoherent(struct device *dev, size_t size, { void *swiotlb_addr = phys_to_virt(dma_to_phys(dev, dma_handle)); + if (__free_from_pool(vaddr, size)) + return; vunmap(vaddr); __dma_free_coherent(dev, size, swiotlb_addr, dma_handle, attrs); } @@ -329,6 +384,67 @@ static struct notifier_block amba_bus_nb = { extern int swiotlb_late_init_with_default_size(size_t default_size); +static int __init atomic_pool_init(void) +{ + pgprot_t prot = __pgprot(PROT_NORMAL_NC); + unsigned long nr_pages = atomic_pool_size >> PAGE_SHIFT; + struct page *page; + void *addr; + unsigned int pool_size_order = get_order(atomic_pool_size); + + if (dev_get_cma_area(NULL)) + page = dma_alloc_from_contiguous(NULL, nr_pages, + pool_size_order); + else + page = alloc_pages(GFP_DMA, pool_size_order); + + if (page) { + int ret; + void *page_addr = page_address(page); + + memset(page_addr, 0, atomic_pool_size); + __dma_flush_range(page_addr, page_addr + atomic_pool_size); + + atomic_pool = gen_pool_create(PAGE_SHIFT, -1); + if (!atomic_pool) + goto free_page; + + addr = dma_common_contiguous_remap(page, atomic_pool_size, + VM_USERMAP, prot, atomic_pool_init); + + if (!addr) + goto destroy_genpool; + + ret = gen_pool_add_virt(atomic_pool, (unsigned long)addr, + page_to_phys(page), + atomic_pool_size, -1); + if (ret) + goto remove_mapping; + + gen_pool_set_algo(atomic_pool, + gen_pool_first_fit_order_align, + (void *)PAGE_SHIFT); + + pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n", + atomic_pool_size / 1024); + return 0; + } + goto out; + +remove_mapping: + dma_common_free_remap(addr, atomic_pool_size, VM_USERMAP); +destroy_genpool: + gen_pool_destroy(atomic_pool); + atomic_pool = NULL; +free_page: + if (!dma_release_from_contiguous(NULL, page, nr_pages)) + __free_pages(page, pool_size_order); +out: + pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n", + atomic_pool_size / 1024); + return -ENOMEM; +} + static int __init swiotlb_late_init(void) { size_t swiotlb_size = min(SZ_64M, MAX_ORDER_NR_PAGES << PAGE_SHIFT); @@ -343,7 +459,17 @@ static int __init swiotlb_late_init(void) return swiotlb_late_init_with_default_size(swiotlb_size); } -arch_initcall(swiotlb_late_init); + +static int __init arm64_dma_init(void) +{ + int ret = 0; + + ret |= swiotlb_late_init(); + ret |= atomic_pool_init(); + + return ret; +} +arch_initcall(arm64_dma_init); #define PREALLOC_DMA_DEBUG_ENTRIES 4096 diff --git a/drivers/base/dma-mapping.c b/drivers/base/dma-mapping.c index 9c36ca9ed1dd..dd302ea3b599 100644 --- a/drivers/base/dma-mapping.c +++ b/drivers/base/dma-mapping.c @@ -287,7 +287,7 @@ void *dma_common_pages_remap(struct page **pages, size_t size, area->pages = pages; - if (map_vm_area(area, prot, pages)) { + if (map_vm_area(area, prot, &pages)) { vunmap(area->addr); return NULL; } -- cgit v1.2.3