From fe7946ce0808eb0e43711f5db7d2d1599b362d02 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sun, 23 Jun 2019 20:41:51 +1000 Subject: powerpc/64s: Rename PPC_INVALIDATE_ERAT to PPC_ISA_3_0_INVALIDATE_ERAT This makes it clear to the caller that it can only be used on POWER9 and later CPUs. Signed-off-by: Nicholas Piggin [mpe: Use "ISA_3_0" rather than "ARCH_300"] Signed-off-by: Michael Ellerman --- arch/powerpc/mm/book3s64/radix_tlb.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'arch/powerpc/mm/book3s64/radix_tlb.c') diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c index 4d841369399f..7539c714438e 100644 --- a/arch/powerpc/mm/book3s64/radix_tlb.c +++ b/arch/powerpc/mm/book3s64/radix_tlb.c @@ -87,7 +87,7 @@ void radix__tlbiel_all(unsigned int action) else WARN(1, "%s called on pre-POWER9 CPU\n", __func__); - asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory"); + asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory"); } static __always_inline void __tlbiel_pid(unsigned long pid, int set, @@ -262,7 +262,7 @@ static inline void _tlbiel_pid(unsigned long pid, unsigned long ric) __tlbiel_pid(pid, set, RIC_FLUSH_TLB); asm volatile("ptesync": : :"memory"); - asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory"); + asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory"); } static inline void _tlbie_pid(unsigned long pid, unsigned long ric) @@ -314,7 +314,7 @@ static inline void _tlbiel_lpid(unsigned long lpid, unsigned long ric) __tlbiel_lpid(lpid, set, RIC_FLUSH_TLB); asm volatile("ptesync": : :"memory"); - asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory"); + asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory"); } static inline void _tlbie_lpid(unsigned long lpid, unsigned long ric) @@ -366,7 +366,7 @@ static inline void _tlbiel_lpid_guest(unsigned long lpid, unsigned long ric) __tlbiel_lpid_guest(lpid, set, RIC_FLUSH_TLB); asm volatile("ptesync": : :"memory"); - asm volatile(PPC_INVALIDATE_ERAT : : :"memory"); + asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory"); } -- cgit v1.2.3 From 6c46fcce39f0eb4830078c5f1db289dd7196f84a Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sun, 23 Jun 2019 20:41:52 +1000 Subject: powerpc/64s/radix: keep kernel ERAT over local process/guest invalidates ISA v3.0 radix modes provide SLBIA variants which can invalidate ERAT for effPID!=0 or for effLPID!=0, which allows user and guest invalidations to retain kernel/host ERAT entries. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/mm/book3s64/radix_tlb.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'arch/powerpc/mm/book3s64/radix_tlb.c') diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c index 7539c714438e..edf39e9a4ec4 100644 --- a/arch/powerpc/mm/book3s64/radix_tlb.c +++ b/arch/powerpc/mm/book3s64/radix_tlb.c @@ -262,7 +262,7 @@ static inline void _tlbiel_pid(unsigned long pid, unsigned long ric) __tlbiel_pid(pid, set, RIC_FLUSH_TLB); asm volatile("ptesync": : :"memory"); - asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory"); + asm volatile(PPC_RADIX_INVALIDATE_ERAT_USER "; isync" : : :"memory"); } static inline void _tlbie_pid(unsigned long pid, unsigned long ric) @@ -314,7 +314,7 @@ static inline void _tlbiel_lpid(unsigned long lpid, unsigned long ric) __tlbiel_lpid(lpid, set, RIC_FLUSH_TLB); asm volatile("ptesync": : :"memory"); - asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory"); + asm volatile(PPC_RADIX_INVALIDATE_ERAT_GUEST "; isync" : : :"memory"); } static inline void _tlbie_lpid(unsigned long lpid, unsigned long ric) @@ -366,7 +366,7 @@ static inline void _tlbiel_lpid_guest(unsigned long lpid, unsigned long ric) __tlbiel_lpid_guest(lpid, set, RIC_FLUSH_TLB); asm volatile("ptesync": : :"memory"); - asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory"); + asm volatile(PPC_RADIX_INVALIDATE_ERAT_GUEST : : :"memory"); } -- cgit v1.2.3 From 6d3ca7e73642ce17398f4cd5df1780da4a1ccdaf Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Tue, 21 May 2019 22:13:24 +0900 Subject: powerpc/mm: mark more tlb functions as __always_inline With CONFIG_OPTIMIZE_INLINING enabled, Laura Abbott reported error with gcc 9.1.1: arch/powerpc/mm/book3s64/radix_tlb.c: In function '_tlbiel_pid': arch/powerpc/mm/book3s64/radix_tlb.c:104:2: warning: asm operand 3 probably doesn't match constraints 104 | asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) | ^~~ arch/powerpc/mm/book3s64/radix_tlb.c:104:2: error: impossible constraint in 'asm' Fixing _tlbiel_pid() is enough to address the warning above, but I inlined more functions to fix all potential issues. To meet the "i" (immediate) constraint for the asm operands, functions propagating "ric" must be always inlined. Fixes: 9012d011660e ("compiler: allow all arches to enable CONFIG_OPTIMIZE_INLINING") Reported-by: Laura Abbott Signed-off-by: Masahiro Yamada Reviewed-by: Christophe Leroy Signed-off-by: Michael Ellerman --- arch/powerpc/mm/book3s64/radix_tlb.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) (limited to 'arch/powerpc/mm/book3s64/radix_tlb.c') diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c index edf39e9a4ec4..8901f5c8ef8c 100644 --- a/arch/powerpc/mm/book3s64/radix_tlb.c +++ b/arch/powerpc/mm/book3s64/radix_tlb.c @@ -29,7 +29,7 @@ * tlbiel instruction for radix, set invalidation * i.e., r=1 and is=01 or is=10 or is=11 */ -static inline void tlbiel_radix_set_isa300(unsigned int set, unsigned int is, +static __always_inline void tlbiel_radix_set_isa300(unsigned int set, unsigned int is, unsigned int pid, unsigned int ric, unsigned int prs) { @@ -150,8 +150,8 @@ static __always_inline void __tlbie_lpid(unsigned long lpid, unsigned long ric) trace_tlbie(lpid, 0, rb, rs, ric, prs, r); } -static inline void __tlbiel_lpid_guest(unsigned long lpid, int set, - unsigned long ric) +static __always_inline void __tlbiel_lpid_guest(unsigned long lpid, int set, + unsigned long ric) { unsigned long rb,rs,prs,r; @@ -167,8 +167,8 @@ static inline void __tlbiel_lpid_guest(unsigned long lpid, int set, } -static inline void __tlbiel_va(unsigned long va, unsigned long pid, - unsigned long ap, unsigned long ric) +static __always_inline void __tlbiel_va(unsigned long va, unsigned long pid, + unsigned long ap, unsigned long ric) { unsigned long rb,rs,prs,r; @@ -183,8 +183,8 @@ static inline void __tlbiel_va(unsigned long va, unsigned long pid, trace_tlbie(0, 1, rb, rs, ric, prs, r); } -static inline void __tlbie_va(unsigned long va, unsigned long pid, - unsigned long ap, unsigned long ric) +static __always_inline void __tlbie_va(unsigned long va, unsigned long pid, + unsigned long ap, unsigned long ric) { unsigned long rb,rs,prs,r; @@ -199,8 +199,8 @@ static inline void __tlbie_va(unsigned long va, unsigned long pid, trace_tlbie(0, 0, rb, rs, ric, prs, r); } -static inline void __tlbie_lpid_va(unsigned long va, unsigned long lpid, - unsigned long ap, unsigned long ric) +static __always_inline void __tlbie_lpid_va(unsigned long va, unsigned long lpid, + unsigned long ap, unsigned long ric) { unsigned long rb,rs,prs,r; @@ -239,7 +239,7 @@ static inline void fixup_tlbie_lpid(unsigned long lpid) /* * We use 128 set in radix mode and 256 set in hpt mode. */ -static inline void _tlbiel_pid(unsigned long pid, unsigned long ric) +static __always_inline void _tlbiel_pid(unsigned long pid, unsigned long ric) { int set; @@ -341,7 +341,7 @@ static inline void _tlbie_lpid(unsigned long lpid, unsigned long ric) asm volatile("eieio; tlbsync; ptesync": : :"memory"); } -static inline void _tlbiel_lpid_guest(unsigned long lpid, unsigned long ric) +static __always_inline void _tlbiel_lpid_guest(unsigned long lpid, unsigned long ric) { int set; @@ -381,8 +381,8 @@ static inline void __tlbiel_va_range(unsigned long start, unsigned long end, __tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB); } -static inline void _tlbiel_va(unsigned long va, unsigned long pid, - unsigned long psize, unsigned long ric) +static __always_inline void _tlbiel_va(unsigned long va, unsigned long pid, + unsigned long psize, unsigned long ric) { unsigned long ap = mmu_get_ap(psize); @@ -413,8 +413,8 @@ static inline void __tlbie_va_range(unsigned long start, unsigned long end, __tlbie_va(addr, pid, ap, RIC_FLUSH_TLB); } -static inline void _tlbie_va(unsigned long va, unsigned long pid, - unsigned long psize, unsigned long ric) +static __always_inline void _tlbie_va(unsigned long va, unsigned long pid, + unsigned long psize, unsigned long ric) { unsigned long ap = mmu_get_ap(psize); @@ -424,7 +424,7 @@ static inline void _tlbie_va(unsigned long va, unsigned long pid, asm volatile("eieio; tlbsync; ptesync": : :"memory"); } -static inline void _tlbie_lpid_va(unsigned long va, unsigned long lpid, +static __always_inline void _tlbie_lpid_va(unsigned long va, unsigned long lpid, unsigned long psize, unsigned long ric) { unsigned long ap = mmu_get_ap(psize); -- cgit v1.2.3