From 6b07d38aaa520cee922fadfeaf90c97faf217045 Mon Sep 17 00:00:00 2001 From: David Daney Date: Fri, 8 Jan 2010 17:17:44 -0800 Subject: MIPS: Octeon: Use optimized memory barrier primitives. In order to achieve correct synchronization semantics, the Octeon port had defined CONFIG_WEAK_REORDERING_BEYOND_LLSC. This resulted in code that looks like: sync ll ... . . . sc ... . . sync The second SYNC was redundant, but harmless. Octeon has a SYNCW instruction that acts as a write-memory-barrier (due to an erratum in some parts two SYNCW are used). It is much faster than SYNC because it imposes ordering on the writes, but doesn't otherwise stall the execution pipeline. On Octeon, SYNC stalls execution until all preceeding writes are committed to the coherent memory system. Using: syncw;syncw ll . . . sc . . Has identical semantics to the first sequence, but is much faster. The SYNCW orders the writes, and the SC will not complete successfully until the write is committed to the coherent memory system. So at the end all preceeding writes have been committed. Since Octeon does not do speculative reads, this functions as a full barrier. The patch removes CONFIG_WEAK_REORDERING_BEYOND_LLSC, and substitutes SYNCW for SYNC in write-memory-barriers. Signed-off-by: David Daney To: linux-mips@linux-mips.org Patchwork: http://patchwork.linux-mips.org/patch/850/ Signed-off-by: Ralf Baechle --- arch/mips/Kconfig | 1 - arch/mips/include/asm/barrier.h | 43 +++++++++++++++++++++++++++++------------ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 8b5d174685f..70376e58631 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -1295,7 +1295,6 @@ config CPU_CAVIUM_OCTEON select SYS_SUPPORTS_SMP select NR_CPUS_DEFAULT_16 select WEAK_ORDERING - select WEAK_REORDERING_BEYOND_LLSC select CPU_SUPPORTS_HIGHMEM select CPU_SUPPORTS_HUGEPAGES help diff --git a/arch/mips/include/asm/barrier.h b/arch/mips/include/asm/barrier.h index 1a5a51c3e96..a2670a239e0 100644 --- a/arch/mips/include/asm/barrier.h +++ b/arch/mips/include/asm/barrier.h @@ -88,12 +88,20 @@ : /* no output */ \ : "m" (*(int *)CKSEG1) \ : "memory") - -#define fast_wmb() __sync() -#define fast_rmb() __sync() -#define fast_mb() __sync() -#ifdef CONFIG_SGI_IP28 -#define fast_iob() \ +#ifdef CONFIG_CPU_CAVIUM_OCTEON +# define OCTEON_SYNCW_STR ".set push\n.set arch=octeon\nsyncw\nsyncw\n.set pop\n" +# define __syncw() __asm__ __volatile__(OCTEON_SYNCW_STR : : : "memory") + +# define fast_wmb() __syncw() +# define fast_rmb() barrier() +# define fast_mb() __sync() +# define fast_iob() do { } while (0) +#else /* ! CONFIG_CPU_CAVIUM_OCTEON */ +# define fast_wmb() __sync() +# define fast_rmb() __sync() +# define fast_mb() __sync() +# ifdef CONFIG_SGI_IP28 +# define fast_iob() \ __asm__ __volatile__( \ ".set push\n\t" \ ".set noreorder\n\t" \ @@ -104,13 +112,14 @@ : /* no output */ \ : "m" (*(int *)CKSEG1ADDR(0x1fa00004)) \ : "memory") -#else -#define fast_iob() \ +# else +# define fast_iob() \ do { \ __sync(); \ __fast_iob(); \ } while (0) -#endif +# endif +#endif /* CONFIG_CPU_CAVIUM_OCTEON */ #ifdef CONFIG_CPU_HAS_WB @@ -131,9 +140,15 @@ #endif /* !CONFIG_CPU_HAS_WB */ #if defined(CONFIG_WEAK_ORDERING) && defined(CONFIG_SMP) -#define smp_mb() __asm__ __volatile__("sync" : : :"memory") -#define smp_rmb() __asm__ __volatile__("sync" : : :"memory") -#define smp_wmb() __asm__ __volatile__("sync" : : :"memory") +# ifdef CONFIG_CPU_CAVIUM_OCTEON +# define smp_mb() __sync() +# define smp_rmb() barrier() +# define smp_wmb() __syncw() +# else +# define smp_mb() __asm__ __volatile__("sync" : : :"memory") +# define smp_rmb() __asm__ __volatile__("sync" : : :"memory") +# define smp_wmb() __asm__ __volatile__("sync" : : :"memory") +# endif #else #define smp_mb() barrier() #define smp_rmb() barrier() @@ -151,6 +166,10 @@ #define smp_llsc_mb() __asm__ __volatile__(__WEAK_LLSC_MB : : :"memory") +#ifdef CONFIG_CPU_CAVIUM_OCTEON +#define smp_mb__before_llsc() smp_wmb() +#else #define smp_mb__before_llsc() smp_llsc_mb() +#endif #endif /* __ASM_BARRIER_H */ -- cgit v1.2.3