aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAndrey Konovalov <andreyknvl@google.com>2020-12-10 15:42:18 +1100
committerStephen Rothwell <sfr@canb.auug.org.au>2020-12-16 18:03:04 +1100
commite53d048940557d72d154a2e2a82d29972ed66567 (patch)
treeafce164e0a652f2793811c035f57ff718c771328
parent864a94571a6f3dd4e0c67ee215f0a4cae08fcd85 (diff)
downloadlinux-e53d048940557d72d154a2e2a82d29972ed66567.tar.gz
kasan, mm: allow cache merging with no metadata
The reason cache merging is disabled with KASAN is because KASAN puts its metadata right after the allocated object. When the merged caches have slightly different sizes, the metadata ends up in different places, which KASAN doesn't support. It might be possible to adjust the metadata allocation algorithm and make it friendly to the cache merging code. Instead this change takes a simpler approach and allows merging caches when no metadata is present. Which is the case for hardware tag-based KASAN with kasan.mode=prod. Link: https://lkml.kernel.org/r/37497e940bfd4b32c0a93a702a9ae4cf061d5392.1606162397.git.andreyknvl@google.com Link: https://linux-review.googlesource.com/id/Ia114847dfb2244f297d2cb82d592bf6a07455dba Co-developed-by: Vincenzo Frascino <Vincenzo.Frascino@arm.com> Signed-off-by: Vincenzo Frascino <Vincenzo.Frascino@arm.com> Signed-off-by: Andrey Konovalov <andreyknvl@google.com> Reviewed-by: Dmitry Vyukov <dvyukov@google.com> Reviewed-by: Marco Elver <elver@google.com> Tested-by: Vincenzo Frascino <vincenzo.frascino@arm.com> Cc: Alexander Potapenko <glider@google.com> Cc: Andrey Ryabinin <aryabinin@virtuozzo.com> Cc: Branislav Rankov <Branislav.Rankov@arm.com> Cc: Catalin Marinas <catalin.marinas@arm.com> Cc: Evgenii Stepanov <eugenis@google.com> Cc: Kevin Brodsky <kevin.brodsky@arm.com> Cc: Vasily Gorbik <gor@linux.ibm.com> Cc: Will Deacon <will.deacon@arm.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Stephen Rothwell <sfr@canb.auug.org.au>
-rw-r--r--include/linux/kasan.h21
-rw-r--r--mm/kasan/common.c11
-rw-r--r--mm/slab_common.c3
3 files changed, 32 insertions, 3 deletions
diff --git a/include/linux/kasan.h b/include/linux/kasan.h
index 6f0c5d9aa43f..5e0655fb2a6f 100644
--- a/include/linux/kasan.h
+++ b/include/linux/kasan.h
@@ -82,17 +82,30 @@ struct kasan_cache {
};
#ifdef CONFIG_KASAN_HW_TAGS
+
DECLARE_STATIC_KEY_FALSE(kasan_flag_enabled);
+
static __always_inline bool kasan_enabled(void)
{
return static_branch_likely(&kasan_flag_enabled);
}
-#else
+
+#else /* CONFIG_KASAN_HW_TAGS */
+
static inline bool kasan_enabled(void)
{
return true;
}
-#endif
+
+#endif /* CONFIG_KASAN_HW_TAGS */
+
+slab_flags_t __kasan_never_merge(void);
+static __always_inline slab_flags_t kasan_never_merge(void)
+{
+ if (kasan_enabled())
+ return __kasan_never_merge();
+ return 0;
+}
void __kasan_unpoison_range(const void *addr, size_t size);
static __always_inline void kasan_unpoison_range(const void *addr, size_t size)
@@ -239,6 +252,10 @@ static inline bool kasan_enabled(void)
{
return false;
}
+static inline slab_flags_t kasan_never_merge(void)
+{
+ return 0;
+}
static inline void kasan_unpoison_range(const void *address, size_t size) {}
static inline void kasan_alloc_pages(struct page *page, unsigned int order) {}
static inline void kasan_free_pages(struct page *page, unsigned int order) {}
diff --git a/mm/kasan/common.c b/mm/kasan/common.c
index 249ccba1ecf5..da79f340f3a6 100644
--- a/mm/kasan/common.c
+++ b/mm/kasan/common.c
@@ -87,6 +87,17 @@ asmlinkage void kasan_unpoison_task_stack_below(const void *watermark)
}
#endif /* CONFIG_KASAN_STACK */
+/*
+ * Only allow cache merging when stack collection is disabled and no metadata
+ * is present.
+ */
+slab_flags_t __kasan_never_merge(void)
+{
+ if (kasan_stack_collection_enabled())
+ return SLAB_KASAN;
+ return 0;
+}
+
void __kasan_alloc_pages(struct page *page, unsigned int order)
{
u8 tag;
diff --git a/mm/slab_common.c b/mm/slab_common.c
index 0b5ae1819a8b..075b23ce94ec 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -19,6 +19,7 @@
#include <linux/seq_file.h>
#include <linux/proc_fs.h>
#include <linux/debugfs.h>
+#include <linux/kasan.h>
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>
#include <asm/page.h>
@@ -54,7 +55,7 @@ static DECLARE_WORK(slab_caches_to_rcu_destroy_work,
*/
#define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
SLAB_TRACE | SLAB_TYPESAFE_BY_RCU | SLAB_NOLEAKTRACE | \
- SLAB_FAILSLAB | SLAB_KASAN)
+ SLAB_FAILSLAB | kasan_never_merge())
#define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \
SLAB_CACHE_DMA32 | SLAB_ACCOUNT)