diff options
Diffstat (limited to 'driver')
35 files changed, 1210 insertions, 494 deletions
diff --git a/driver/Makefile b/driver/Makefile index d22d29d..3af8b8d 100644 --- a/driver/Makefile +++ b/driver/Makefile @@ -58,5 +58,6 @@ all: clean: rm -f *.o .*.cmd gator_events.h modules.order Module.symvers gator.ko gator.mod.c + rm -rf .tmp_versions endif diff --git a/driver/gator.h b/driver/gator.h index 9a4617b..205cbcd 100644 --- a/driver/gator.h +++ b/driver/gator.h @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * Copyright (C) ARM Limited 2010-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -18,6 +18,9 @@ #define GATOR_PERF_PMU_SUPPORT GATOR_PERF_SUPPORT && defined(CONFIG_PERF_EVENTS) && (!(defined(__arm__) || defined(__aarch64__)) || defined(CONFIG_HW_PERF_EVENTS)) #define GATOR_NO_PERF_SUPPORT (!(GATOR_PERF_SUPPORT)) #define GATOR_CPU_FREQ_SUPPORT (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38)) && defined(CONFIG_CPU_FREQ) +#define GATOR_IKS_SUPPORT defined(CONFIG_BL_SWITCHER) + +#define GATOR_LIVE 1 // cpu ids #define ARM1136 0xb36 @@ -42,14 +45,15 @@ #define MAXSIZE_CORE_NAME 32 struct gator_cpu { - const int cpuid; - const char core_name[MAXSIZE_CORE_NAME]; - const char * const pmnc_name; - const int pmnc_counters; - const int ccnt; + const int cpuid; + const char core_name[MAXSIZE_CORE_NAME]; + const char * const pmu_name; + const char * const pmnc_name; + const int pmnc_counters; }; -extern struct gator_cpu gator_cpus[]; +const struct gator_cpu *gator_find_cpu_by_cpuid(const u32 cpuid); +const struct gator_cpu *gator_find_cpu_by_pmu_name(const char *const name); /****************************************************************************** * Filesystem @@ -98,10 +102,10 @@ struct gator_interface { int (*create_files)(struct super_block *sb, struct dentry *root); int (*start)(void); void (*stop)(void); // Complementary function to start - int (*online)(int **buffer); - int (*offline)(int **buffer); - void (*online_dispatch)(int cpu); // called in process context but may not be running on core 'cpu' - void (*offline_dispatch)(int cpu); // called in process context but may not be running on core 'cpu' + int (*online)(int **buffer, bool migrate); + int (*offline)(int **buffer, bool migrate); + void (*online_dispatch)(int cpu, bool migrate); // called in process context but may not be running on core 'cpu' + void (*offline_dispatch)(int cpu, bool migrate); // called in process context but may not be running on core 'cpu' int (*read)(int **buffer); int (*read64)(long long **buffer); struct list_head list; @@ -118,4 +122,21 @@ u32 gator_cpuid(void); void gator_backtrace_handler(struct pt_regs *const regs); +#if !GATOR_IKS_SUPPORT + +#define get_physical_cpu() smp_processor_id() +#define lcpu_to_pcpu(lcpu) lcpu +#define pcpu_to_lcpu(pcpu) pcpu + +#else + +#define get_physical_cpu() lcpu_to_pcpu(get_logical_cpu()) +int lcpu_to_pcpu(const int lcpu); +int pcpu_to_lcpu(const int pcpu); + +#endif + +#define get_logical_cpu() smp_processor_id() +#define on_primary_core() (get_logical_cpu() == 0) + #endif // GATOR_H_ diff --git a/driver/gator_annotate.c b/driver/gator_annotate.c index 42f9951..ad9f309 100644 --- a/driver/gator_annotate.c +++ b/driver/gator_annotate.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * Copyright (C) ARM Limited 2010-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -50,6 +50,7 @@ static ssize_t annotate_write(struct file *file, char const __user *buf, size_t return -EINVAL; } + retry: // synchronize between cores and with collect_annotations spin_lock(&annotate_lock); @@ -74,17 +75,18 @@ static ssize_t annotate_write(struct file *file, char const __user *buf, size_t size = count < available ? count : available; if (size <= 0) { - // Buffer is full but don't return an error. Instead return 0 so the - // caller knows nothing was written and they can try again. - size = 0; - goto annotate_write_out; + // Buffer is full, wait until space is available + spin_unlock(&annotate_lock); + wait_event_interruptible(gator_annotate_wait, buffer_bytes_available(cpu, ANNOTATE_BUF) > header_size || !collect_annotations); + goto retry; } // synchronize shared variables annotateBuf and annotatePos if (per_cpu(gator_buffer, cpu)[ANNOTATE_BUF]) { - gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, smp_processor_id()); + u64 time = gator_get_time(); + gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, get_physical_cpu()); gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, pid); - gator_buffer_write_packed_int64(cpu, ANNOTATE_BUF, gator_get_time()); + gator_buffer_write_packed_int64(cpu, ANNOTATE_BUF, time); gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, size); // determine the sizes to capture, length1 + length2 will equal size @@ -108,7 +110,7 @@ static ssize_t annotate_write(struct file *file, char const __user *buf, size_t } // Check and commit; commit is set to occur once buffer is 3/4 full - buffer_check(cpu, ANNOTATE_BUF); + buffer_check(cpu, ANNOTATE_BUF, time); } annotate_write_out: @@ -129,14 +131,14 @@ static int annotate_release(struct inode *inode, struct file *file) if (per_cpu(gator_buffer, cpu)[ANNOTATE_BUF] && buffer_check_space(cpu, ANNOTATE_BUF, MAXSIZE_PACK64 + 3 * MAXSIZE_PACK32)) { uint32_t pid = current->pid; - gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, smp_processor_id()); + gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, get_physical_cpu()); gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, pid); gator_buffer_write_packed_int64(cpu, ANNOTATE_BUF, 0); // time gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, 0); // size } // Check and commit; commit is set to occur once buffer is 3/4 full - buffer_check(cpu, ANNOTATE_BUF); + buffer_check(cpu, ANNOTATE_BUF, gator_get_time()); spin_unlock(&annotate_lock); @@ -164,5 +166,6 @@ static void gator_annotate_stop(void) // the spinlock here will ensure that when this function exits, we are not in the middle of an annotation spin_lock(&annotate_lock); collect_annotations = false; + wake_up(&gator_annotate_wait); spin_unlock(&annotate_lock); } diff --git a/driver/gator_annotate_kernel.c b/driver/gator_annotate_kernel.c index 67d2d6c..4715f64 100644 --- a/driver/gator_annotate_kernel.c +++ b/driver/gator_annotate_kernel.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2012. All rights reserved. + * Copyright (C) ARM Limited 2012-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as diff --git a/driver/gator_backtrace.c b/driver/gator_backtrace.c index e6125b3..94f01e6 100644 --- a/driver/gator_backtrace.c +++ b/driver/gator_backtrace.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * Copyright (C) ARM Limited 2010-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -11,8 +11,17 @@ * EABI backtrace stores {fp,lr} on the stack. */ struct frame_tail_eabi { - unsigned long fp; // points to prev_lr - unsigned long lr; + union { + struct { + unsigned long fp; // points to prev_lr + unsigned long lr; + }; + // Used to read 32 bit fp/lr from a 64 bit kernel + struct { + u32 fp_32; + u32 lr_32; + }; + }; }; static void arm_backtrace_eabi(int cpu, struct pt_regs *const regs, unsigned int depth) @@ -20,18 +29,20 @@ static void arm_backtrace_eabi(int cpu, struct pt_regs *const regs, unsigned int #if defined(__arm__) || defined(__aarch64__) struct frame_tail_eabi *tail; struct frame_tail_eabi *next; - struct frame_tail_eabi *ptrtail; struct frame_tail_eabi buftail; #if defined(__arm__) + const bool is_compat = false; unsigned long fp = regs->ARM_fp; unsigned long sp = regs->ARM_sp; unsigned long lr = regs->ARM_lr; const int frame_offset = 4; #else - unsigned long fp = regs->regs[29]; - unsigned long sp = regs->sp; - unsigned long lr = regs->regs[30]; - const int frame_offset = 0; + // Is userspace aarch32 (32 bit) + const bool is_compat = compat_user_mode(regs); + unsigned long fp = (is_compat ? regs->regs[11] : regs->regs[29]); + unsigned long sp = (is_compat ? regs->compat_sp : regs->sp); + unsigned long lr = (is_compat ? regs->compat_lr : regs->regs[30]); + const int frame_offset = (is_compat ? 4 : 0); #endif int is_user_mode = user_mode(regs); @@ -55,15 +66,14 @@ static void arm_backtrace_eabi(int cpu, struct pt_regs *const regs, unsigned int return; if (__copy_from_user_inatomic(&buftail, tail, sizeof(struct frame_tail_eabi))) return; - ptrtail = &buftail; - lr = ptrtail[0].lr; + lr = (is_compat ? buftail.lr_32 : buftail.lr); gator_add_trace(cpu, lr); /* frame pointers should progress back up the stack, towards higher addresses */ next = (struct frame_tail_eabi *)(lr - frame_offset); if (tail >= next || lr == 0) { - fp = ptrtail[0].fp; + fp = (is_compat ? buftail.fp_32 : buftail.fp); next = (struct frame_tail_eabi *)(fp - frame_offset); /* check tail is valid */ if (tail >= next || fp == 0) { @@ -79,16 +89,17 @@ static void arm_backtrace_eabi(int cpu, struct pt_regs *const regs, unsigned int #if defined(__arm__) || defined(__aarch64__) static int report_trace(struct stackframe *frame, void *d) { - struct module *mod; - unsigned int *depth = d, cookie = NO_COOKIE, cpu = smp_processor_id(); + unsigned int *depth = d, cookie = NO_COOKIE, cpu = get_physical_cpu(); unsigned long addr = frame->pc; if (*depth) { - mod = __module_address(addr); +#if defined(MODULE) + struct module *mod = __module_address(addr); if (mod) { cookie = get_cookie(cpu, current, mod->name, false); addr = addr - (unsigned long)mod->module_core; } +#endif marshal_backtrace(addr & ~1, cookie); (*depth)--; } diff --git a/driver/gator_cookies.c b/driver/gator_cookies.c index bb401bb..c332187 100644 --- a/driver/gator_cookies.c +++ b/driver/gator_cookies.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * Copyright (C) ARM Limited 2010-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -61,7 +61,7 @@ static uint32_t gator_chksum_crc32(const char *data) static uint32_t cookiemap_exists(uint64_t key) { unsigned long x, flags, retval = 0; - int cpu = smp_processor_id(); + int cpu = get_physical_cpu(); uint32_t cookiecode = cookiemap_code(key); uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]); uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]); @@ -93,7 +93,7 @@ static uint32_t cookiemap_exists(uint64_t key) */ static void cookiemap_add(uint64_t key, uint32_t value) { - int cpu = smp_processor_id(); + int cpu = get_physical_cpu(); int cookiecode = cookiemap_code(key); uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]); uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]); @@ -124,7 +124,7 @@ static void wq_cookie_handler(struct work_struct *unused) { struct task_struct *task; char *text; - int cpu = smp_processor_id(); + int cpu = get_physical_cpu(); unsigned int commit; mutex_lock(&start_mutex); diff --git a/driver/gator_events_armv6.c b/driver/gator_events_armv6.c index ee36dd0..4f1bca6 100644 --- a/driver/gator_events_armv6.c +++ b/driver/gator_events_armv6.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * Copyright (C) ARM Limited 2010-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -93,7 +93,7 @@ int gator_events_armv6_create_files(struct super_block *sb, struct dentry *root) return 0; } -static int gator_events_armv6_online(int **buffer) +static int gator_events_armv6_online(int **buffer, bool migrate) { unsigned int cnt, len = 0, cpu = smp_processor_id(); u32 pmnc; @@ -141,7 +141,7 @@ static int gator_events_armv6_online(int **buffer) return len; } -static int gator_events_armv6_offline(int **buffer) +static int gator_events_armv6_offline(int **buffer, bool migrate) { unsigned int cnt; diff --git a/driver/gator_events_armv7.c b/driver/gator_events_armv7.c index 212b17b..58f2956 100644 --- a/driver/gator_events_armv7.c +++ b/driver/gator_events_armv7.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * Copyright (C) ARM Limited 2010-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -159,7 +159,7 @@ static int gator_events_armv7_create_files(struct super_block *sb, struct dentry return 0; } -static int gator_events_armv7_online(int **buffer) +static int gator_events_armv7_online(int **buffer, bool migrate) { unsigned int cnt, len = 0, cpu = smp_processor_id(); @@ -214,7 +214,7 @@ static int gator_events_armv7_online(int **buffer) return len; } -static int gator_events_armv7_offline(int **buffer) +static int gator_events_armv7_offline(int **buffer, bool migrate) { // disable all counters, including PMCCNTR; overflow IRQs will not be signaled armv7_pmnc_write(armv7_pmnc_read() & ~PMNC_E); diff --git a/driver/gator_events_block.c b/driver/gator_events_block.c index f512b13..56c6a67 100644 --- a/driver/gator_events_block.c +++ b/driver/gator_events_block.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * Copyright (C) ARM Limited 2010-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -30,7 +30,6 @@ static int blockGet[BLOCK_TOTAL * 4]; GATOR_DEFINE_PROBE(block_rq_complete, TP_PROTO(struct request_queue *q, struct request *rq)) { - unsigned long flags; int write, size; if (!rq) @@ -42,9 +41,6 @@ GATOR_DEFINE_PROBE(block_rq_complete, TP_PROTO(struct request_queue *q, struct r if (!size) return; - // disable interrupts to synchronize with gator_events_block_read() - // spinlocks not needed since percpu buffers are used - local_irq_save(flags); if (write) { if (block_rq_wr_enabled) { atomic_add(size, &blockCnt[BLOCK_RQ_WR]); @@ -54,7 +50,6 @@ GATOR_DEFINE_PROBE(block_rq_complete, TP_PROTO(struct request_queue *q, struct r atomic_add(size, &blockCnt[BLOCK_RQ_RD]); } } - local_irq_restore(flags); } static int gator_events_block_create_files(struct super_block *sb, struct dentry *root) @@ -111,7 +106,7 @@ static int gator_events_block_read(int **buffer) { int len, value, data = 0; - if (smp_processor_id() != 0) { + if (!on_primary_core()) { return 0; } diff --git a/driver/gator_events_irq.c b/driver/gator_events_irq.c index 1221372..b4df7fa 100644 --- a/driver/gator_events_irq.c +++ b/driver/gator_events_irq.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * Copyright (C) ARM Limited 2010-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -18,19 +18,13 @@ static ulong hardirq_enabled; static ulong softirq_enabled; static ulong hardirq_key; static ulong softirq_key; -static DEFINE_PER_CPU(int[TOTALIRQ], irqCnt); +static DEFINE_PER_CPU(atomic_t[TOTALIRQ], irqCnt); static DEFINE_PER_CPU(int[TOTALIRQ * 2], irqGet); GATOR_DEFINE_PROBE(irq_handler_exit, TP_PROTO(int irq, struct irqaction *action, int ret)) { - unsigned long flags; - - // disable interrupts to synchronize with gator_events_irq_read() - // spinlocks not needed since percpu buffers are used - local_irq_save(flags); - per_cpu(irqCnt, smp_processor_id())[HARDIRQ]++; - local_irq_restore(flags); + atomic_inc(&per_cpu(irqCnt, get_physical_cpu())[HARDIRQ]); } #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 37) @@ -39,13 +33,7 @@ GATOR_DEFINE_PROBE(softirq_exit, TP_PROTO(struct softirq_action *h, struct softi GATOR_DEFINE_PROBE(softirq_exit, TP_PROTO(unsigned int vec_nr)) #endif { - unsigned long flags; - - // disable interrupts to synchronize with gator_events_irq_read() - // spinlocks not needed since percpu buffers are used - local_irq_save(flags); - per_cpu(irqCnt, smp_processor_id())[SOFTIRQ]++; - local_irq_restore(flags); + atomic_inc(&per_cpu(irqCnt, get_physical_cpu())[SOFTIRQ]); } static int gator_events_irq_create_files(struct super_block *sb, struct dentry *root) @@ -71,24 +59,19 @@ static int gator_events_irq_create_files(struct super_block *sb, struct dentry * return 0; } -static int gator_events_irq_online(int **buffer) +static int gator_events_irq_online(int **buffer, bool migrate) { - int len = 0, cpu = smp_processor_id(); - unsigned long flags; // not necessary as we are in interrupt context anyway, but doesn't hurt + int len = 0, cpu = get_physical_cpu(); // synchronization with the irq_exit functions is not necessary as the values are being reset if (hardirq_enabled) { - local_irq_save(flags); - per_cpu(irqCnt, cpu)[HARDIRQ] = 0; - local_irq_restore(flags); + atomic_set(&per_cpu(irqCnt, cpu)[HARDIRQ], 0); per_cpu(irqGet, cpu)[len++] = hardirq_key; per_cpu(irqGet, cpu)[len++] = 0; } if (softirq_enabled) { - local_irq_save(flags); - per_cpu(irqCnt, cpu)[SOFTIRQ] = 0; - local_irq_restore(flags); + atomic_set(&per_cpu(irqCnt, cpu)[SOFTIRQ], 0); per_cpu(irqGet, cpu)[len++] = softirq_key; per_cpu(irqGet, cpu)[len++] = 0; } @@ -136,26 +119,21 @@ static void gator_events_irq_stop(void) static int gator_events_irq_read(int **buffer) { - unsigned long flags; // not necessary as we are in interrupt context anyway, but doesn't hurt int len, value; - int cpu = smp_processor_id(); + int cpu = get_physical_cpu(); len = 0; if (hardirq_enabled) { - local_irq_save(flags); - value = per_cpu(irqCnt, cpu)[HARDIRQ]; - per_cpu(irqCnt, cpu)[HARDIRQ] = 0; - local_irq_restore(flags); + value = atomic_read(&per_cpu(irqCnt, cpu)[HARDIRQ]); + atomic_sub(value, &per_cpu(irqCnt, cpu)[HARDIRQ]); per_cpu(irqGet, cpu)[len++] = hardirq_key; per_cpu(irqGet, cpu)[len++] = value; } if (softirq_enabled) { - local_irq_save(flags); - value = per_cpu(irqCnt, cpu)[SOFTIRQ]; - per_cpu(irqCnt, cpu)[SOFTIRQ] = 0; - local_irq_restore(flags); + value = atomic_read(&per_cpu(irqCnt, cpu)[SOFTIRQ]); + atomic_sub(value, &per_cpu(irqCnt, cpu)[SOFTIRQ]); per_cpu(irqGet, cpu)[len++] = softirq_key; per_cpu(irqGet, cpu)[len++] = value; diff --git a/driver/gator_events_l2c-310.c b/driver/gator_events_l2c-310.c index 197af04..52472c7 100644 --- a/driver/gator_events_l2c-310.c +++ b/driver/gator_events_l2c-310.c @@ -1,7 +1,7 @@ /** * l2c310 (L2 Cache Controller) event counters for gator * - * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * Copyright (C) ARM Limited 2010-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -10,6 +10,7 @@ #include <linux/init.h> #include <linux/io.h> +#include <linux/module.h> #include <asm/hardware/cache-l2x0.h> #include "gator.h" @@ -95,7 +96,7 @@ static int gator_events_l2c310_read(int **buffer) int i; int len = 0; - if (smp_processor_id()) + if (!on_primary_core()) return 0; for (i = 0; i < L2C310_COUNTERS_NUM; i++) { @@ -122,20 +123,48 @@ static struct gator_interface gator_events_l2c310_interface = { .read = gator_events_l2c310_read, }; -static void __maybe_unused gator_events_l2c310_probe(unsigned long phys) +#define L2C310_ADDR_PROBE (~0) + +MODULE_PARM_DESC(l2c310_addr, "L2C310 physical base address (0 to disable)"); +static unsigned long l2c310_addr = L2C310_ADDR_PROBE; +module_param(l2c310_addr, ulong, 0444); + +static void __iomem *gator_events_l2c310_probe(void) { - if (l2c310_base) - return; + phys_addr_t variants[] = { +#if defined(CONFIG_ARCH_EXYNOS4) || defined(CONFIG_ARCH_S5PV310) + 0x10502000, +#endif +#if defined(CONFIG_ARCH_OMAP4) + 0x48242000, +#endif +#if defined(CONFIG_ARCH_TEGRA) + 0x50043000, +#endif +#if defined(CONFIG_ARCH_U8500) + 0xa0412000, +#endif +#if defined(CONFIG_ARCH_VEXPRESS) + 0x1e00a000, // A9x4 core tile (HBI-0191) + 0x2c0f0000, // New memory map tiles +#endif + }; + int i; - l2c310_base = ioremap(phys, SZ_4K); - if (l2c310_base) { - u32 cache_id = readl(l2c310_base + L2X0_CACHE_ID); + for (i = 0; i < ARRAY_SIZE(variants); i++) { + void __iomem *base = ioremap(variants[i], SZ_4K); - if ((cache_id & 0xff0003c0) != 0x410000c0) { - iounmap(l2c310_base); - l2c310_base = NULL; + if (base) { + u32 cache_id = readl(base + L2X0_CACHE_ID); + + if ((cache_id & 0xff0003c0) == 0x410000c0) + return base; + + iounmap(base); } } + + return NULL; } int gator_events_l2c310_init(void) @@ -145,24 +174,11 @@ int gator_events_l2c310_init(void) if (gator_cpuid() != CORTEX_A5 && gator_cpuid() != CORTEX_A9) return -1; -#if defined(CONFIG_ARCH_EXYNOS4) || defined(CONFIG_ARCH_S5PV310) - gator_events_l2c310_probe(0x10502000); -#endif -#if defined(CONFIG_ARCH_OMAP4) - gator_events_l2c310_probe(0x48242000); -#endif -#if defined(CONFIG_ARCH_TEGRA) - gator_events_l2c310_probe(0x50043000); -#endif -#if defined(CONFIG_ARCH_U8500) - gator_events_l2c310_probe(0xa0412000); -#endif -#if defined(CONFIG_ARCH_VEXPRESS) - // A9x4 core tile (HBI-0191) - gator_events_l2c310_probe(0x1e00a000); - // New memory map tiles - gator_events_l2c310_probe(0x2c0f0000); -#endif + if (l2c310_addr == L2C310_ADDR_PROBE) + l2c310_base = gator_events_l2c310_probe(); + else if (l2c310_addr) + l2c310_base = ioremap(l2c310_addr, SZ_4K); + if (!l2c310_base) return -1; diff --git a/driver/gator_events_mali_400.c b/driver/gator_events_mali_400.c index 34a73c8..38c97d1 100644 --- a/driver/gator_events_mali_400.c +++ b/driver/gator_events_mali_400.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * Copyright (C) ARM Limited 2010-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -630,7 +630,7 @@ static int read(int **buffer) { int cnt, len = 0; - if (smp_processor_id()) + if (!on_primary_core()) return 0; // Read the L2 C0 and C1 here. diff --git a/driver/gator_events_mali_400.h b/driver/gator_events_mali_400.h index a09757e..43aec49 100644 --- a/driver/gator_events_mali_400.h +++ b/driver/gator_events_mali_400.h @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2011-2012. All rights reserved. + * Copyright (C) ARM Limited 2011-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as diff --git a/driver/gator_events_mali_common.c b/driver/gator_events_mali_common.c index 2186eee..22a517d 100644 --- a/driver/gator_events_mali_common.c +++ b/driver/gator_events_mali_common.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2012. All rights reserved. + * Copyright (C) ARM Limited 2012-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as diff --git a/driver/gator_events_mali_common.h b/driver/gator_events_mali_common.h index 8e33edf..27eaacc 100644 --- a/driver/gator_events_mali_common.h +++ b/driver/gator_events_mali_common.h @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2012. All rights reserved. + * Copyright (C) ARM Limited 2012-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as diff --git a/driver/gator_events_mali_t6xx.c b/driver/gator_events_mali_t6xx.c index 1b3a53d..2576a99 100644 --- a/driver/gator_events_mali_t6xx.c +++ b/driver/gator_events_mali_t6xx.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2011-2012. All rights reserved. + * Copyright (C) ARM Limited 2011-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -411,7 +411,7 @@ static int read(int **buffer) long sample_interval_us = 0; struct timespec read_timestamp; - if (smp_processor_id() != 0) { + if (!on_primary_core()) { return 0; } diff --git a/driver/gator_events_mali_t6xx_hw.c b/driver/gator_events_mali_t6xx_hw.c index 72498c8..fb2e15c 100644 --- a/driver/gator_events_mali_t6xx_hw.c +++ b/driver/gator_events_mali_t6xx_hw.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2012. All rights reserved. + * Copyright (C) ARM Limited 2012-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -608,7 +608,7 @@ static int read(int **buffer) static u32 prev_time_s = 0; static s32 next_read_time_ns = 0; - if (smp_processor_id() != 0) { + if (!on_primary_core()) { return 0; } diff --git a/driver/gator_events_mali_t6xx_hw_test.c b/driver/gator_events_mali_t6xx_hw_test.c index eb77110..efb32dd 100644 --- a/driver/gator_events_mali_t6xx_hw_test.c +++ b/driver/gator_events_mali_t6xx_hw_test.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2012. All rights reserved. + * Copyright (C) ARM Limited 2012-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as diff --git a/driver/gator_events_meminfo.c b/driver/gator_events_meminfo.c index fd063b2..c1e360d 100644 --- a/driver/gator_events_meminfo.c +++ b/driver/gator_events_meminfo.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * Copyright (C) ARM Limited 2010-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -198,7 +198,7 @@ static int gator_events_meminfo_read(long long **buffer) { static unsigned int last_mem_event = 0; - if (smp_processor_id() || !meminfo_global_enabled) + if (!on_primary_core() || !meminfo_global_enabled) return 0; if (last_mem_event != mem_event) { diff --git a/driver/gator_events_mmaped.c b/driver/gator_events_mmaped.c index c4cb44f..0027564 100644 --- a/driver/gator_events_mmaped.c +++ b/driver/gator_events_mmaped.c @@ -1,7 +1,7 @@ /* * Example events provider * - * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * Copyright (C) ARM Limited 2010-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -141,7 +141,7 @@ static int mmaped_simulate(int counter, int delta_in_us) break; case 2: /* PWM signal */ { - static int t, dc, x; + static int dc, x, t = 0; t += delta_in_us; if (t > 1000000) @@ -170,7 +170,7 @@ static int gator_events_mmaped_read(int **buffer) #endif /* System wide counters - read from one core only */ - if (smp_processor_id()) + if (!on_primary_core()) return 0; #ifndef TODO diff --git a/driver/gator_events_net.c b/driver/gator_events_net.c index b6ce06a..80cdee4 100644 --- a/driver/gator_events_net.c +++ b/driver/gator_events_net.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * Copyright (C) ARM Limited 2010-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -117,7 +117,7 @@ static int gator_events_net_read(int **buffer) int len, rx_delta, tx_delta; static int last_rx_delta = 0, last_tx_delta = 0; - if (smp_processor_id() != 0) + if (!on_primary_core()) return 0; if (!netrx_enabled && !nettx_enabled) diff --git a/driver/gator_events_perf_pmu.c b/driver/gator_events_perf_pmu.c index ce3a40f..34a6bc7 100644 --- a/driver/gator_events_perf_pmu.c +++ b/driver/gator_events_perf_pmu.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * Copyright (C) ARM Limited 2010-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -13,49 +13,74 @@ // gator_events_armvX.c is used for Linux 2.6.x #if GATOR_PERF_PMU_SUPPORT -static const char *pmnc_name; -int pmnc_counters; -int ccnt = 0; +extern bool event_based_sampling; -#define CNTMAX (6+1) +#define CNTMAX 16 +#define CCI_400 4 +// + 1 for the cci-400 cycles counter +#define UCCNT (CCI_400 + 1) -static DEFINE_MUTEX(perf_mutex); +struct gator_attr { + char name[40]; + unsigned long enabled; + unsigned long type; + unsigned long event; + unsigned long count; + unsigned long key; +}; -unsigned long pmnc_enabled[CNTMAX]; -unsigned long pmnc_event[CNTMAX]; -unsigned long pmnc_count[CNTMAX]; -unsigned long pmnc_key[CNTMAX]; +static struct gator_attr attrs[CNTMAX]; +static int attr_count; +static struct gator_attr uc_attrs[UCCNT]; +static int uc_attr_count; + +struct gator_event { + int curr; + int prev; + int prev_delta; + bool zero; + struct perf_event *pevent; + struct perf_event_attr *pevent_attr; +}; -static DEFINE_PER_CPU(int[CNTMAX], perfCurr); -static DEFINE_PER_CPU(int[CNTMAX], perfPrev); -static DEFINE_PER_CPU(int[CNTMAX], perfPrevDelta); -static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt); -static DEFINE_PER_CPU(struct perf_event *[CNTMAX], pevent); -static DEFINE_PER_CPU(struct perf_event_attr *[CNTMAX], pevent_attr); +static DEFINE_PER_CPU(struct gator_event[CNTMAX], events); +static struct gator_event uc_events[UCCNT]; +static DEFINE_PER_CPU(int[(CNTMAX + UCCNT)*2], perf_cnt); static void gator_events_perf_pmu_stop(void); -static int gator_events_perf_pmu_create_files(struct super_block *sb, struct dentry *root) +static int __create_files(struct super_block *sb, struct dentry *root, struct gator_attr *const attr) { struct dentry *dir; - int i; - for (i = 0; i < pmnc_counters; i++) { - char buf[40]; - if (i == 0) { - snprintf(buf, sizeof buf, "%s_ccnt", pmnc_name); - } else { - snprintf(buf, sizeof buf, "%s_cnt%d", pmnc_name, i - 1); - } - dir = gatorfs_mkdir(sb, root, buf); - if (!dir) { + if (attr->name[0] == '\0') { + return 0; + } + dir = gatorfs_mkdir(sb, root, attr->name); + if (!dir) { + return -1; + } + gatorfs_create_ulong(sb, dir, "enabled", &attr->enabled); + gatorfs_create_ulong(sb, dir, "count", &attr->count); + gatorfs_create_ro_ulong(sb, dir, "key", &attr->key); + gatorfs_create_ulong(sb, dir, "event", &attr->event); + + return 0; +} + +static int gator_events_perf_pmu_create_files(struct super_block *sb, struct dentry *root) +{ + int cnt; + + for (cnt = 0; cnt < attr_count; cnt++) { + if (__create_files(sb, root, &attrs[cnt]) != 0) { return -1; } - gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); - gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]); - gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); - if (i > 0) { - gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); + } + + for (cnt = 0; cnt < uc_attr_count; cnt++) { + if (__create_files(sb, root, &uc_attrs[cnt]) != 0) { + return -1; } } @@ -80,177 +105,268 @@ static void dummy_handler(struct perf_event *event, struct perf_sample_data *dat // Required as perf_event_create_kernel_counter() requires an overflow handler, even though all we do is poll } -static int gator_events_perf_pmu_online(int **buffer) +static int gator_events_perf_pmu_read(int **buffer); + +static int gator_events_perf_pmu_online(int **buffer, bool migrate) { - int cnt, len = 0, cpu = smp_processor_id(); + return gator_events_perf_pmu_read(buffer); +} - // read the counters and toss the invalid data, return zero instead - for (cnt = 0; cnt < pmnc_counters; cnt++) { - struct perf_event *ev = per_cpu(pevent, cpu)[cnt]; - if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) { - ev->pmu->read(ev); - per_cpu(perfPrev, cpu)[cnt] = per_cpu(perfCurr, cpu)[cnt] = local64_read(&ev->count); - per_cpu(perfPrevDelta, cpu)[cnt] = 0; - per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; - per_cpu(perfCnt, cpu)[len++] = 0; - } +static void __online_dispatch(int cpu, bool migrate, struct gator_attr *const attr, struct gator_event *const event) +{ + perf_overflow_handler_t handler; + + event->zero = true; + + if (event->pevent != NULL || event->pevent_attr == 0 || migrate) { + return; } - if (buffer) - *buffer = per_cpu(perfCnt, cpu); + if (attr->count > 0) { + handler = ebs_overflow_handler; + } else { + handler = dummy_handler; + } - return len; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0) + event->pevent = perf_event_create_kernel_counter(event->pevent_attr, cpu, 0, handler); +#else + event->pevent = perf_event_create_kernel_counter(event->pevent_attr, cpu, 0, handler, 0); +#endif + if (IS_ERR(event->pevent)) { + pr_debug("gator: unable to online a counter on cpu %d\n", cpu); + event->pevent = NULL; + return; + } + + if (event->pevent->state != PERF_EVENT_STATE_ACTIVE) { + pr_debug("gator: inactive counter on cpu %d\n", cpu); + perf_event_release_kernel(event->pevent); + event->pevent = NULL; + return; + } } -static void gator_events_perf_pmu_online_dispatch(int cpu) +static void gator_events_perf_pmu_online_dispatch(int cpu, bool migrate) { int cnt; - perf_overflow_handler_t handler; - for (cnt = 0; cnt < pmnc_counters; cnt++) { - if (per_cpu(pevent, cpu)[cnt] != NULL || per_cpu(pevent_attr, cpu)[cnt] == 0) - continue; + cpu = pcpu_to_lcpu(cpu); - if (pmnc_count[cnt] > 0) { - handler = ebs_overflow_handler; - } else { - handler = dummy_handler; - } + for (cnt = 0; cnt < attr_count; cnt++) { + __online_dispatch(cpu, migrate, &attrs[cnt], &per_cpu(events, cpu)[cnt]); + } -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0) - per_cpu(pevent, cpu)[cnt] = perf_event_create_kernel_counter(per_cpu(pevent_attr, cpu)[cnt], cpu, 0, handler); -#else - per_cpu(pevent, cpu)[cnt] = perf_event_create_kernel_counter(per_cpu(pevent_attr, cpu)[cnt], cpu, 0, handler, 0); -#endif - if (IS_ERR(per_cpu(pevent, cpu)[cnt])) { - pr_debug("gator: unable to online a counter on cpu %d\n", cpu); - per_cpu(pevent, cpu)[cnt] = NULL; - continue; + if (cpu == 0) { + for (cnt = 0; cnt < uc_attr_count; cnt++) { + __online_dispatch(cpu, migrate, &uc_attrs[cnt], &uc_events[cnt]); } + } +} - if (per_cpu(pevent, cpu)[cnt]->state != PERF_EVENT_STATE_ACTIVE) { - pr_debug("gator: inactive counter on cpu %d\n", cpu); - perf_event_release_kernel(per_cpu(pevent, cpu)[cnt]); - per_cpu(pevent, cpu)[cnt] = NULL; - continue; - } +static void __offline_dispatch(int cpu, struct gator_event *const event) +{ + struct perf_event *pe = NULL; + + if (event->pevent) { + pe = event->pevent; + event->pevent = NULL; + } + + if (pe) { + perf_event_release_kernel(pe); } } -static void gator_events_perf_pmu_offline_dispatch(int cpu) +static void gator_events_perf_pmu_offline_dispatch(int cpu, bool migrate) { int cnt; - struct perf_event *pe; - for (cnt = 0; cnt < pmnc_counters; cnt++) { - pe = NULL; - mutex_lock(&perf_mutex); - if (per_cpu(pevent, cpu)[cnt]) { - pe = per_cpu(pevent, cpu)[cnt]; - per_cpu(pevent, cpu)[cnt] = NULL; + if (migrate) { + return; + } + cpu = pcpu_to_lcpu(cpu); + + for (cnt = 0; cnt < attr_count; cnt++) { + __offline_dispatch(cpu, &per_cpu(events, cpu)[cnt]); + } + + if (cpu == 0) { + for (cnt = 0; cnt < uc_attr_count; cnt++) { + __offline_dispatch(cpu, &uc_events[cnt]); } - mutex_unlock(&perf_mutex); + } +} - if (pe) { - perf_event_release_kernel(pe); +static int __check_ebs(struct gator_attr *const attr) +{ + if (attr->count > 0) { + if (!event_based_sampling) { + event_based_sampling = true; + } else { + printk(KERN_WARNING "gator: Only one ebs counter is allowed\n"); + return -1; } } + + return 0; +} + +static int __start(struct gator_attr *const attr, struct gator_event *const event) +{ + u32 size = sizeof(struct perf_event_attr); + + event->pevent = NULL; + if (!attr->enabled) { // Skip disabled counters + return 0; + } + + event->prev = 0; + event->curr = 0; + event->prev_delta = 0; + event->pevent_attr = kmalloc(size, GFP_KERNEL); + if (!event->pevent_attr) { + gator_events_perf_pmu_stop(); + return -1; + } + + memset(event->pevent_attr, 0, size); + event->pevent_attr->type = attr->type; + event->pevent_attr->size = size; + event->pevent_attr->config = attr->event; + event->pevent_attr->sample_period = attr->count; + event->pevent_attr->pinned = 1; + + return 0; } static int gator_events_perf_pmu_start(void) { int cnt, cpu; - u32 size = sizeof(struct perf_event_attr); - int found_ebs = false; - - for (cnt = 0; cnt < pmnc_counters; cnt++) { - if (pmnc_count[cnt] > 0) { - if (!found_ebs) { - found_ebs = true; - } else { - // Only one ebs counter is allowed - return -1; - } + + event_based_sampling = false; + for (cnt = 0; cnt < attr_count; cnt++) { + if (__check_ebs(&attrs[cnt]) != 0) { + return -1; + } + } + + for (cnt = 0; cnt < uc_attr_count; cnt++) { + if (__check_ebs(&uc_attrs[cnt]) != 0) { + return -1; } } for_each_present_cpu(cpu) { - for (cnt = 0; cnt < pmnc_counters; cnt++) { - per_cpu(pevent, cpu)[cnt] = NULL; - if (!pmnc_enabled[cnt]) // Skip disabled counters - continue; - - per_cpu(perfPrev, cpu)[cnt] = 0; - per_cpu(perfCurr, cpu)[cnt] = 0; - per_cpu(perfPrevDelta, cpu)[cnt] = 0; - per_cpu(pevent_attr, cpu)[cnt] = kmalloc(size, GFP_KERNEL); - if (!per_cpu(pevent_attr, cpu)[cnt]) { - gator_events_perf_pmu_stop(); + for (cnt = 0; cnt < attr_count; cnt++) { + if (__start(&attrs[cnt], &per_cpu(events, cpu)[cnt]) != 0) { return -1; } + } + } - memset(per_cpu(pevent_attr, cpu)[cnt], 0, size); - per_cpu(pevent_attr, cpu)[cnt]->type = PERF_TYPE_RAW; - per_cpu(pevent_attr, cpu)[cnt]->size = size; - per_cpu(pevent_attr, cpu)[cnt]->config = pmnc_event[cnt]; - per_cpu(pevent_attr, cpu)[cnt]->sample_period = pmnc_count[cnt]; - per_cpu(pevent_attr, cpu)[cnt]->pinned = 1; - - // handle special case for ccnt - if (cnt == ccnt) { - per_cpu(pevent_attr, cpu)[cnt]->type = PERF_TYPE_HARDWARE; - per_cpu(pevent_attr, cpu)[cnt]->config = PERF_COUNT_HW_CPU_CYCLES; - } + for (cnt = 0; cnt < uc_attr_count; cnt++) { + if (__start(&uc_attrs[cnt], &uc_events[cnt]) != 0) { + return -1; } } return 0; } +static void __event_stop(struct gator_event *const event) +{ + if (event->pevent_attr) { + kfree(event->pevent_attr); + event->pevent_attr = NULL; + } +} + +static void __attr_stop(struct gator_attr *const attr) +{ + attr->enabled = 0; + attr->event = 0; + attr->count = 0; +} + static void gator_events_perf_pmu_stop(void) { unsigned int cnt, cpu; for_each_present_cpu(cpu) { - for (cnt = 0; cnt < pmnc_counters; cnt++) { - if (per_cpu(pevent_attr, cpu)[cnt]) { - kfree(per_cpu(pevent_attr, cpu)[cnt]); - per_cpu(pevent_attr, cpu)[cnt] = NULL; - } + for (cnt = 0; cnt < attr_count; cnt++) { + __event_stop(&per_cpu(events, cpu)[cnt]); } } - for (cnt = 0; cnt < pmnc_counters; cnt++) { - pmnc_enabled[cnt] = 0; - pmnc_event[cnt] = 0; - pmnc_count[cnt] = 0; + for (cnt = 0; cnt < uc_attr_count; cnt++) { + __event_stop(&uc_events[cnt]); + } + + for (cnt = 0; cnt < attr_count; cnt++) { + __attr_stop(&attrs[cnt]); + } + + for (cnt = 0; cnt < uc_attr_count; cnt++) { + __attr_stop(&uc_attrs[cnt]); } } -static int gator_events_perf_pmu_read(int **buffer) +static void __read(int *const len, int cpu, struct gator_attr *const attr, struct gator_event *const event) { - int cnt, delta, len = 0; - int cpu = smp_processor_id(); - - for (cnt = 0; cnt < pmnc_counters; cnt++) { - struct perf_event *ev = per_cpu(pevent, cpu)[cnt]; - if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) { + int delta; + + struct perf_event *const ev = event->pevent; + if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) { + /* After creating the perf counter in __online_dispatch, there + * is a race condition between gator_events_perf_pmu_online and + * gator_events_perf_pmu_read. So have + * gator_events_perf_pmu_online call gator_events_perf_pmu_read + * and in __read check to see if it's the first call after + * __online_dispatch and if so, run the online code. + */ + if (event->zero) { + ev->pmu->read(ev); + event->prev = event->curr = local64_read(&ev->count); + event->prev_delta = 0; + per_cpu(perf_cnt, cpu)[(*len)++] = attr->key; + per_cpu(perf_cnt, cpu)[(*len)++] = 0; + event->zero = false; + } else { ev->pmu->read(ev); - per_cpu(perfCurr, cpu)[cnt] = local64_read(&ev->count); - delta = per_cpu(perfCurr, cpu)[cnt] - per_cpu(perfPrev, cpu)[cnt]; - if (delta != 0 || delta != per_cpu(perfPrevDelta, cpu)[cnt]) { - per_cpu(perfPrevDelta, cpu)[cnt] = delta; - per_cpu(perfPrev, cpu)[cnt] = per_cpu(perfCurr, cpu)[cnt]; - per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; - if (delta < 0) + event->curr = local64_read(&ev->count); + delta = event->curr - event->prev; + if (delta != 0 || delta != event->prev_delta) { + event->prev_delta = delta; + event->prev = event->curr; + per_cpu(perf_cnt, cpu)[(*len)++] = attr->key; + if (delta < 0) { delta *= -1; - per_cpu(perfCnt, cpu)[len++] = delta; + } + per_cpu(perf_cnt, cpu)[(*len)++] = delta; } } } +} - if (buffer) - *buffer = per_cpu(perfCnt, cpu); +static int gator_events_perf_pmu_read(int **buffer) +{ + int cnt, len = 0; + const int cpu = get_logical_cpu(); + + for (cnt = 0; cnt < attr_count; cnt++) { + __read(&len, cpu, &attrs[cnt], &per_cpu(events, cpu)[cnt]); + } + + if (cpu == 0) { + for (cnt = 0; cnt < uc_attr_count; cnt++) { + __read(&len, cpu, &uc_attrs[cnt], &uc_events[cnt]); + } + } + + if (buffer) { + *buffer = per_cpu(perf_cnt, cpu); + } return len; } @@ -265,30 +381,116 @@ static struct gator_interface gator_events_perf_pmu_interface = { .read = gator_events_perf_pmu_read, }; +static void __attr_init(struct gator_attr *const attr) +{ + attr->name[0] = '\0'; + attr->enabled = 0; + attr->type = 0; + attr->event = 0; + attr->count = 0; + attr->key = gator_events_get_key(); +} + +static void gator_events_perf_pmu_cci_init(const int type) +{ + int cnt; + + strncpy(uc_attrs[uc_attr_count].name, "cci-400_ccnt", sizeof(uc_attrs[uc_attr_count].name)); + uc_attrs[uc_attr_count].type = type; + ++uc_attr_count; + + for (cnt = 0; cnt < CCI_400; ++cnt, ++uc_attr_count) { + struct gator_attr *const attr = &uc_attrs[uc_attr_count]; + snprintf(attr->name, sizeof(attr->name), "cci-400_cnt%d", cnt); + attr->type = type; + } +} + +static void gator_events_perf_pmu_cpu_init(const struct gator_cpu *const gator_cpu, const int type) +{ + int cnt; + + snprintf(attrs[attr_count].name, sizeof(attrs[attr_count].name), "%s_ccnt", gator_cpu->pmnc_name); + attrs[attr_count].type = type; + ++attr_count; + + for (cnt = 0; cnt < gator_cpu->pmnc_counters; ++cnt, ++attr_count) { + struct gator_attr *const attr = &attrs[attr_count]; + snprintf(attr->name, sizeof(attr->name), "%s_cnt%d", gator_cpu->pmnc_name, cnt); + attr->type = type; + } +} + int gator_events_perf_pmu_init(void) { - unsigned int cnt; - const u32 cpuid = gator_cpuid(); - - for (cnt = 0; gator_cpus[cnt].cpuid != 0; ++cnt) { - if (gator_cpus[cnt].cpuid == cpuid) { - pmnc_name = gator_cpus[cnt].pmnc_name; - pmnc_counters = gator_cpus[cnt].pmnc_counters; - ccnt = gator_cpus[cnt].ccnt; + struct perf_event_attr pea; + struct perf_event *pe; + const struct gator_cpu *gator_cpu; + int type; + int cpu; + int cnt; + bool found_cpu = false; + + for (cnt = 0; cnt < CNTMAX; cnt++) { + __attr_init(&attrs[cnt]); + } + for (cnt = 0; cnt < UCCNT; cnt++) { + __attr_init(&uc_attrs[cnt]); + } + + memset(&pea, 0, sizeof(pea)); + pea.size = sizeof(pea); + pea.config = 0xFF; + attr_count = 0; + uc_attr_count = 0; + for (type = PERF_TYPE_MAX; type < 0x20; ++type) { + pea.type = type; + + // A particular PMU may work on some but not all cores, so try on each core + pe = NULL; + for_each_present_cpu(cpu) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0) + pe = perf_event_create_kernel_counter(&pea, cpu, 0, dummy_handler); +#else + pe = perf_event_create_kernel_counter(&pea, cpu, 0, dummy_handler, 0); +#endif + if (!IS_ERR(pe)) { + break; + } + } + // Assume that valid PMUs are contigious + if (IS_ERR(pe)) { break; } + + if (pe->pmu != NULL && type == pe->pmu->type) { + if (strcmp("CCI", pe->pmu->name) == 0) { + gator_events_perf_pmu_cci_init(type); + } else if ((gator_cpu = gator_find_cpu_by_pmu_name(pe->pmu->name)) != NULL) { + found_cpu = true; + gator_events_perf_pmu_cpu_init(gator_cpu, type); + } + } + + perf_event_release_kernel(pe); } - if (gator_cpus[cnt].cpuid == 0) { - return -1; + + if (!found_cpu) { + const struct gator_cpu *const gator_cpu = gator_find_cpu_by_cpuid(gator_cpuid()); + if (gator_cpu == NULL) { + return -1; + } + gator_events_perf_pmu_cpu_init(gator_cpu, PERF_TYPE_RAW); } - pmnc_counters++; // CNT[n] + CCNT + if (attr_count > CNTMAX) { + printk(KERN_ERR "gator: Too many perf counters\n"); + return -1; + } - for (cnt = 0; cnt < CNTMAX; cnt++) { - pmnc_enabled[cnt] = 0; - pmnc_event[cnt] = 0; - pmnc_count[cnt] = 0; - pmnc_key[cnt] = gator_events_get_key(); + if (uc_attr_count > UCCNT) { + printk(KERN_ERR "gator: Too many perf uncore counters\n"); + return -1; } return gator_events_install(&gator_events_perf_pmu_interface); diff --git a/driver/gator_events_sched.c b/driver/gator_events_sched.c index ba6744d..461a051 100644 --- a/driver/gator_events_sched.c +++ b/driver/gator_events_sched.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * Copyright (C) ARM Limited 2010-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -29,7 +29,7 @@ GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_ // disable interrupts to synchronize with gator_events_sched_read() // spinlocks not needed since percpu buffers are used local_irq_save(flags); - per_cpu(schedCnt, smp_processor_id())[SCHED_SWITCH]++; + per_cpu(schedCnt, get_physical_cpu())[SCHED_SWITCH]++; local_irq_restore(flags); } @@ -78,7 +78,7 @@ static int gator_events_sched_read(int **buffer) { unsigned long flags; int len, value; - int cpu = smp_processor_id(); + int cpu = get_physical_cpu(); len = 0; if (sched_switch_enabled) { diff --git a/driver/gator_events_scorpion.c b/driver/gator_events_scorpion.c index 5ffc63a..aaf306a 100644 --- a/driver/gator_events_scorpion.c +++ b/driver/gator_events_scorpion.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2011-2012. All rights reserved. + * Copyright (C) ARM Limited 2011-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -524,7 +524,7 @@ static int gator_events_scorpion_create_files(struct super_block *sb, struct den return 0; } -static int gator_events_scorpion_online(int **buffer) +static int gator_events_scorpion_online(int **buffer, bool migrate) { unsigned int cnt, len = 0, cpu = smp_processor_id(); @@ -581,7 +581,7 @@ static int gator_events_scorpion_online(int **buffer) return len; } -static int gator_events_scorpion_offline(int **buffer) +static int gator_events_scorpion_offline(int **buffer, bool migrate) { scorpion_pmnc_write(scorpion_pmnc_read() & ~PMNC_E); return 0; diff --git a/driver/gator_fs.c b/driver/gator_fs.c index 9ff118b..fe6f83d 100644 --- a/driver/gator_fs.c +++ b/driver/gator_fs.c @@ -53,6 +53,15 @@ ssize_t gatorfs_ulong_to_user(unsigned long val, char __user *buf, size_t count, return simple_read_from_buffer(buf, count, offset, tmpbuf, maxlen); } +ssize_t gatorfs_u64_to_user(u64 val, char __user *buf, size_t count, loff_t *offset) +{ + char tmpbuf[TMPBUFSIZE]; + size_t maxlen = snprintf(tmpbuf, TMPBUFSIZE, "%llu\n", val); + if (maxlen > TMPBUFSIZE) + maxlen = TMPBUFSIZE; + return simple_read_from_buffer(buf, count, offset, tmpbuf, maxlen); +} + int gatorfs_ulong_from_user(unsigned long *val, char const __user *buf, size_t count) { char tmpbuf[TMPBUFSIZE]; @@ -75,12 +84,40 @@ int gatorfs_ulong_from_user(unsigned long *val, char const __user *buf, size_t c return 0; } +int gatorfs_u64_from_user(u64 *val, char const __user *buf, size_t count) +{ + char tmpbuf[TMPBUFSIZE]; + unsigned long flags; + + if (!count) + return 0; + + if (count > TMPBUFSIZE - 1) + return -EINVAL; + + memset(tmpbuf, 0x0, TMPBUFSIZE); + + if (copy_from_user(tmpbuf, buf, count)) + return -EFAULT; + + spin_lock_irqsave(&gatorfs_lock, flags); + *val = simple_strtoull(tmpbuf, NULL, 0); + spin_unlock_irqrestore(&gatorfs_lock, flags); + return 0; +} + static ssize_t ulong_read_file(struct file *file, char __user *buf, size_t count, loff_t *offset) { unsigned long *val = file->private_data; return gatorfs_ulong_to_user(*val, buf, count, offset); } +static ssize_t u64_read_file(struct file *file, char __user *buf, size_t count, loff_t *offset) +{ + u64 *val = file->private_data; + return gatorfs_u64_to_user(*val, buf, count, offset); +} + static ssize_t ulong_write_file(struct file *file, char const __user *buf, size_t count, loff_t *offset) { unsigned long *value = file->private_data; @@ -96,6 +133,21 @@ static ssize_t ulong_write_file(struct file *file, char const __user *buf, size_ return count; } +static ssize_t u64_write_file(struct file *file, char const __user *buf, size_t count, loff_t *offset) +{ + u64 *value = file->private_data; + int retval; + + if (*offset) + return -EINVAL; + + retval = gatorfs_u64_from_user(value, buf, count); + + if (retval) + return retval; + return count; +} + static int default_open(struct inode *inode, struct file *filp) { if (inode->i_private) @@ -109,11 +161,22 @@ static const struct file_operations ulong_fops = { .open = default_open, }; +static const struct file_operations u64_fops = { + .read = u64_read_file, + .write = u64_write_file, + .open = default_open, +}; + static const struct file_operations ulong_ro_fops = { .read = ulong_read_file, .open = default_open, }; +static const struct file_operations u64_ro_fops = { + .read = u64_read_file, + .open = default_open, +}; + static struct dentry *__gatorfs_create_file(struct super_block *sb, struct dentry *root, char const *name, @@ -148,6 +211,18 @@ int gatorfs_create_ulong(struct super_block *sb, struct dentry *root, return 0; } +int gatorfs_create_u64(struct super_block *sb, struct dentry *root, + char const *name, u64 *val) +{ + struct dentry *d = __gatorfs_create_file(sb, root, name, + &u64_fops, 0644); + if (!d) + return -EFAULT; + + d->d_inode->i_private = val; + return 0; +} + int gatorfs_create_ro_ulong(struct super_block *sb, struct dentry *root, char const *name, unsigned long *val) { @@ -160,6 +235,18 @@ int gatorfs_create_ro_ulong(struct super_block *sb, struct dentry *root, return 0; } +int gatorfs_create_ro_u64(struct super_block *sb, struct dentry *root, + char const *name, u64 * val) +{ + struct dentry *d = + __gatorfs_create_file(sb, root, name, &u64_ro_fops, 0444); + if (!d) + return -EFAULT; + + d->d_inode->i_private = val; + return 0; +} + static ssize_t atomic_read_file(struct file *file, char __user *buf, size_t count, loff_t *offset) { atomic_t *val = file->private_data; diff --git a/driver/gator_hrtimer_gator.c b/driver/gator_hrtimer_gator.c index 846fba4..8c35d49 100644 --- a/driver/gator_hrtimer_gator.c +++ b/driver/gator_hrtimer_gator.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2011-2012. All rights reserved. + * Copyright (C) ARM Limited 2011-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -15,8 +15,8 @@ void (*callback)(void); DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer); DEFINE_PER_CPU(int, hrtimer_is_active); static ktime_t profiling_interval; -static void gator_hrtimer_online(int cpu); -static void gator_hrtimer_offline(int cpu); +static void gator_hrtimer_online(void); +static void gator_hrtimer_offline(void); static enum hrtimer_restart gator_hrtimer_notify(struct hrtimer *hrtimer) { @@ -25,43 +25,28 @@ static enum hrtimer_restart gator_hrtimer_notify(struct hrtimer *hrtimer) return HRTIMER_RESTART; } -static void gator_hrtimer_switch_cpus_online(void *unused) -{ - gator_hrtimer_online(smp_processor_id()); -} - -static void gator_hrtimer_online(int cpu) +static void gator_hrtimer_online(void) { + int cpu = get_logical_cpu(); struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu); - if (cpu != smp_processor_id()) { - smp_call_function_single(cpu, gator_hrtimer_switch_cpus_online, NULL, 1); - return; - } - if (per_cpu(hrtimer_is_active, cpu) || profiling_interval.tv64 == 0) return; per_cpu(hrtimer_is_active, cpu) = 1; hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); hrtimer->function = gator_hrtimer_notify; +#ifdef CONFIG_PREEMPT_RT_BASE + hrtimer->irqsafe = 1; +#endif hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED); } -static void gator_hrtimer_switch_cpus_offline(void *unused) -{ - gator_hrtimer_offline(smp_processor_id()); -} - -static void gator_hrtimer_offline(int cpu) +static void gator_hrtimer_offline(void) { + int cpu = get_logical_cpu(); struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu); - if (cpu != smp_processor_id()) { - smp_call_function_single(cpu, gator_hrtimer_switch_cpus_offline, NULL, 1); - return; - } - if (!per_cpu(hrtimer_is_active, cpu)) return; diff --git a/driver/gator_hrtimer_perf.c b/driver/gator_hrtimer_perf.c index 7c0333f..7b95399 100644 --- a/driver/gator_hrtimer_perf.c +++ b/driver/gator_hrtimer_perf.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2011-2012. All rights reserved. + * Copyright (C) ARM Limited 2011-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as diff --git a/driver/gator_iks.c b/driver/gator_iks.c new file mode 100644 index 0000000..932be26 --- /dev/null +++ b/driver/gator_iks.c @@ -0,0 +1,144 @@ +/** + * Copyright (C) ARM Limited 2013. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#if GATOR_IKS_SUPPORT + +#include <linux/of.h> +#include <asm/bL_switcher.h> +#include <asm/smp_plat.h> +#include <trace/events/power_cpu_migrate.h> + +static int mpidr_cpuids[NR_CPUS]; +static int __lcpu_to_pcpu[NR_CPUS]; + +static void calc_first_cluster_size(void) +{ + int len; + const u32 *val; + struct device_node *cn = NULL; + int mpidr_cpuids_count = 0; + + // Zero is a valid cpuid, so initialize the array to 0xff's + memset(&mpidr_cpuids, 0xff, sizeof(mpidr_cpuids)); + + while ((cn = of_find_node_by_type(cn, "cpu"))) { + BUG_ON(mpidr_cpuids_count >= NR_CPUS); + + val = of_get_property(cn, "reg", &len); + if (!val || len != 4) { + pr_err("%s missing reg property\n", cn->full_name); + continue; + } + + mpidr_cpuids[mpidr_cpuids_count] = be32_to_cpup(val); + ++mpidr_cpuids_count; + } + + BUG_ON(mpidr_cpuids_count != nr_cpu_ids); +} + +static int linearize_mpidr(int mpidr) +{ + int i; + for (i = 0; i < nr_cpu_ids; ++i) { + if (mpidr_cpuids[i] == mpidr) { + return i; + } + } + + BUG(); +} + +int lcpu_to_pcpu(const int lcpu) +{ + int pcpu; + BUG_ON(lcpu >= nr_cpu_ids || lcpu < 0); + pcpu = __lcpu_to_pcpu[lcpu]; + BUG_ON(pcpu >= nr_cpu_ids || pcpu < 0); + return pcpu; +} + +int pcpu_to_lcpu(const int pcpu) +{ + int lcpu; + BUG_ON(pcpu >= nr_cpu_ids || pcpu < 0); + for (lcpu = 0; lcpu < nr_cpu_ids; ++lcpu) { + if (__lcpu_to_pcpu[lcpu] == pcpu) { + BUG_ON(lcpu >= nr_cpu_ids || lcpu < 0); + return lcpu; + } + } + BUG(); +} + +static void gator_update_cpu_mapping(u32 cpu_hwid) +{ + int lcpu = smp_processor_id(); + int pcpu = linearize_mpidr(cpu_hwid & MPIDR_HWID_BITMASK); + BUG_ON(lcpu >= nr_cpu_ids || lcpu < 0); + BUG_ON(pcpu >= nr_cpu_ids || pcpu < 0); + __lcpu_to_pcpu[lcpu] = pcpu; +} + +GATOR_DEFINE_PROBE(cpu_migrate_begin, TP_PROTO(u64 timestamp, u32 cpu_hwid)) +{ + const int cpu = get_physical_cpu(); + + gator_timer_offline((void *)1); + gator_timer_offline_dispatch(cpu, true); +} + +GATOR_DEFINE_PROBE(cpu_migrate_finish, TP_PROTO(u64 timestamp, u32 cpu_hwid)) +{ + int cpu; + + gator_update_cpu_mapping(cpu_hwid); + + // get_physical_cpu must be called after gator_update_cpu_mapping + cpu = get_physical_cpu(); + gator_timer_online_dispatch(cpu, true); + gator_timer_online((void *)1); +} + +GATOR_DEFINE_PROBE(cpu_migrate_current, TP_PROTO(u64 timestamp, u32 cpu_hwid)) +{ + gator_update_cpu_mapping(cpu_hwid); +} + +static int gator_migrate_start(void) +{ + int retval = 0; + if (retval == 0) + retval = GATOR_REGISTER_TRACE(cpu_migrate_begin); + if (retval == 0) + retval = GATOR_REGISTER_TRACE(cpu_migrate_finish); + if (retval == 0) + retval = GATOR_REGISTER_TRACE(cpu_migrate_current); + if (retval == 0) { + // Initialize the logical to physical cpu mapping + memset(&__lcpu_to_pcpu, 0xff, sizeof(__lcpu_to_pcpu)); + bL_switcher_trace_trigger(); + } + return retval; +} + +static void gator_migrate_stop(void) +{ + GATOR_UNREGISTER_TRACE(cpu_migrate_current); + GATOR_UNREGISTER_TRACE(cpu_migrate_finish); + GATOR_UNREGISTER_TRACE(cpu_migrate_begin); +} + +#else + +#define calc_first_cluster_size() +#define gator_migrate_start() 0 +#define gator_migrate_stop() + +#endif diff --git a/driver/gator_main.c b/driver/gator_main.c index 9e031c1..3e62b59 100644 --- a/driver/gator_main.c +++ b/driver/gator_main.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * Copyright (C) ARM Limited 2010-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -7,7 +7,8 @@ * */ -static unsigned long gator_protocol_version = 12; +// This version must match the gator daemon version +static unsigned long gator_protocol_version = 13; #include <linux/slab.h> #include <linux/cpu.h> @@ -20,6 +21,7 @@ static unsigned long gator_protocol_version = 12; #include <linux/suspend.h> #include <linux/module.h> #include <linux/perf_event.h> +#include <linux/utsname.h> #include <asm/stacktrace.h> #include <asm/uaccess.h> @@ -92,9 +94,14 @@ static unsigned long gator_protocol_version = 12; #define MESSAGE_SCHED_SWITCH 1 #define MESSAGE_SCHED_EXIT 2 +#define MESSAGE_IDLE_ENTER 1 +#define MESSAGE_IDLE_EXIT 2 + #define MAXSIZE_PACK32 5 #define MAXSIZE_PACK64 10 +#define FRAME_HEADER_SIZE 3 + #if defined(__arm__) #define PC_REG regs->ARM_pc #elif defined(__aarch64__) @@ -123,23 +130,35 @@ static unsigned long gator_cpu_cores; // Size of the largest buffer. Effectively constant, set in gator_op_create_files static unsigned long userspace_buffer_size; static unsigned long gator_backtrace_depth; +// How often to commit the buffers for live in nanoseconds +static u64 gator_live_rate; static unsigned long gator_started; -static uint64_t monotonic_started; +static u64 gator_monotonic_started; static unsigned long gator_buffer_opened; static unsigned long gator_timer_count; static unsigned long gator_response_type; static DEFINE_MUTEX(start_mutex); static DEFINE_MUTEX(gator_buffer_mutex); +bool event_based_sampling; + static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait); +static DECLARE_WAIT_QUEUE_HEAD(gator_annotate_wait); static struct timer_list gator_buffer_wake_up_timer; static LIST_HEAD(gator_events); +static DEFINE_PER_CPU(u64, last_timestamp); + +static bool printed_monotonic_warning; + +static bool sent_core_name[NR_CPUS]; + /****************************************************************************** * Prototypes ******************************************************************************/ -static void buffer_check(int cpu, int buftype); +static void buffer_check(int cpu, int buftype, u64 time); +static void gator_commit_buffer(int cpu, int buftype, u64 time); static int buffer_bytes_available(int cpu, int buftype); static bool buffer_check_space(int cpu, int buftype, int bytes); static int contiguous_space_available(int cpu, int bufytpe); @@ -149,7 +168,7 @@ static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int le static void gator_buffer_write_string(int cpu, int buftype, const char *x); static void gator_add_trace(int cpu, unsigned long address); static void gator_add_sample(int cpu, struct pt_regs *const regs); -static uint64_t gator_get_time(void); +static u64 gator_get_time(void); // Size of the buffer, must be a power of 2. Effectively constant, set in gator_op_setup. static uint32_t gator_buffer_size[NUM_GATOR_BUFS]; @@ -167,6 +186,11 @@ static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available); // The buffer. Allocated in gator_op_setup static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer); +#if GATOR_LIVE +// The time after which the buffer should be committed for live display +static DEFINE_PER_CPU(u64, gator_buffer_commit_time); +#endif + /****************************************************************************** * Application Includes ******************************************************************************/ @@ -186,83 +210,85 @@ static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer); * Misc ******************************************************************************/ -struct gator_cpu gator_cpus[] = { - { +const struct gator_cpu gator_cpus[] = { + { .cpuid = ARM1136, .core_name = "ARM1136", .pmnc_name = "ARM_ARM11", .pmnc_counters = 3, - .ccnt = 2, }, - { + { .cpuid = ARM1156, .core_name = "ARM1156", .pmnc_name = "ARM_ARM11", .pmnc_counters = 3, - .ccnt = 2, }, - { + { .cpuid = ARM1176, .core_name = "ARM1176", .pmnc_name = "ARM_ARM11", .pmnc_counters = 3, - .ccnt = 2, }, - { + { .cpuid = ARM11MPCORE, .core_name = "ARM11MPCore", .pmnc_name = "ARM_ARM11MPCore", .pmnc_counters = 3, }, - { + { .cpuid = CORTEX_A5, .core_name = "Cortex-A5", + .pmu_name = "ARMv7_Cortex_A5", .pmnc_name = "ARM_Cortex-A5", .pmnc_counters = 2, }, - { + { .cpuid = CORTEX_A7, .core_name = "Cortex-A7", + .pmu_name = "ARMv7_Cortex_A7", .pmnc_name = "ARM_Cortex-A7", .pmnc_counters = 4, }, - { + { .cpuid = CORTEX_A8, .core_name = "Cortex-A8", + .pmu_name = "ARMv7_Cortex_A8", .pmnc_name = "ARM_Cortex-A8", .pmnc_counters = 4, }, - { + { .cpuid = CORTEX_A9, .core_name = "Cortex-A9", + .pmu_name = "ARMv7_Cortex_A9", .pmnc_name = "ARM_Cortex-A9", .pmnc_counters = 6, }, - { + { .cpuid = CORTEX_A15, .core_name = "Cortex-A15", + .pmu_name = "ARMv7_Cortex_A15", .pmnc_name = "ARM_Cortex-A15", .pmnc_counters = 6, }, - { + { .cpuid = SCORPION, .core_name = "Scorpion", .pmnc_name = "Scorpion", .pmnc_counters = 4, }, - { + { .cpuid = SCORPIONMP, .core_name = "ScorpionMP", .pmnc_name = "ScorpionMP", .pmnc_counters = 4, }, - { + { .cpuid = KRAITSIM, .core_name = "KraitSIM", .pmnc_name = "Krait", .pmnc_counters = 4, }, - { + { .cpuid = KRAIT, .core_name = "Krait", .pmnc_name = "Krait", @@ -274,19 +300,19 @@ struct gator_cpu gator_cpus[] = { .pmnc_name = "Krait", .pmnc_counters = 4, }, - { + { .cpuid = CORTEX_A53, .core_name = "Cortex-A53", .pmnc_name = "ARM_Cortex-A53", .pmnc_counters = 6, }, - { + { .cpuid = CORTEX_A57, .core_name = "Cortex-A57", .pmnc_name = "ARM_Cortex-A57", .pmnc_counters = 6, }, - { + { .cpuid = AARCH64, .core_name = "AArch64", .pmnc_name = "ARM_AArch64", @@ -298,9 +324,37 @@ struct gator_cpu gator_cpus[] = { .pmnc_name = "Other", .pmnc_counters = 6, }, - {} + {} }; +const struct gator_cpu *gator_find_cpu_by_cpuid(const u32 cpuid) +{ + int i; + + for (i = 0; gator_cpus[i].cpuid != 0; ++i) { + const struct gator_cpu *const gator_cpu = &gator_cpus[i]; + if (gator_cpu->cpuid == cpuid) { + return gator_cpu; + } + } + + return NULL; +} + +const struct gator_cpu *gator_find_cpu_by_pmu_name(const char *const name) +{ + int i; + + for (i = 0; gator_cpus[i].cpuid != 0; ++i) { + const struct gator_cpu *const gator_cpu = &gator_cpus[i]; + if (gator_cpu->pmu_name != NULL && strcmp(gator_cpu->pmu_name, name) == 0) { + return gator_cpu; + } + } + + return NULL; +} + u32 gator_cpuid(void) { #if defined(__arm__) || defined(__aarch64__) @@ -408,7 +462,7 @@ static void gator_buffer_write_string(int cpu, int buftype, const char *x) gator_buffer_write_bytes(cpu, buftype, x, len); } -static void gator_commit_buffer(int cpu, int buftype) +static void gator_commit_buffer(int cpu, int buftype, u64 time) { int type_length, commit, length, byte; @@ -423,25 +477,40 @@ static void gator_commit_buffer(int cpu, int buftype) length += gator_buffer_size[buftype]; } length = length - type_length - sizeof(int); + + if (length <= FRAME_HEADER_SIZE) { + // Nothing to write, only the frame header is present + return; + } + for (byte = 0; byte < sizeof(int); byte++) { per_cpu(gator_buffer, cpu)[buftype][(commit + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF; } per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype]; + +#if GATOR_LIVE + if (gator_live_rate > 0) { + while (time > per_cpu(gator_buffer_commit_time, cpu)) { + per_cpu(gator_buffer_commit_time, cpu) += gator_live_rate; + } + } +#endif + marshal_frame(cpu, buftype); // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater mod_timer(&gator_buffer_wake_up_timer, jiffies + 1); } -static void buffer_check(int cpu, int buftype) +static void buffer_check(int cpu, int buftype, u64 time) { int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype]; if (filled < 0) { filled += gator_buffer_size[buftype]; } if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) { - gator_commit_buffer(cpu, buftype); + gator_commit_buffer(cpu, buftype, time); } } @@ -496,7 +565,7 @@ static void gator_timer_interrupt(void) void gator_backtrace_handler(struct pt_regs *const regs) { - int cpu = smp_processor_id(); + int cpu = get_physical_cpu(); // Output backtrace gator_add_sample(cpu, regs); @@ -510,40 +579,48 @@ void gator_backtrace_handler(struct pt_regs *const regs) static int gator_running; // This function runs in interrupt context and on the appropriate core -static void gator_timer_offline(void *unused) +static void gator_timer_offline(void *migrate) { struct gator_interface *gi; - int i, len, cpu = smp_processor_id(); + int i, len, cpu = get_physical_cpu(); int *buffer; + u64 time; gator_trace_sched_offline(); gator_trace_power_offline(); - gator_hrtimer_offline(cpu); + if (!migrate) { + gator_hrtimer_offline(); + } // Offline any events and output counters + time = gator_get_time(); if (marshal_event_header()) { list_for_each_entry(gi, &gator_events, list) { if (gi->offline) { - len = gi->offline(&buffer); + len = gi->offline(&buffer, migrate); marshal_event(len, buffer); } } + // Only check after writing all counters so that time and corresponding counters appear in the same frame + buffer_check(cpu, BLOCK_COUNTER_BUF, time); } // Flush all buffers on this core for (i = 0; i < NUM_GATOR_BUFS; i++) - gator_commit_buffer(cpu, i); + gator_commit_buffer(cpu, i, time); } -// This function runs in process context and may be running on a core other than core 'cpu' -static void gator_timer_offline_dispatch(int cpu) +// This function runs in interrupt context and may be running on a core other than core 'cpu' +static void gator_timer_offline_dispatch(int cpu, bool migrate) { struct gator_interface *gi; - list_for_each_entry(gi, &gator_events, list) - if (gi->offline_dispatch) - gi->offline_dispatch(cpu); + list_for_each_entry(gi, &gator_events, list) { + if (gi->offline_dispatch) { + gi->offline_dispatch(cpu, migrate); + } + } } static void gator_timer_stop(void) @@ -553,7 +630,7 @@ static void gator_timer_stop(void) if (gator_running) { on_each_cpu(gator_timer_offline, NULL, 1); for_each_online_cpu(cpu) { - gator_timer_offline_dispatch(cpu); + gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false); } gator_running = 0; @@ -562,10 +639,10 @@ static void gator_timer_stop(void) } // This function runs in interrupt context and on the appropriate core -static void gator_timer_online(void *unused) +static void gator_timer_online(void *migrate) { struct gator_interface *gi; - int len, cpu = smp_processor_id(); + int len, cpu = get_physical_cpu(); int *buffer; gator_trace_power_online(); @@ -574,39 +651,48 @@ static void gator_timer_online(void *unused) if (marshal_event_header()) { list_for_each_entry(gi, &gator_events, list) { if (gi->online) { - len = gi->online(&buffer); + len = gi->online(&buffer, migrate); marshal_event(len, buffer); } } + // Only check after writing all counters so that time and corresponding counters appear in the same frame + buffer_check(cpu, BLOCK_COUNTER_BUF, gator_get_time()); + } + + if (!migrate) { + gator_hrtimer_online(); } - gator_hrtimer_online(cpu); #if defined(__arm__) || defined(__aarch64__) - { - const char *core_name = "Unknown"; + if (!sent_core_name[cpu]) { + const char *core_name = NULL; const u32 cpuid = gator_cpuid(); - int i; - - for (i = 0; gator_cpus[i].cpuid != 0; ++i) { - if (gator_cpus[i].cpuid == cpuid) { - core_name = gator_cpus[i].core_name; - break; - } + const struct gator_cpu *const gator_cpu = gator_find_cpu_by_cpuid(cpuid); + char core_name_buf[32]; + + if (gator_cpu != NULL) { + core_name = gator_cpu->core_name; + } else { + snprintf(core_name_buf, sizeof(core_name_buf), "Unknown (0x%.3x)", cpuid); + core_name = core_name_buf; } - marshal_core_name(core_name); + marshal_core_name(cpuid, core_name); + sent_core_name[cpu] = true; } #endif } // This function runs in interrupt context and may be running on a core other than core 'cpu' -static void gator_timer_online_dispatch(int cpu) +static void gator_timer_online_dispatch(int cpu, bool migrate) { struct gator_interface *gi; - list_for_each_entry(gi, &gator_events, list) - if (gi->online_dispatch) - gi->online_dispatch(cpu); + list_for_each_entry(gi, &gator_events, list) { + if (gi->online_dispatch) { + gi->online_dispatch(cpu, migrate); + } + } } int gator_timer_start(unsigned long sample_rate) @@ -620,45 +706,72 @@ int gator_timer_start(unsigned long sample_rate) gator_running = 1; + // event based sampling trumps hr timer based sampling + if (event_based_sampling) { + sample_rate = 0; + } + if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1) return -1; for_each_online_cpu(cpu) { - gator_timer_online_dispatch(cpu); + gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false); } on_each_cpu(gator_timer_online, NULL, 1); return 0; } -static uint64_t gator_get_time(void) +static u64 gator_get_time(void) { struct timespec ts; - uint64_t timestamp; + u64 timestamp; + u64 prev_timestamp; + u64 delta; + int cpu = smp_processor_id(); - //getnstimeofday(&ts); - do_posix_clock_monotonic_gettime(&ts); - timestamp = timespec_to_ns(&ts) - monotonic_started; + // Match clock_gettime(CLOCK_MONOTONIC_RAW, &ts) from userspace + getrawmonotonic(&ts); + timestamp = timespec_to_ns(&ts); - return timestamp; + // getrawmonotonic is not monotonic on all systems. Detect and attempt to correct these cases. + // up to 0.5ms delta has been seen on some systems, which can skew Streamline data when viewing at high resolution. + prev_timestamp = per_cpu(last_timestamp, cpu); + if (prev_timestamp <= timestamp) { + per_cpu(last_timestamp, cpu) = timestamp; + } else { + delta = prev_timestamp - timestamp; + // Log the error once + if (!printed_monotonic_warning && delta > 500000) { + printk(KERN_ERR "%s: getrawmonotonic is not monotonic cpu: %i delta: %lli\nSkew in Streamline data may be present at the fine zoom levels\n", __FUNCTION__, cpu, delta); + printed_monotonic_warning = true; + } else { + pr_debug("%s: getrawmonotonic is not monotonic cpu: %i delta: %lli\n", __FUNCTION__, cpu, delta); + } + timestamp = prev_timestamp; + } + + return timestamp - gator_monotonic_started; } /****************************************************************************** * cpu hotplug and pm notifiers ******************************************************************************/ +#include "gator_iks.c" + static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { - long cpu = (long)hcpu; + int cpu = lcpu_to_pcpu((long)hcpu); switch (action) { case CPU_DOWN_PREPARE: case CPU_DOWN_PREPARE_FROZEN: smp_call_function_single(cpu, gator_timer_offline, NULL, 1); - gator_timer_offline_dispatch(cpu); + gator_timer_offline_dispatch(cpu, false); break; case CPU_ONLINE: case CPU_ONLINE_FROZEN: - gator_timer_online_dispatch(cpu); + gator_timer_online_dispatch(cpu, false); smp_call_function_single(cpu, gator_timer_online, NULL, 1); break; } @@ -683,13 +796,13 @@ static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void unregister_scheduler_tracepoints(); on_each_cpu(gator_timer_offline, NULL, 1); for_each_online_cpu(cpu) { - gator_timer_offline_dispatch(cpu); + gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false); } break; case PM_POST_HIBERNATION: case PM_POST_SUSPEND: for_each_online_cpu(cpu) { - gator_timer_online_dispatch(cpu); + gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false); } on_each_cpu(gator_timer_online, NULL, 1); register_scheduler_tracepoints(); @@ -724,15 +837,33 @@ static void gator_notifier_stop(void) ******************************************************************************/ static void gator_summary(void) { - uint64_t timestamp; - struct timespec uptime_ts; + u64 timestamp, uptime; + struct timespec ts; + char uname_buf[512]; + void (*m2b)(struct timespec *ts); + unsigned long flags; - timestamp = gator_get_time(); + snprintf(uname_buf, sizeof(uname_buf), "%s %s %s %s %s GNU/Linux", utsname()->sysname, utsname()->nodename, utsname()->release, utsname()->version, utsname()->machine); - do_posix_clock_monotonic_gettime(&uptime_ts); - monotonic_started = timespec_to_ns(&uptime_ts); + getnstimeofday(&ts); + timestamp = timespec_to_ns(&ts); - marshal_summary(timestamp, monotonic_started); + do_posix_clock_monotonic_gettime(&ts); + // monotonic_to_bootbased is not defined for some versions of Android + m2b = symbol_get(monotonic_to_bootbased); + if (m2b) { + m2b(&ts); + } + uptime = timespec_to_ns(&ts); + + // Disable interrupts as gator_get_time calls smp_processor_id to verify time is monotonic + local_irq_save(flags); + // Set monotonic_started to zero as gator_get_time is uptime minus monotonic_started + gator_monotonic_started = 0; + gator_monotonic_started = gator_get_time(); + local_irq_restore(flags); + + marshal_summary(timestamp, uptime, uname_buf); } int gator_events_install(struct gator_interface *interface) @@ -747,13 +878,17 @@ int gator_events_get_key(void) // key of zero is reserved as a timestamp static int key = 1; - return key++; + const int ret = key; + key += 2; + return ret; } static int gator_init(void) { int i; + calc_first_cluster_size(); + // events sources (gator_events.h, generated by gator_events.sh) for (i = 0; i < ARRAY_SIZE(gator_events_list); i++) if (gator_events_list[i]) @@ -778,14 +913,19 @@ static int gator_start(void) unsigned long cpu, i; struct gator_interface *gi; + if (gator_migrate_start()) + goto migrate_failure; + // Initialize the buffer with the frame type and core for_each_present_cpu(cpu) { for (i = 0; i < NUM_GATOR_BUFS; i++) { marshal_frame(cpu, i); } + per_cpu(last_timestamp, cpu) = 0; } + printed_monotonic_warning = false; - // Capture the start time + // Capture the start time gator_summary(); // start all events @@ -838,9 +978,11 @@ annotate_failure: cookies_failure: // stop all events list_for_each_entry(gi, &gator_events, list) - if (gi->stop) - gi->stop(); + if (gi->stop) + gi->stop(); events_failure: + gator_migrate_stop(); +migrate_failure: return -1; } @@ -860,8 +1002,10 @@ static void gator_stop(void) // stop all events list_for_each_entry(gi, &gator_events, list) - if (gi->stop) - gi->stop(); + if (gi->stop) + gi->stop(); + + gator_migrate_stop(); } /****************************************************************************** @@ -915,6 +1059,9 @@ static int gator_op_setup(void) per_cpu(gator_buffer_write, cpu)[i] = 0; per_cpu(gator_buffer_commit, cpu)[i] = 0; per_cpu(buffer_space_available, cpu)[i] = true; +#if GATOR_LIVE + per_cpu(gator_buffer_commit_time, cpu) = gator_live_rate; +#endif // Annotation is a special case that only uses a single buffer if (cpu > 0 && i == ANNOTATE_BUF) { @@ -963,6 +1110,7 @@ static void gator_op_stop(void) mutex_lock(&gator_buffer_mutex); gator_started = 0; + gator_monotonic_started = 0; cookies_release(); wake_up(&gator_buffer_wait); @@ -987,10 +1135,15 @@ static void gator_shutdown(void) per_cpu(gator_buffer_write, cpu)[i] = 0; per_cpu(gator_buffer_commit, cpu)[i] = 0; per_cpu(buffer_space_available, cpu)[i] = true; +#if GATOR_LIVE + per_cpu(gator_buffer_commit_time, cpu) = 0; +#endif } mutex_unlock(&gator_buffer_mutex); } + memset(&sent_core_name, 0, sizeof(sent_core_name)); + mutex_unlock(&start_mutex); } @@ -1143,6 +1296,11 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf, /* kick just in case we've lost an SMP event */ wake_up(&gator_buffer_wait); + // Wake up annotate_write if more space is available + if (buftype == ANNOTATE_BUF) { + wake_up(&gator_annotate_wait); + } + out: mutex_unlock(&gator_buffer_mutex); return retval; @@ -1196,6 +1354,7 @@ void gator_op_create_files(struct super_block *sb, struct dentry *root) } userspace_buffer_size = BACKTRACE_BUFFER_SIZE; gator_response_type = 1; + gator_live_rate = 0; gatorfs_create_file(sb, root, "enable", &enable_fops); gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops); @@ -1205,6 +1364,8 @@ void gator_op_create_files(struct super_block *sb, struct dentry *root) gatorfs_create_ulong(sb, root, "tick", &gator_timer_count); gatorfs_create_ulong(sb, root, "response_type", &gator_response_type); gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version); + gatorfs_create_ro_u64(sb, root, "started", &gator_monotonic_started); + gatorfs_create_u64(sb, root, "live_rate", &gator_live_rate); // Annotate interface gator_annotate_create_files(sb, root); @@ -1212,8 +1373,8 @@ void gator_op_create_files(struct super_block *sb, struct dentry *root) // Linux Events dir = gatorfs_mkdir(sb, root, "events"); list_for_each_entry(gi, &gator_events, list) - if (gi->create_files) - gi->create_files(sb, dir); + if (gi->create_files) + gi->create_files(sb, dir); // Power interface gator_trace_power_create_files(sb, dir); diff --git a/driver/gator_marshaling.c b/driver/gator_marshaling.c index b2efdd2..627b441 100644 --- a/driver/gator_marshaling.c +++ b/driver/gator_marshaling.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2012. All rights reserved. + * Copyright (C) ARM Limited 2012-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -7,50 +7,72 @@ * */ -static void marshal_summary(long long timestamp, long long uptime) +#define NEWLINE_CANARY \ + /* Unix */ \ + "1\n" \ + /* Windows */ \ + "2\r\n" \ + /* Mac OS */ \ + "3\r" \ + /* RISC OS */ \ + "4\n\r" \ + /* Add another character so the length isn't 0x0a bytes */ \ + "5" + +static void marshal_summary(long long timestamp, long long uptime, const char * uname) { + unsigned long flags; int cpu = 0; + + local_irq_save(flags); + gator_buffer_write_string(cpu, SUMMARY_BUF, NEWLINE_CANARY); gator_buffer_write_packed_int64(cpu, SUMMARY_BUF, timestamp); gator_buffer_write_packed_int64(cpu, SUMMARY_BUF, uptime); - buffer_check(cpu, SUMMARY_BUF); + gator_buffer_write_string(cpu, SUMMARY_BUF, uname); + // Commit the buffer now so it can be one of the first frames read by Streamline + gator_commit_buffer(cpu, SUMMARY_BUF, gator_get_time()); + local_irq_restore(flags); } static bool marshal_cookie_header(const char *text) { - int cpu = smp_processor_id(); + int cpu = get_physical_cpu(); return buffer_check_space(cpu, NAME_BUF, strlen(text) + 3 * MAXSIZE_PACK32); } static void marshal_cookie(int cookie, const char *text) { - int cpu = smp_processor_id(); + int cpu = get_physical_cpu(); // buffer_check_space already called by marshal_cookie_header gator_buffer_write_packed_int(cpu, NAME_BUF, MESSAGE_COOKIE); gator_buffer_write_packed_int(cpu, NAME_BUF, cookie); gator_buffer_write_string(cpu, NAME_BUF, text); - buffer_check(cpu, NAME_BUF); + buffer_check(cpu, NAME_BUF, gator_get_time()); } static void marshal_thread_name(int pid, char *name) { unsigned long flags, cpu; + u64 time; local_irq_save(flags); - cpu = smp_processor_id(); + cpu = get_physical_cpu(); + time = gator_get_time(); if (buffer_check_space(cpu, NAME_BUF, TASK_COMM_LEN + 3 * MAXSIZE_PACK32 + MAXSIZE_PACK64)) { gator_buffer_write_packed_int(cpu, NAME_BUF, MESSAGE_THREAD_NAME); - gator_buffer_write_packed_int64(cpu, NAME_BUF, gator_get_time()); + gator_buffer_write_packed_int64(cpu, NAME_BUF, time); gator_buffer_write_packed_int(cpu, NAME_BUF, pid); gator_buffer_write_string(cpu, NAME_BUF, name); } - buffer_check(cpu, NAME_BUF); + buffer_check(cpu, NAME_BUF, time); local_irq_restore(flags); } static bool marshal_backtrace_header(int exec_cookie, int tgid, int pid, int inKernel) { - int cpu = smp_processor_id(); + int cpu = get_physical_cpu(); + u64 time = gator_get_time(); if (buffer_check_space(cpu, BACKTRACE_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32 + gator_backtrace_depth * 2 * MAXSIZE_PACK32)) { - gator_buffer_write_packed_int64(cpu, BACKTRACE_BUF, gator_get_time()); + gator_buffer_write_packed_int64(cpu, BACKTRACE_BUF, time); gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, exec_cookie); gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, tgid); gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, pid); @@ -59,30 +81,30 @@ static bool marshal_backtrace_header(int exec_cookie, int tgid, int pid, int inK } // Check and commit; commit is set to occur once buffer is 3/4 full - buffer_check(cpu, BACKTRACE_BUF); + buffer_check(cpu, BACKTRACE_BUF, time); return false; } static void marshal_backtrace(unsigned long address, int cookie) { - int cpu = smp_processor_id(); + int cpu = get_physical_cpu(); gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, cookie); gator_buffer_write_packed_int64(cpu, BACKTRACE_BUF, address); } static void marshal_backtrace_footer(void) { - int cpu = smp_processor_id(); + int cpu = get_physical_cpu(); gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, MESSAGE_END_BACKTRACE); // Check and commit; commit is set to occur once buffer is 3/4 full - buffer_check(cpu, BACKTRACE_BUF); + buffer_check(cpu, BACKTRACE_BUF, gator_get_time()); } static bool marshal_event_header(void) { - unsigned long flags, cpu = smp_processor_id(); + unsigned long flags, cpu = get_physical_cpu(); bool retval = false; local_irq_save(flags); @@ -91,8 +113,6 @@ static bool marshal_event_header(void) gator_buffer_write_packed_int64(cpu, BLOCK_COUNTER_BUF, gator_get_time()); retval = true; } - // Check and commit; commit is set to occur once buffer is 3/4 full - buffer_check(cpu, BLOCK_COUNTER_BUF); local_irq_restore(flags); return retval; @@ -100,7 +120,7 @@ static bool marshal_event_header(void) static void marshal_event(int len, int *buffer) { - unsigned long i, flags, cpu = smp_processor_id(); + unsigned long i, flags, cpu = get_physical_cpu(); if (len <= 0) return; @@ -120,14 +140,12 @@ static void marshal_event(int len, int *buffer) gator_buffer_write_packed_int(cpu, BLOCK_COUNTER_BUF, buffer[i]); gator_buffer_write_packed_int(cpu, BLOCK_COUNTER_BUF, buffer[i + 1]); } - // Check and commit; commit is set to occur once buffer is 3/4 full - buffer_check(cpu, BLOCK_COUNTER_BUF); local_irq_restore(flags); } static void marshal_event64(int len, long long *buffer64) { - unsigned long i, flags, cpu = smp_processor_id(); + unsigned long i, flags, cpu = get_physical_cpu(); if (len <= 0) return; @@ -147,8 +165,6 @@ static void marshal_event64(int len, long long *buffer64) gator_buffer_write_packed_int64(cpu, BLOCK_COUNTER_BUF, buffer64[i]); gator_buffer_write_packed_int64(cpu, BLOCK_COUNTER_BUF, buffer64[i + 1]); } - // Check and commit; commit is set to occur once buffer is 3/4 full - buffer_check(cpu, BLOCK_COUNTER_BUF); local_irq_restore(flags); } @@ -156,97 +172,107 @@ static void marshal_event64(int len, long long *buffer64) static void marshal_event_single(int core, int key, int value) { unsigned long flags, cpu; + u64 time; local_irq_save(flags); - cpu = smp_processor_id(); + cpu = get_physical_cpu(); + time = gator_get_time(); if (buffer_check_space(cpu, COUNTER_BUF, MAXSIZE_PACK64 + 3 * MAXSIZE_PACK32)) { - gator_buffer_write_packed_int64(cpu, COUNTER_BUF, gator_get_time()); + gator_buffer_write_packed_int64(cpu, COUNTER_BUF, time); gator_buffer_write_packed_int(cpu, COUNTER_BUF, core); gator_buffer_write_packed_int(cpu, COUNTER_BUF, key); gator_buffer_write_packed_int(cpu, COUNTER_BUF, value); } // Check and commit; commit is set to occur once buffer is 3/4 full - buffer_check(cpu, COUNTER_BUF); + buffer_check(cpu, COUNTER_BUF, time); local_irq_restore(flags); } #endif static void marshal_sched_gpu_start(int unit, int core, int tgid, int pid) { - unsigned long cpu = smp_processor_id(), flags; + unsigned long cpu = get_physical_cpu(), flags; + u64 time; if (!per_cpu(gator_buffer, cpu)[GPU_TRACE_BUF]) return; local_irq_save(flags); + time = gator_get_time(); if (buffer_check_space(cpu, GPU_TRACE_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32)) { gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, MESSAGE_GPU_START); - gator_buffer_write_packed_int64(cpu, GPU_TRACE_BUF, gator_get_time()); + gator_buffer_write_packed_int64(cpu, GPU_TRACE_BUF, time); gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, unit); gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, core); gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, tgid); gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, pid); } // Check and commit; commit is set to occur once buffer is 3/4 full - buffer_check(cpu, GPU_TRACE_BUF); + buffer_check(cpu, GPU_TRACE_BUF, time); local_irq_restore(flags); } static void marshal_sched_gpu_stop(int unit, int core) { - unsigned long cpu = smp_processor_id(), flags; + unsigned long cpu = get_physical_cpu(), flags; + u64 time; if (!per_cpu(gator_buffer, cpu)[GPU_TRACE_BUF]) return; local_irq_save(flags); + time = gator_get_time(); if (buffer_check_space(cpu, GPU_TRACE_BUF, MAXSIZE_PACK64 + 3 * MAXSIZE_PACK32)) { gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, MESSAGE_GPU_STOP); - gator_buffer_write_packed_int64(cpu, GPU_TRACE_BUF, gator_get_time()); + gator_buffer_write_packed_int64(cpu, GPU_TRACE_BUF, time); gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, unit); gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, core); } // Check and commit; commit is set to occur once buffer is 3/4 full - buffer_check(cpu, GPU_TRACE_BUF); + buffer_check(cpu, GPU_TRACE_BUF, time); local_irq_restore(flags); } static void marshal_sched_trace_switch(int tgid, int pid, int cookie, int state) { - unsigned long cpu = smp_processor_id(), flags; + unsigned long cpu = get_physical_cpu(), flags; + u64 time; if (!per_cpu(gator_buffer, cpu)[SCHED_TRACE_BUF]) return; local_irq_save(flags); + time = gator_get_time(); if (buffer_check_space(cpu, SCHED_TRACE_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32)) { gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, MESSAGE_SCHED_SWITCH); - gator_buffer_write_packed_int64(cpu, SCHED_TRACE_BUF, gator_get_time()); + gator_buffer_write_packed_int64(cpu, SCHED_TRACE_BUF, time); gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, tgid); gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, pid); gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, cookie); gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, state); } // Check and commit; commit is set to occur once buffer is 3/4 full - buffer_check(cpu, SCHED_TRACE_BUF); + buffer_check(cpu, SCHED_TRACE_BUF, time); local_irq_restore(flags); } static void marshal_sched_trace_exit(int tgid, int pid) { - unsigned long cpu = smp_processor_id(), flags; + unsigned long cpu = get_physical_cpu(), flags; + u64 time; if (!per_cpu(gator_buffer, cpu)[SCHED_TRACE_BUF]) return; local_irq_save(flags); + time = gator_get_time(); if (buffer_check_space(cpu, SCHED_TRACE_BUF, MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, MESSAGE_SCHED_EXIT); - gator_buffer_write_packed_int64(cpu, SCHED_TRACE_BUF, gator_get_time()); + gator_buffer_write_packed_int64(cpu, SCHED_TRACE_BUF, time); gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, pid); } // Check and commit; commit is set to occur once buffer is 3/4 full - buffer_check(cpu, SCHED_TRACE_BUF); + buffer_check(cpu, SCHED_TRACE_BUF, time); local_irq_restore(flags); } @@ -254,16 +280,18 @@ static void marshal_sched_trace_exit(int tgid, int pid) static void marshal_idle(int core, int state) { unsigned long flags, cpu; + u64 time; local_irq_save(flags); - cpu = smp_processor_id(); + cpu = get_physical_cpu(); + time = gator_get_time(); if (buffer_check_space(cpu, IDLE_BUF, MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { gator_buffer_write_packed_int(cpu, IDLE_BUF, state); - gator_buffer_write_packed_int64(cpu, IDLE_BUF, gator_get_time()); + gator_buffer_write_packed_int64(cpu, IDLE_BUF, time); gator_buffer_write_packed_int(cpu, IDLE_BUF, core); } // Check and commit; commit is set to occur once buffer is 3/4 full - buffer_check(cpu, IDLE_BUF); + buffer_check(cpu, IDLE_BUF, time); local_irq_restore(flags); } #endif @@ -323,16 +351,17 @@ static void marshal_frame(int cpu, int buftype) } #if defined(__arm__) || defined(__aarch64__) -static void marshal_core_name(const char *name) +static void marshal_core_name(const int cpuid, const char *name) { - int cpu = smp_processor_id(); + int cpu = get_physical_cpu(); unsigned long flags; local_irq_save(flags); if (buffer_check_space(cpu, NAME_BUF, MAXSIZE_PACK32 + MAXSIZE_CORE_NAME)) { gator_buffer_write_packed_int(cpu, NAME_BUF, HRTIMER_CORE_NAME); + gator_buffer_write_packed_int(cpu, NAME_BUF, cpuid); gator_buffer_write_string(cpu, NAME_BUF, name); } - buffer_check(cpu, NAME_BUF); + buffer_check(cpu, NAME_BUF, gator_get_time()); local_irq_restore(flags); } #endif diff --git a/driver/gator_pack.c b/driver/gator_pack.c index 119746b..2bddcbe 100644 --- a/driver/gator_pack.c +++ b/driver/gator_pack.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * Copyright (C) ARM Limited 2010-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as diff --git a/driver/gator_trace_gpu.c b/driver/gator_trace_gpu.c index 9fc488b..c94f6a0 100644 --- a/driver/gator_trace_gpu.c +++ b/driver/gator_trace_gpu.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * Copyright (C) ARM Limited 2010-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -32,14 +32,72 @@ static int mali_timeline_trace_registered; static int mali_job_slots_trace_registered; static int gpu_trace_registered; -#define GPU_UNIT_NONE 0 -#define GPU_UNIT_VP 1 -#define GPU_UNIT_FP 2 -#define GPU_UNIT_CL 3 +enum { + GPU_UNIT_NONE = 0, + GPU_UNIT_VP, + GPU_UNIT_FP, + GPU_UNIT_CL, + NUMBER_OF_GPU_UNITS +}; #define MALI_400 (0x0b07) #define MALI_T6xx (0x0056) +struct mali_gpu_job { + int count; + int last_core; + int last_tgid; + int last_pid; +}; + +#define NUMBER_OF_GPU_CORES 16 +static struct mali_gpu_job mali_gpu_jobs[NUMBER_OF_GPU_UNITS][NUMBER_OF_GPU_CORES]; +static DEFINE_SPINLOCK(mali_gpu_jobs_lock); + +static void mali_gpu_enqueue(int unit, int core, int tgid, int pid) +{ + int count; + + spin_lock(&mali_gpu_jobs_lock); + count = mali_gpu_jobs[unit][core].count; + BUG_ON(count < 0); + ++mali_gpu_jobs[unit][core].count; + if (count) { + mali_gpu_jobs[unit][core].last_core = core; + mali_gpu_jobs[unit][core].last_tgid = tgid; + mali_gpu_jobs[unit][core].last_pid = pid; + } + spin_unlock(&mali_gpu_jobs_lock); + + if (!count) { + marshal_sched_gpu_start(unit, core, tgid, pid); + } +} + +static void mali_gpu_stop(int unit, int core) +{ + int count; + int last_core = 0; + int last_tgid = 0; + int last_pid = 0; + + spin_lock(&mali_gpu_jobs_lock); + --mali_gpu_jobs[unit][core].count; + count = mali_gpu_jobs[unit][core].count; + BUG_ON(count < 0); + if (count) { + last_core = mali_gpu_jobs[unit][core].last_core; + last_tgid = mali_gpu_jobs[unit][core].last_tgid; + last_pid = mali_gpu_jobs[unit][core].last_pid; + } + spin_unlock(&mali_gpu_jobs_lock); + + marshal_sched_gpu_stop(unit, core); + if (count) { + marshal_sched_gpu_start(unit, last_core, last_tgid, last_pid); + } +} + #if defined(MALI_SUPPORT) && (MALI_SUPPORT != MALI_T6xx) #include "gator_events_mali_400.h" @@ -80,18 +138,18 @@ GATOR_DEFINE_PROBE(mali_timeline_event, TP_PROTO(unsigned int event_id, unsigned case EVENT_TYPE_START: if (component == EVENT_CHANNEL_VP0) { /* tgid = d0; pid = d1; */ - marshal_sched_gpu_start(GPU_UNIT_VP, 0, d0, d1); + mali_gpu_enqueue(GPU_UNIT_VP, 0, d0, d1); } else if (component >= EVENT_CHANNEL_FP0 && component <= EVENT_CHANNEL_FP7) { /* tgid = d0; pid = d1; */ - marshal_sched_gpu_start(GPU_UNIT_FP, component - EVENT_CHANNEL_FP0, d0, d1); + mali_gpu_enqueue(GPU_UNIT_FP, component - EVENT_CHANNEL_FP0, d0, d1); } break; case EVENT_TYPE_STOP: if (component == EVENT_CHANNEL_VP0) { - marshal_sched_gpu_stop(GPU_UNIT_VP, 0); + mali_gpu_stop(GPU_UNIT_VP, 0); } else if (component >= EVENT_CHANNEL_FP0 && component <= EVENT_CHANNEL_FP7) { - marshal_sched_gpu_stop(GPU_UNIT_FP, component - EVENT_CHANNEL_FP0); + mali_gpu_stop(GPU_UNIT_FP, component - EVENT_CHANNEL_FP0); } break; @@ -136,16 +194,16 @@ GATOR_DEFINE_PROBE(mali_job_slots_event, TP_PROTO(unsigned int event_id, unsigne if (unit != GPU_UNIT_NONE) { switch (state) { case EVENT_TYPE_START: - marshal_sched_gpu_start(unit, 0, tgid, (pid != 0 ? pid : tgid)); + mali_gpu_enqueue(unit, 0, tgid, (pid != 0 ? pid : tgid)); break; case EVENT_TYPE_STOP: - marshal_sched_gpu_stop(unit, 0); + mali_gpu_stop(unit, 0); break; default: /* * Some jobs can be soft-stopped, so ensure that this terminates the activity trace. */ - marshal_sched_gpu_stop(unit, 0); + mali_gpu_stop(unit, 0); } } } @@ -153,12 +211,12 @@ GATOR_DEFINE_PROBE(mali_job_slots_event, TP_PROTO(unsigned int event_id, unsigne GATOR_DEFINE_PROBE(gpu_activity_start, TP_PROTO(int gpu_unit, int gpu_core, struct task_struct *p)) { - marshal_sched_gpu_start(gpu_unit, gpu_core, (int)p->tgid, (int)p->pid); + mali_gpu_enqueue(gpu_unit, gpu_core, (int)p->tgid, (int)p->pid); } GATOR_DEFINE_PROBE(gpu_activity_stop, TP_PROTO(int gpu_unit, int gpu_core)) { - marshal_sched_gpu_stop(gpu_unit, gpu_core); + mali_gpu_stop(gpu_unit, gpu_core); } int gator_trace_gpu_start(void) diff --git a/driver/gator_trace_gpu.h b/driver/gator_trace_gpu.h index efb47c6..bb0f42d 100644 --- a/driver/gator_trace_gpu.h +++ b/driver/gator_trace_gpu.h @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * Copyright (C) ARM Limited 2010-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as diff --git a/driver/gator_trace_power.c b/driver/gator_trace_power.c index 79fa13c..272e056 100644 --- a/driver/gator_trace_power.c +++ b/driver/gator_trace_power.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2011-2012. All rights reserved. + * Copyright (C) ARM Limited 2011-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -38,14 +38,27 @@ static ulong power_cpu_key[POWER_TOTAL]; static int gator_trace_power_create_files(struct super_block *sb, struct dentry *root) { struct dentry *dir; + int cpu; + bool found_nonzero_freq = false; + + // Even if CONFIG_CPU_FREQ is defined, it still may not be used. Check + // for non-zero values from cpufreq_quick_get + for_each_online_cpu(cpu) { + if (cpufreq_quick_get(cpu) > 0) { + found_nonzero_freq = true; + break; + } + } - // cpu_frequency - dir = gatorfs_mkdir(sb, root, "Linux_power_cpu_freq"); - if (!dir) { - return -1; + if (found_nonzero_freq) { + // cpu_frequency + dir = gatorfs_mkdir(sb, root, "Linux_power_cpu_freq"); + if (!dir) { + return -1; + } + gatorfs_create_ulong(sb, dir, "enabled", &power_cpu_enabled[POWER_CPU_FREQ]); + gatorfs_create_ro_ulong(sb, dir, "key", &power_cpu_key[POWER_CPU_FREQ]); } - gatorfs_create_ulong(sb, dir, "enabled", &power_cpu_enabled[POWER_CPU_FREQ]); - gatorfs_create_ro_ulong(sb, dir, "key", &power_cpu_key[POWER_CPU_FREQ]); // cpu_idle dir = gatorfs_mkdir(sb, root, "Linux_power_cpu_idle"); @@ -61,13 +74,14 @@ static int gator_trace_power_create_files(struct super_block *sb, struct dentry // 'cpu' may not equal smp_processor_id(), i.e. may not be running on the core that is having the freq/idle state change GATOR_DEFINE_PROBE(cpu_frequency, TP_PROTO(unsigned int frequency, unsigned int cpu)) { + cpu = lcpu_to_pcpu(cpu); marshal_event_single(cpu, power_cpu_key[POWER_CPU_FREQ], frequency * 1000); } -#define WFI_EXIT 2 -#define WFI_ENTER 1 GATOR_DEFINE_PROBE(cpu_idle, TP_PROTO(unsigned int state, unsigned int cpu)) { + cpu = lcpu_to_pcpu(cpu); + if (state == per_cpu(idle_prev_state, cpu)) { return; } @@ -75,10 +89,10 @@ GATOR_DEFINE_PROBE(cpu_idle, TP_PROTO(unsigned int state, unsigned int cpu)) if (implements_wfi()) { if (state == PWR_EVENT_EXIT) { // transition from wfi to non-wfi - marshal_idle(cpu, WFI_EXIT); + marshal_idle(cpu, MESSAGE_IDLE_EXIT); } else { // transition from non-wfi to wfi - marshal_idle(cpu, WFI_ENTER); + marshal_idle(cpu, MESSAGE_IDLE_ENTER); } } @@ -92,16 +106,17 @@ GATOR_DEFINE_PROBE(cpu_idle, TP_PROTO(unsigned int state, unsigned int cpu)) static void gator_trace_power_online(void) { - int cpu = smp_processor_id(); + int pcpu = get_physical_cpu(); + int lcpu = get_logical_cpu(); if (power_cpu_enabled[POWER_CPU_FREQ]) { - marshal_event_single(cpu, power_cpu_key[POWER_CPU_FREQ], cpufreq_quick_get(cpu) * 1000); + marshal_event_single(pcpu, power_cpu_key[POWER_CPU_FREQ], cpufreq_quick_get(lcpu) * 1000); } } static void gator_trace_power_offline(void) { // Set frequency to zero on an offline - int cpu = smp_processor_id(); + int cpu = get_physical_cpu(); if (power_cpu_enabled[POWER_CPU_FREQ]) { marshal_event_single(cpu, power_cpu_key[POWER_CPU_FREQ], 0); } diff --git a/driver/gator_trace_sched.c b/driver/gator_trace_sched.c index d0336f9..eb989b5 100644 --- a/driver/gator_trace_sched.c +++ b/driver/gator_trace_sched.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * Copyright (C) ARM Limited 2010-2013. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -23,14 +23,13 @@ enum { STATE_WAIT_ON_OTHER = 0, STATE_CONTENTION, STATE_WAIT_ON_IO, - STATE_WAIT_ON_MUTEX, }; void emit_pid_name(struct task_struct *task) { bool found = false; char taskcomm[TASK_COMM_LEN + 3]; - unsigned long x, cpu = smp_processor_id(); + unsigned long x, cpu = get_physical_cpu(); uint64_t *keys = &(per_cpu(taskname_keys, cpu)[(task->pid & 0xFF) * TASK_MAX_COLLISIONS]); uint64_t value; @@ -66,9 +65,10 @@ void emit_pid_name(struct task_struct *task) static void collect_counters(void) { - int *buffer, len; + int *buffer, len, cpu = get_physical_cpu(); long long *buffer64; struct gator_interface *gi; + u64 time; if (marshal_event_header()) { list_for_each_entry(gi, &gator_events, list) { @@ -80,13 +80,27 @@ static void collect_counters(void) marshal_event64(len, buffer64); } } + // Only check after writing all counters so that time and corresponding counters appear in the same frame + time = gator_get_time(); + buffer_check(cpu, BLOCK_COUNTER_BUF, time); + +#if GATOR_LIVE + // Commit buffers on timeout + if (gator_live_rate > 0 && time >= per_cpu(gator_buffer_commit_time, cpu)) { + static const int buftypes[] = { COUNTER_BUF, BLOCK_COUNTER_BUF, SCHED_TRACE_BUF }; + int i; + for (i = 0; i < sizeof(buftypes)/sizeof(buftypes[0]); ++i) { + gator_commit_buffer(cpu, buftypes[i], time); + } + } +#endif } } static void probe_sched_write(int type, struct task_struct *task, struct task_struct *old_task) { int cookie = 0, state = 0; - int cpu = smp_processor_id(); + int cpu = get_physical_cpu(); int tgid = task->tgid; int pid = task->pid; @@ -98,10 +112,6 @@ static void probe_sched_write(int type, struct task_struct *task, struct task_st state = STATE_CONTENTION; } else if (old_task->in_iowait) { state = STATE_WAIT_ON_IO; -#ifdef CONFIG_DEBUG_MUTEXES - } else if (old_task->blocked_on) { - state = STATE_WAIT_ON_MUTEX; -#endif } else { state = STATE_WAIT_ON_OTHER; } |