diff options
Diffstat (limited to 'driver')
28 files changed, 2235 insertions, 544 deletions
diff --git a/driver/Makefile b/driver/Makefile index b3981ff..e521b99 100644 --- a/driver/Makefile +++ b/driver/Makefile @@ -10,7 +10,9 @@ gator-y := gator_main.o \ gator_events_sched.o \ gator_events_net.o \ gator_events_block.o \ - gator_events_meminfo.o + gator_events_meminfo.o \ + gator_events_power.o \ + gator_events_perf_pmu.o gator-y += gator_events_mmaped.o diff --git a/driver/gator.h b/driver/gator.h index 724ae19..a7a323c 100644 --- a/driver/gator.h +++ b/driver/gator.h @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2011. All rights reserved. + * Copyright (C) ARM Limited 2010-2012. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -14,15 +14,25 @@ #include <linux/mm.h> #include <linux/list.h> +#define GATOR_PERF_SUPPORT LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0) +#define GATOR_PERF_PMU_SUPPORT GATOR_PERF_SUPPORT && defined(CONFIG_PERF_EVENTS) && defined(CONFIG_HW_PERF_EVENTS) +#define GATOR_NO_PERF_SUPPORT (!(GATOR_PERF_SUPPORT)) +#define GATOR_CPU_FREQ_SUPPORT (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38)) && defined(CONFIG_CPU_FREQ) + // cpu ids #define ARM1136 0xb36 #define ARM1156 0xb56 #define ARM1176 0xb76 #define ARM11MPCORE 0xb02 #define CORTEX_A5 0xc05 +#define CORTEX_A7 0xc07 #define CORTEX_A8 0xc08 #define CORTEX_A9 0xc09 #define CORTEX_A15 0xc0f +#define SCORPION 0x00f +#define SCORPIONMP 0x02d +#define KRAITSIM 0x049 +#define KRAIT 0x04d /****************************************************************************** * Filesystem @@ -69,8 +79,10 @@ struct gator_interface { int (*create_files)(struct super_block *sb, struct dentry *root); int (*start)(void); void (*stop)(void); - void (*online)(void); - void (*offline)(void); + int (*online)(int** buffer); + int (*offline)(int** buffer); + void (*online_dispatch)(int cpu); // called in process context but may not be running on core 'cpu' + void (*offline_dispatch)(int cpu); // called in process context but may not be running on core 'cpu' int (*read)(int **buffer); int (*read64)(long long **buffer); struct list_head list; @@ -84,6 +96,4 @@ int gator_events_install(struct gator_interface *interface); int gator_events_get_key(void); extern u32 gator_cpuid(void); -extern unsigned long gator_net_traffic; - #endif // GATOR_H_ diff --git a/driver/gator_annotate.c b/driver/gator_annotate.c index ee5a160..36a921c 100644 --- a/driver/gator_annotate.c +++ b/driver/gator_annotate.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2011. All rights reserved. + * Copyright (C) ARM Limited 2010-2012. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as diff --git a/driver/gator_annotate_kernel.c b/driver/gator_annotate_kernel.c index 4f690e8..ffab087 100644 --- a/driver/gator_annotate_kernel.c +++ b/driver/gator_annotate_kernel.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2011. All rights reserved. + * Copyright (C) ARM Limited 2012. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -25,7 +25,6 @@ static void kannotate_write(char* ptr, unsigned int size) // String annotation void gator_annotate(char* string) { - printk(KERN_ERR "module: %s\n", string); kannotate_write(string, strlen(string) + 1); } EXPORT_SYMBOL(gator_annotate); diff --git a/driver/gator_backtrace.c b/driver/gator_backtrace.c index fc81233..26503ef 100644 --- a/driver/gator_backtrace.c +++ b/driver/gator_backtrace.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2011. All rights reserved. + * Copyright (C) ARM Limited 2010-2012. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as diff --git a/driver/gator_cookies.c b/driver/gator_cookies.c index c0c4b9c..1beb34f 100644 --- a/driver/gator_cookies.c +++ b/driver/gator_cookies.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2011. All rights reserved. + * Copyright (C) ARM Limited 2010-2012. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -250,12 +250,15 @@ static inline uint32_t get_cookie(int cpu, int buftype, struct task_struct *task // Can be called from interrupt handler or from work queue or from scheduler trace local_irq_save(flags); - cookie = per_cpu(cookie_next_key, cpu)+=nr_cpu_ids; - cookiemap_add(key, cookie); + cookie = INVALID_COOKIE; + if (buffer_check_space(cpu, buftype, strlen(text) + 2 * MAXSIZE_PACK32)) { + cookie = per_cpu(cookie_next_key, cpu) += nr_cpu_ids; + cookiemap_add(key, cookie); - gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COOKIE); - gator_buffer_write_packed_int(cpu, buftype, cookie); - gator_buffer_write_string(cpu, buftype, text); + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COOKIE); + gator_buffer_write_packed_int(cpu, buftype, cookie); + gator_buffer_write_string(cpu, buftype, text); + } local_irq_restore(flags); diff --git a/driver/gator_ebs.c b/driver/gator_ebs.c index 8b4b5ff..8c2997c 100644 --- a/driver/gator_ebs.c +++ b/driver/gator_ebs.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2011. All rights reserved. + * Copyright (C) ARM Limited 2012. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -7,171 +7,186 @@ * */ -/****************************************************************************** - * event based sampling handling - ******************************************************************************/ - -#if defined (__arm__) -#include "gator_events_armv7.h" +#if defined(__arm__) && (GATOR_PERF_PMU_SUPPORT) #include <linux/platform_device.h> #include <linux/interrupt.h> #include <linux/irq.h> -#if LINUX_PMU_SUPPORT #include <asm/pmu.h> -static struct platform_device *pmu_device; +extern int pmnc_counters; +extern int ccnt; +extern unsigned long pmnc_enabled[]; +extern unsigned long pmnc_event[]; +extern unsigned long pmnc_count[]; +extern unsigned long pmnc_key[]; + +static DEFINE_PER_CPU(struct perf_event *, pevent); +static DEFINE_PER_CPU(struct perf_event_attr *, pevent_attr); +static DEFINE_PER_CPU(int, key); +static DEFINE_PER_CPU(unsigned int, prev_value); -static irqreturn_t armv7_pmnc_interrupt(int irq, void *arg) +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0) +static void ebs_overflow_handler(struct perf_event *event, int unused, struct perf_sample_data *data, struct pt_regs *regs) +#else +static void ebs_overflow_handler(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs) +#endif { - unsigned int cnt, cpu = smp_processor_id(), buftype = EVENT_BUF; - struct pt_regs * const regs = get_irq_regs(); - u32 flags; - - // Stop irq generation - armv7_pmnc_write(armv7_pmnc_read() & ~PMNC_E); - - // Get and reset overflow status flags - flags = armv7_pmnc_reset_interrupt(); - - // Counters header - gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); // type - gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); // time - - // Cycle counter - if (flags & (1 << 31)) { - int value = armv7_ccnt_read(pmnc_count[CCNT]); // overrun - gator_buffer_write_packed_int(cpu, buftype, 2); // length - gator_buffer_write_packed_int(cpu, buftype, pmnc_key[CCNT]); // key - gator_buffer_write_packed_int(cpu, buftype, value); // value - } + unsigned int value, delta, cpu = smp_processor_id(), buftype = EVENT_BUF; - // PMNC counters - for (cnt = CNT0; cnt < CNTMAX; cnt++) { - if (flags & (1 << (cnt - CNT0))) { - int value = armv7_cntn_read(cnt, pmnc_count[cnt]); // overrun - gator_buffer_write_packed_int(cpu, buftype, 2); // length - gator_buffer_write_packed_int(cpu, buftype, pmnc_key[cnt]); // key - gator_buffer_write_packed_int(cpu, buftype, value); // value - } - } + if (event != per_cpu(pevent, cpu)) + return; - // End Counters, length of zero - gator_buffer_write_packed_int(cpu, buftype, 0); + if (buffer_check_space(cpu, buftype, 5 * MAXSIZE_PACK32 + MAXSIZE_PACK64)) { + value = local64_read(&event->count); + delta = value - per_cpu(prev_value, cpu); + per_cpu(prev_value, cpu) = value; + + // Counters header + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); // type + gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); // time + + // Output counter + gator_buffer_write_packed_int(cpu, buftype, 2); // length + gator_buffer_write_packed_int(cpu, buftype, per_cpu(key, cpu)); // key + gator_buffer_write_packed_int(cpu, buftype, delta); // delta + + // End Counters, length of zero + gator_buffer_write_packed_int(cpu, buftype, 0); + } // Output backtrace - gator_add_sample(cpu, buftype, regs); + if (buffer_check_space(cpu, buftype, gator_backtrace_depth * 2 * MAXSIZE_PACK32)) + gator_add_sample(cpu, buftype, regs); // Check and commit; commit is set to occur once buffer is 3/4 full - event_buffer_check(cpu); + buffer_check(cpu, buftype); +} + +static void gator_event_sampling_online(void) +{ + int cpu = smp_processor_id(), buftype = EVENT_BUF; - // Allow irq generation - armv7_pmnc_write(armv7_pmnc_read() | PMNC_E); + // read the counter and toss the invalid data, return zero instead + struct perf_event * ev = per_cpu(pevent, cpu); + if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) { + ev->pmu->read(ev); + per_cpu(prev_value, cpu) = local64_read(&ev->count); - return IRQ_HANDLED; + // Counters header + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); // type + gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); // time + + // Output counter + gator_buffer_write_packed_int(cpu, buftype, 2); // length + gator_buffer_write_packed_int(cpu, buftype, per_cpu(key, cpu)); // key + gator_buffer_write_packed_int(cpu, buftype, 0); // delta - zero for initialization + + // End Counters, length of zero + gator_buffer_write_packed_int(cpu, buftype, 0); + } } + +static void gator_event_sampling_online_dispatch(int cpu) +{ + struct perf_event * ev; + + if (!event_based_sampling) + return; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0) + ev = per_cpu(pevent, cpu) = perf_event_create_kernel_counter(per_cpu(pevent_attr, cpu), cpu, 0, ebs_overflow_handler); +#else + ev = per_cpu(pevent, cpu) = perf_event_create_kernel_counter(per_cpu(pevent_attr, cpu), cpu, 0, ebs_overflow_handler, 0); #endif + if (IS_ERR(ev)) { + pr_err("gator: unable to start event-based-sampling"); + return; + } + + if (ev->state != PERF_EVENT_STATE_ACTIVE) { + pr_err("gator: unable to start event-based-sampling"); + perf_event_release_kernel(ev); + return; + } + + ev->pmu->read(ev); + per_cpu(prev_value, cpu) = local64_read(&ev->count); +} + +static void gator_event_sampling_offline_dispatch(int cpu) +{ + if (per_cpu(pevent, cpu)) { + perf_event_release_kernel(per_cpu(pevent, cpu)); + per_cpu(pevent, cpu) = NULL; + } +} + static int gator_event_sampling_start(void) { - int cnt; + int cnt, event = 0, count = 0, ebs_key = 0, cpu; + + for_each_present_cpu(cpu) { + per_cpu(pevent, cpu) = NULL; + per_cpu(pevent_attr, cpu) = NULL; + } event_based_sampling = false; - for (cnt = CCNT; cnt < CNTMAX; cnt++) { + for (cnt = 0; cnt < pmnc_counters; cnt++) { if (pmnc_count[cnt] > 0) { event_based_sampling = true; + event = pmnc_event[cnt]; + count = pmnc_count[cnt]; + ebs_key = pmnc_key[cnt]; break; } } -#if LINUX_PMU_SUPPORT - pmu_device = reserve_pmu(ARM_PMU_DEVICE_CPU); - if (IS_ERR(pmu_device) && (unsigned int)pmu_device != -ENODEV) { - pr_err("gator: unable to reserve the pmu\n"); - return -1; - } - - if (event_based_sampling) { - int irq, i; + if (!event_based_sampling) + return 0; - if (IS_ERR(pmu_device)) { - pr_err("gator: event based sampling is not supported as the kernel function reserve_pmu() failed\n"); + for_each_present_cpu(cpu) { + u32 size = sizeof(struct perf_event_attr); + per_cpu(pevent_attr, cpu) = kmalloc(size, GFP_KERNEL); + if (!per_cpu(pevent_attr, cpu)) return -1; - } - // init_pmu sets the irq affinity, therefore we do not care if it fails for single core - if (init_pmu(ARM_PMU_DEVICE_CPU) != 0 && gator_cpu_cores > 1) { - pr_err("gator: unable to initialize the pmu\n"); - goto out_ebs_start; + memset(per_cpu(pevent_attr, cpu), 0, size); + per_cpu(pevent_attr, cpu)->type = PERF_TYPE_RAW; + per_cpu(pevent_attr, cpu)->size = size; + per_cpu(pevent_attr, cpu)->config = event; + per_cpu(pevent_attr, cpu)->sample_period = count; + per_cpu(pevent_attr, cpu)->pinned = 1; + + // handle special case for ccnt + if (cnt == ccnt) { + per_cpu(pevent_attr, cpu)->type = PERF_TYPE_HARDWARE; + per_cpu(pevent_attr, cpu)->config = PERF_COUNT_HW_CPU_CYCLES; } - if (pmu_device->num_resources == 0) { - pr_err("gator: no irqs for PMUs defined\n"); - goto out_ebs_start; - } - - for (i = 0; i < pmu_device->num_resources; ++i) { - irq = platform_get_irq(pmu_device, i); - if (irq < 0) - continue; - - if (request_irq(irq, armv7_pmnc_interrupt, IRQF_DISABLED | IRQF_NOBALANCING, "armpmu", NULL)) { - pr_err("gator: unable to request IRQ%d for ARM perf counters\n", irq); - - // clean up and exit - for (i = i - 1; i >= 0; --i) { - irq = platform_get_irq(pmu_device, i); - if (irq >= 0) - free_irq(irq, NULL); - } - goto out_ebs_start; - } - } + per_cpu(key, cpu) = ebs_key; } -#else - if (event_based_sampling) { - pr_err("gator: event based sampling only supported in kernel versions 2.6.35 and higher and CONFIG_CPU_HAS_PMU=y\n"); - return -1; - } -#endif return 0; - -#if LINUX_PMU_SUPPORT -out_ebs_start: -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0) - release_pmu(pmu_device); -#else - release_pmu(ARM_PMU_DEVICE_CPU); -#endif - pmu_device = NULL; - return -1; -#endif } static void gator_event_sampling_stop(void) { -#if LINUX_PMU_SUPPORT - if (event_based_sampling) { - int i, irq; - for (i = pmu_device->num_resources - 1; i >= 0; --i) { - irq = platform_get_irq(pmu_device, i); - if (irq >= 0) - free_irq(irq, NULL); + int cpu; + + for_each_present_cpu(cpu) { + if (per_cpu(pevent_attr, cpu)) { + kfree(per_cpu(pevent_attr, cpu)); + per_cpu(pevent_attr, cpu) = NULL; } } - if (!IS_ERR(pmu_device)) { -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0) - release_pmu(pmu_device); -#else - release_pmu(ARM_PMU_DEVICE_CPU); -#endif - } - pmu_device = NULL; -#endif } #else +static void gator_event_sampling_online(void) {} +static void gator_event_sampling_online_dispatch(int cpu) {} +static void gator_event_sampling_offline_dispatch(int cpu) {} static int gator_event_sampling_start(void) {return 0;} static void gator_event_sampling_stop(void) {} #endif diff --git a/driver/gator_events_armv6.c b/driver/gator_events_armv6.c index 170f066..ef51898 100644 --- a/driver/gator_events_armv6.c +++ b/driver/gator_events_armv6.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2011. All rights reserved. + * Copyright (C) ARM Limited 2010-2012. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -8,6 +8,9 @@ #include "gator.h" +// gator_events_perf_pmu.c is used if perf is supported +#if GATOR_NO_PERF_SUPPORT + static const char *pmnc_name; /* @@ -28,7 +31,6 @@ static const char *pmnc_name; static int pmnc_counters = 0; static unsigned long pmnc_enabled[CNTMAX]; static unsigned long pmnc_event[CNTMAX]; -static unsigned long pmnc_count[CNTMAX]; static unsigned long pmnc_key[CNTMAX]; static DEFINE_PER_CPU(int[CNTMAX], perfPrev); @@ -83,7 +85,6 @@ int gator_events_armv6_create_files(struct super_block *sb, struct dentry *root) return -1; } gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); - gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]); gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); if (i != CCNT) { gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); @@ -93,9 +94,9 @@ int gator_events_armv6_create_files(struct super_block *sb, struct dentry *root) return 0; } -static void gator_events_armv6_online(void) +static int gator_events_armv6_online(int** buffer) { - unsigned int cnt; + unsigned int cnt, len = 0, cpu = smp_processor_id(); u32 pmnc; if (armv6_pmnc_read() & PMCR_E) { @@ -110,7 +111,7 @@ static void gator_events_armv6_online(void) for (pmnc = 0, cnt = PMN0; cnt <= CCNT; cnt++) { unsigned long event; - per_cpu(perfPrev, smp_processor_id())[cnt] = 0; + per_cpu(perfPrev, cpu)[cnt] = 0; if (!pmnc_enabled[cnt]) continue; @@ -128,9 +129,22 @@ static void gator_events_armv6_online(void) armv6_pmnc_reset_counter(cnt); } armv6_pmnc_write(pmnc | PMCR_E); + + // return zero values, no need to read as the counters were just reset + for (cnt = PMN0; cnt <= CCNT; cnt++) { + if (pmnc_enabled[cnt]) { + per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; + per_cpu(perfCnt, cpu)[len++] = 0; + } + } + + if (buffer) + *buffer = per_cpu(perfCnt, cpu); + + return len; } -static void gator_events_armv6_offline(void) +static int gator_events_armv6_offline(int** buffer) { unsigned int cnt; @@ -138,18 +152,6 @@ static void gator_events_armv6_offline(void) for (cnt = PMN0; cnt <= CCNT; cnt++) { armv6_pmnc_reset_counter(cnt); } -} - -static int gator_events_armv6_start(void) -{ - int cnt; - - for (cnt = CCNT; cnt < CNTMAX; cnt++) { - if (pmnc_count[cnt] > 0) { - pr_err("gator: event based sampling not supported on ARM v6 architectures\n"); - return -1; - } - } return 0; } @@ -161,7 +163,6 @@ static void gator_events_armv6_stop(void) for (cnt = PMN0; cnt <= CCNT; cnt++) { pmnc_enabled[cnt] = 0; pmnc_event[cnt] = 0; - pmnc_count[cnt] = 0; } } @@ -193,7 +194,6 @@ static int gator_events_armv6_read(int **buffer) } } - // update or discard if (buffer) *buffer = per_cpu(perfCnt, cpu); @@ -202,7 +202,6 @@ static int gator_events_armv6_read(int **buffer) static struct gator_interface gator_events_armv6_interface = { .create_files = gator_events_armv6_create_files, - .start = gator_events_armv6_start, .stop = gator_events_armv6_stop, .online = gator_events_armv6_online, .offline = gator_events_armv6_offline, @@ -229,10 +228,17 @@ int gator_events_armv6_init(void) for (cnt = PMN0; cnt <= CCNT; cnt++) { pmnc_enabled[cnt] = 0; pmnc_event[cnt] = 0; - pmnc_count[cnt] = 0; pmnc_key[cnt] = gator_events_get_key(); } return gator_events_install(&gator_events_armv6_interface); } + gator_events_init(gator_events_armv6_init); + +#else +int gator_events_armv6_init(void) +{ + return -1; +} +#endif diff --git a/driver/gator_events_armv7.c b/driver/gator_events_armv7.c index e1434e2..cdf450f 100644 --- a/driver/gator_events_armv7.c +++ b/driver/gator_events_armv7.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2011. All rights reserved. + * Copyright (C) ARM Limited 2010-2012. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -14,15 +14,29 @@ */ #include "gator.h" -#include "gator_events_armv7.h" -const char *pmnc_name; -int pmnc_counters; +// gator_events_perf_pmu.c is used if perf is supported +#if GATOR_NO_PERF_SUPPORT -unsigned long pmnc_enabled[CNTMAX]; -unsigned long pmnc_event[CNTMAX]; -unsigned long pmnc_count[CNTMAX]; -unsigned long pmnc_key[CNTMAX]; +// Per-CPU PMNC: config reg +#define PMNC_E (1 << 0) /* Enable all counters */ +#define PMNC_P (1 << 1) /* Reset all counters */ +#define PMNC_C (1 << 2) /* Cycle counter reset */ +#define PMNC_MASK 0x3f /* Mask for writable bits */ + +// ccnt reg +#define CCNT_REG (1 << 31) + +#define CCNT 0 +#define CNT0 1 +#define CNTMAX (6+1) + +static const char *pmnc_name; +static int pmnc_counters; + +static unsigned long pmnc_enabled[CNTMAX]; +static unsigned long pmnc_event[CNTMAX]; +static unsigned long pmnc_key[CNTMAX]; static DEFINE_PER_CPU(int[CNTMAX], perfPrev); static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt); @@ -76,19 +90,13 @@ inline u32 armv7_cntn_read(unsigned int cnt, u32 reset_value) return oldval; } -static inline void armv7_pmnc_enable_interrupt(unsigned int cnt) -{ - u32 val = cnt ? (1 << (cnt - CNT0)) : (1 << 31); - asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (val)); -} - static inline void armv7_pmnc_disable_interrupt(unsigned int cnt) { u32 val = cnt ? (1 << (cnt - CNT0)) : (1 << 31); asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (val)); } -inline u32 armv7_pmnc_reset_interrupt() +inline u32 armv7_pmnc_reset_interrupt(void) { // Get and reset overflow status flags u32 flags; @@ -143,7 +151,6 @@ static int gator_events_armv7_create_files(struct super_block *sb, struct dentry return -1; } gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); - gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]); gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); if (i > 0) { gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); @@ -153,10 +160,9 @@ static int gator_events_armv7_create_files(struct super_block *sb, struct dentry return 0; } -static void gator_events_armv7_online(void) +static int gator_events_armv7_online(int** buffer) { - unsigned int cnt; - int cpu = smp_processor_id(); + unsigned int cnt, len = 0, cpu = smp_processor_id(); if (armv7_pmnc_read() & PMNC_E) { armv7_pmnc_write(armv7_pmnc_read() & ~PMNC_E); @@ -185,14 +191,10 @@ static void gator_events_armv7_online(void) if (cnt != CCNT) armv7_pmnc_write_evtsel(cnt, event); - // Enable/disable interrupt - if (pmnc_count[cnt] > 0) - armv7_pmnc_enable_interrupt(cnt); - else - armv7_pmnc_disable_interrupt(cnt); + armv7_pmnc_disable_interrupt(cnt); // Reset counter - cnt ? armv7_cntn_read(cnt, pmnc_count[cnt]) : armv7_ccnt_read(pmnc_count[cnt]); + cnt ? armv7_cntn_read(cnt, 0) : armv7_ccnt_read(0); // Enable counter armv7_pmnc_enable_counter(cnt); @@ -200,12 +202,27 @@ static void gator_events_armv7_online(void) // enable armv7_pmnc_write(armv7_pmnc_read() | PMNC_E); + + // return zero values, no need to read as the counters were just reset + for (cnt = 0; cnt < pmnc_counters; cnt++) { + if (pmnc_enabled[cnt]) { + per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; + per_cpu(perfCnt, cpu)[len++] = 0; + } + } + + if (buffer) + *buffer = per_cpu(perfCnt, cpu); + + return len; } -static void gator_events_armv7_offline(void) +static int gator_events_armv7_offline(int** buffer) { // disbale all counters, including PMCCNTR; overflow IRQs will not be signaled armv7_pmnc_write(armv7_pmnc_read() & ~PMNC_E); + + return 0; } static void gator_events_armv7_stop(void) @@ -215,7 +232,6 @@ static void gator_events_armv7_stop(void) for (cnt = CCNT; cnt < CNTMAX; cnt++) { pmnc_enabled[cnt] = 0; pmnc_event[cnt] = 0; - pmnc_count[cnt] = 0; } } @@ -224,11 +240,8 @@ static int gator_events_armv7_read(int **buffer) int cnt, len = 0; int cpu = smp_processor_id(); - if (!pmnc_counters) - return 0; - for (cnt = 0; cnt < pmnc_counters; cnt++) { - if (pmnc_enabled[cnt] && pmnc_count[cnt] == 0) { + if (pmnc_enabled[cnt]) { int value; if (cnt == CCNT) { value = armv7_ccnt_read(0); @@ -243,7 +256,6 @@ static int gator_events_armv7_read(int **buffer) } } - // update or discard if (buffer) *buffer = per_cpu(perfCnt, cpu); @@ -267,6 +279,10 @@ int gator_events_armv7_init(void) pmnc_name = "Cortex-A5"; pmnc_counters = 2; break; + case CORTEX_A7: + pmnc_name = "Cortex-A7"; + pmnc_counters = 4; + break; case CORTEX_A8: pmnc_name = "Cortex-A8"; pmnc_counters = 4; @@ -288,10 +304,17 @@ int gator_events_armv7_init(void) for (cnt = CCNT; cnt < CNTMAX; cnt++) { pmnc_enabled[cnt] = 0; pmnc_event[cnt] = 0; - pmnc_count[cnt] = 0; pmnc_key[cnt] = gator_events_get_key(); } return gator_events_install(&gator_events_armv7_interface); } + gator_events_init(gator_events_armv7_init); + +#else +int gator_events_armv7_init(void) +{ + return -1; +} +#endif diff --git a/driver/gator_events_armv7.h b/driver/gator_events_armv7.h deleted file mode 100644 index d5e8d6e..0000000 --- a/driver/gator_events_armv7.h +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright (C) ARM Limited 2011. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ - -#ifndef GATOR_EVENTS_ARMV7_H_ -#define GATOR_EVENTS_ARMV7_H_ - -// Per-CPU PMNC: config reg -#define PMNC_E (1 << 0) /* Enable all counters */ -#define PMNC_P (1 << 1) /* Reset all counters */ -#define PMNC_C (1 << 2) /* Cycle counter reset */ -#define PMNC_MASK 0x3f /* Mask for writable bits */ - -// ccnt reg -#define CCNT_REG (1 << 31) - -#define CCNT 0 -#define CNT0 1 -#define CNTMAX (6+1) - -// Function prototypes -extern void armv7_pmnc_write(u32 val); -extern u32 armv7_pmnc_read(void); -extern u32 armv7_ccnt_read(u32 reset_value); -extern u32 armv7_cntn_read(unsigned int cnt, u32 reset_value); -extern u32 armv7_pmnc_reset_interrupt(void); - -// Externed variables -extern unsigned long pmnc_enabled[CNTMAX]; -extern unsigned long pmnc_event[CNTMAX]; -extern unsigned long pmnc_count[CNTMAX]; -extern unsigned long pmnc_key[CNTMAX]; - -#endif // GATOR_EVENTS_ARMV7_H_ diff --git a/driver/gator_events_block.c b/driver/gator_events_block.c index cbfed86..f1bbbc8 100644 --- a/driver/gator_events_block.c +++ b/driver/gator_events_block.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2011. All rights reserved. + * Copyright (C) ARM Limited 2010-2012. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as diff --git a/driver/gator_events_irq.c b/driver/gator_events_irq.c index 36a6589..59461b9 100644 --- a/driver/gator_events_irq.c +++ b/driver/gator_events_irq.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2011. All rights reserved. + * Copyright (C) ARM Limited 2010-2012. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -72,15 +72,38 @@ static int gator_events_irq_create_files(struct super_block *sb, struct dentry * return 0; } -static int gator_events_irq_start(void) +static int gator_events_irq_online(int** buffer) { - int cpu, i; + int len = 0, cpu = smp_processor_id(); + unsigned long flags; // not necessary as we are in interrupt context anyway, but doesn't hurt + + // synchronization with the irq_exit functions is not necessary as the values are being reset + if (hardirq_enabled) { + local_irq_save(flags); + per_cpu(irqCnt, cpu)[HARDIRQ] = 0; + local_irq_restore(flags); + per_cpu(irqPrev, cpu)[HARDIRQ] = 0; + per_cpu(irqGet, cpu)[len++] = hardirq_key; + per_cpu(irqGet, cpu)[len++] = 0; + } - for_each_present_cpu(cpu) { - for (i = 0; i < TOTALIRQ; i++) - per_cpu(irqPrev, cpu)[i] = 0; + if (softirq_enabled) { + local_irq_save(flags); + per_cpu(irqCnt, cpu)[SOFTIRQ] = 0; + local_irq_restore(flags); + per_cpu(irqPrev, cpu)[SOFTIRQ] = 0; + per_cpu(irqGet, cpu)[len++] = softirq_key; + per_cpu(irqGet, cpu)[len++] = 0; } + if (buffer) + *buffer = per_cpu(irqGet, cpu); + + return len; +} + +static int gator_events_irq_start(void) +{ // register tracepoints if (hardirq_enabled) if (GATOR_REGISTER_TRACE(irq_handler_exit)) @@ -116,7 +139,7 @@ static void gator_events_irq_stop(void) static int gator_events_irq_read(int **buffer) { - unsigned long flags; + unsigned long flags; // not necessary as we are in interrupt context anyway, but doesn't hurt int len, value; int cpu = smp_processor_id(); @@ -153,6 +176,7 @@ static int gator_events_irq_read(int **buffer) static struct gator_interface gator_events_irq_interface = { .create_files = gator_events_irq_create_files, + .online = gator_events_irq_online, .start = gator_events_irq_start, .stop = gator_events_irq_stop, .read = gator_events_irq_read, diff --git a/driver/gator_events_l2c-310.c b/driver/gator_events_l2c-310.c index 96683b3..bd1c48a 100644 --- a/driver/gator_events_l2c-310.c +++ b/driver/gator_events_l2c-310.c @@ -1,7 +1,7 @@ /** * l2c310 (L2 Cache Controller) event counters for gator * - * Copyright (C) ARM Limited 2010-2011. All rights reserved. + * Copyright (C) ARM Limited 2010-2012. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -148,10 +148,7 @@ int gator_events_l2c310_init(void) if (gator_cpuid() != CORTEX_A5 && gator_cpuid() != CORTEX_A9) return -1; -#if defined(CONFIG_ARCH_EXYNOS4) - gator_events_l2c310_probe(0xfe600000); -#endif -#if defined(CONFIG_ARCH_S5PV310) +#if defined(CONFIG_ARCH_EXYNOS4) || defined(CONFIG_ARCH_S5PV310) gator_events_l2c310_probe(0x10502000); #endif #if defined(CONFIG_ARCH_OMAP4) diff --git a/driver/gator_events_mali.c b/driver/gator_events_mali.c new file mode 100644 index 0000000..31e8f0d --- /dev/null +++ b/driver/gator_events_mali.c @@ -0,0 +1,611 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include "gator.h" + +#include <linux/module.h> +#include <linux/time.h> +#include <linux/math64.h> + +#include "linux/mali_linux_trace.h" + +#define ACTIVITY_START 1 +#define ACTIVITY_STOP 2 + +#ifndef MALI_SUPPORT +#error MALI_SUPPORT not defined! +#endif + +#define MALI_200 0x0a07 +#define MALI_300 0x0b06 //This is not actually true; Mali-300 is also 0x0b07 +#define MALI_400 0x0b07 +#define MALI_T6xx 0x0056 + +static const char *mali_name; + +enum counters { + /* Timeline activity */ + ACTIVITY_VP = 0, + ACTIVITY_FP0, + ACTIVITY_FP1, + ACTIVITY_FP2, + ACTIVITY_FP3, + + /* L2 cache counters */ + COUNTER_L2_C0, + COUNTER_L2_C1, + + /* Vertex processor counters */ + COUNTER_VP_C0, + COUNTER_VP_C1, + + /* Fragment processor counters */ + COUNTER_FP0_C0, + COUNTER_FP0_C1, + COUNTER_FP1_C0, + COUNTER_FP1_C1, + COUNTER_FP2_C0, + COUNTER_FP2_C1, + COUNTER_FP3_C0, + COUNTER_FP3_C1, + + /* EGL Software Counters */ + COUNTER_EGL_BLIT_TIME, + + /* GLES Software Counters */ + COUNTER_GLES_DRAW_ELEMENTS_CALLS, + COUNTER_GLES_DRAW_ELEMENTS_NUM_INDICES, + COUNTER_GLES_DRAW_ELEMENTS_NUM_TRANSFORMED, + COUNTER_GLES_DRAW_ARRAYS_CALLS, + COUNTER_GLES_DRAW_ARRAYS_NUM_TRANSFORMED, + COUNTER_GLES_DRAW_POINTS, + COUNTER_GLES_DRAW_LINES, + COUNTER_GLES_DRAW_LINE_LOOP, + COUNTER_GLES_DRAW_LINE_STRIP, + COUNTER_GLES_DRAW_TRIANGLES, + COUNTER_GLES_DRAW_TRIANGLE_STRIP, + COUNTER_GLES_DRAW_TRIANGLE_FAN, + COUNTER_GLES_NON_VBO_DATA_COPY_TIME, + COUNTER_GLES_UNIFORM_BYTES_COPIED_TO_MALI, + COUNTER_GLES_UPLOAD_TEXTURE_TIME, + COUNTER_GLES_UPLOAD_VBO_TIME, + COUNTER_GLES_NUM_FLUSHES, + COUNTER_GLES_NUM_VSHADERS_GENERATED, + COUNTER_GLES_NUM_FSHADERS_GENERATED, + COUNTER_GLES_VSHADER_GEN_TIME, + COUNTER_GLES_FSHADER_GEN_TIME, + COUNTER_GLES_INPUT_TRIANGLES, + COUNTER_GLES_VXCACHE_HIT, + COUNTER_GLES_VXCACHE_MISS, + COUNTER_GLES_VXCACHE_COLLISION, + COUNTER_GLES_CULLED_TRIANGLES, + COUNTER_GLES_CULLED_LINES, + COUNTER_GLES_BACKFACE_TRIANGLES, + COUNTER_GLES_GBCLIP_TRIANGLES, + COUNTER_GLES_GBCLIP_LINES, + COUNTER_GLES_TRIANGLES_DRAWN, + COUNTER_GLES_DRAWCALL_TIME, + COUNTER_GLES_TRIANGLES_COUNT, + COUNTER_GLES_INDEPENDENT_TRIANGLES_COUNT, + COUNTER_GLES_STRIP_TRIANGLES_COUNT, + COUNTER_GLES_FAN_TRIANGLES_COUNT, + COUNTER_GLES_LINES_COUNT, + COUNTER_GLES_INDEPENDENT_LINES_COUNT, + COUNTER_GLES_STRIP_LINES_COUNT, + COUNTER_GLES_LOOP_LINES_COUNT, + + COUNTER_FILMSTRIP, + + NUMBER_OF_EVENTS +}; + +#define FIRST_ACTIVITY_EVENT ACTIVITY_VP +#define LAST_ACTIVITY_EVENT ACTIVITY_FP3 + +#define FIRST_HW_COUNTER COUNTER_L2_C0 +#define LAST_HW_COUNTER COUNTER_FP3_C1 + +#define FIRST_SW_COUNTER COUNTER_EGL_BLIT_TIME +#define LAST_SW_COUNTER COUNTER_GLES_LOOP_LINES_COUNT + +#define FIRST_SPECIAL_COUNTER COUNTER_FILMSTRIP +#define LAST_SPECIAL_COUNTER COUNTER_FILMSTRIP + +/* gatorfs variables for counter enable state, + * the event the counter should count and the + * 'key' (a unique id set by gatord and returned + * by gator.ko) + */ +static unsigned long counter_enabled[NUMBER_OF_EVENTS]; +static unsigned long counter_event[NUMBER_OF_EVENTS]; +static unsigned long counter_key[NUMBER_OF_EVENTS]; + +/* The data we have recorded */ +static u32 counter_data[NUMBER_OF_EVENTS]; +/* The address to sample (or 0 if samples are sent to us) */ +static u32* counter_address[NUMBER_OF_EVENTS]; + +/* An array used to return the data we recorded + * as key,value pairs hence the *2 + */ +static unsigned long counter_dump[NUMBER_OF_EVENTS * 2]; +static unsigned long counter_prev[NUMBER_OF_EVENTS]; + +/* Note whether tracepoints have been registered */ +static int trace_registered; + +/** + * Calculate the difference and handle the overflow. + */ +static u32 get_difference(u32 start, u32 end) +{ + if (start - end >= 0) + { + return start - end; + } + + // Mali counters are unsigned 32 bit values that wrap. + return (4294967295u - end) + start; +} + +/** + * Returns non-zero if the given counter ID is an activity counter. + */ +static inline int is_activity_counter(unsigned int event_id) +{ + return (event_id >= FIRST_ACTIVITY_EVENT && + event_id <= LAST_ACTIVITY_EVENT); +} + +/** + * Returns non-zero if the given counter ID is a hardware counter. + */ +static inline int is_hw_counter(unsigned int event_id) +{ + return (event_id >= FIRST_HW_COUNTER && event_id <= LAST_HW_COUNTER); +} + +/** + * Returns non-zero if the given counter ID is a software counter. + */ +static inline int is_sw_counter(unsigned int event_id) +{ + return (event_id >= FIRST_SW_COUNTER && event_id <= LAST_SW_COUNTER); +} + +/* + * The Mali DDK uses s64 types to contain software counter values, but gator + * can only use a maximum of 32 bits. This function scales a software counter + * to an appopriate range. + */ +static u32 scale_sw_counter_value(unsigned int event_id, signed long long value) +{ + u32 scaled_value; + + switch (event_id) { + case COUNTER_GLES_UPLOAD_TEXTURE_TIME: + case COUNTER_GLES_UPLOAD_VBO_TIME: + scaled_value = (u32)div_s64(value, 1000000); + break; + default: + scaled_value = (u32)value; + break; + } + + return scaled_value; +} + +/* Probe for continuously sampled counter */ +#if 0 //WE_DONT_CURRENTLY_USE_THIS_SO_SUPPRESS_WARNING +GATOR_DEFINE_PROBE(mali_sample_address, TP_PROTO(unsigned int event_id, u32* addr)) +{ + /* Turning on too many pr_debug statements in frequently called functions + * can cause stability and/or performance problems + */ + //pr_debug("gator: mali_sample_address %d %d\n", event_id, addr); + if (event_id >= ACTIVITY_VP && event_id <= COUNTER_FP3_C1) { + counter_address[event_id] = addr; + } +} +#endif + +/* Probe for hardware counter events */ +GATOR_DEFINE_PROBE(mali_hw_counter, TP_PROTO(unsigned int event_id, unsigned int value)) +{ + /* Turning on too many pr_debug statements in frequently called functions + * can cause stability and/or performance problems + */ + //pr_debug("gator: mali_hw_counter %d %d\n", event_id, value); + if (is_hw_counter(event_id)) { + counter_data[event_id] = value; + } +} + +GATOR_DEFINE_PROBE(mali_sw_counter, TP_PROTO(unsigned int event_id, signed long long value)) +{ + if (is_sw_counter(event_id)) { + counter_data[event_id] = scale_sw_counter_value(event_id, value); + } +} + +//TODO need to work out how many fp units we have +u32 gator_mali_get_n_fp(void) { + return 4; +} + +//TODO need to work out what kind of Mali we are looking at +u32 gator_mali_get_id(void) { + return MALI_SUPPORT; +} + +int gator_events_mali_create_files(struct super_block *sb, struct dentry *root) { + struct dentry *dir; + int event; + int n_fp = gator_mali_get_n_fp(); + + /* + * Create the filesystem entries for vertex processor, fragement processor + * and L2 cache timeline and hardware counters. Software counters get + * special handling after this block. + */ + for (event = FIRST_ACTIVITY_EVENT; event <= LAST_HW_COUNTER; event++) + { + char buf[40]; + + /* + * We can skip this event if it's for a non-existent fragment + * processor. + */ + if (((event - ACTIVITY_FP0 >= n_fp) && (event < COUNTER_L2_C0)) || + (((event - COUNTER_FP0_C0)/2 >= n_fp))) + { + continue; + } + + /* Otherwise, set up the filesystem entry for this event. */ + switch (event) { + case ACTIVITY_VP: + snprintf(buf, sizeof buf, "ARM_%s_VP_active", mali_name); + break; + case ACTIVITY_FP0: + case ACTIVITY_FP1: + case ACTIVITY_FP2: + case ACTIVITY_FP3: + snprintf(buf, sizeof buf, "ARM_%s_FP%d_active", + mali_name, event - ACTIVITY_FP0); + break; + case COUNTER_L2_C0: + case COUNTER_L2_C1: + snprintf(buf, sizeof buf, "ARM_%s_L2_cnt%d", + mali_name, event - COUNTER_L2_C0); + break; + case COUNTER_VP_C0: + case COUNTER_VP_C1: + snprintf(buf, sizeof buf, "ARM_%s_VP_cnt%d", + mali_name, event - COUNTER_VP_C0); + break; + case COUNTER_FP0_C0: + case COUNTER_FP0_C1: + case COUNTER_FP1_C0: + case COUNTER_FP1_C1: + case COUNTER_FP2_C0: + case COUNTER_FP2_C1: + case COUNTER_FP3_C0: + case COUNTER_FP3_C1: + snprintf(buf, sizeof buf, "ARM_%s_FP%d_cnt%d", mali_name, + (event - COUNTER_FP0_C0) / 2, (event - COUNTER_FP0_C0) % 2); + break; + default: + printk("gator: trying to create file for non-existent counter (%d)\n", event); + continue; + } + + dir = gatorfs_mkdir(sb, root, buf); + + if (!dir) { + return -1; + } + + gatorfs_create_ulong(sb, dir, "enabled", &counter_enabled[event]); + + /* Only create an event node for counters that can change what they count */ + if (event >= COUNTER_L2_C0) { + gatorfs_create_ulong(sb, dir, "event", &counter_event[event]); + } + + gatorfs_create_ro_ulong(sb, dir, "key", &counter_key[event]); + } + + /* Now set up the software counter entries */ + for (event = FIRST_SW_COUNTER; event <= LAST_SW_COUNTER; event++) + { + char buf[40]; + + snprintf(buf, sizeof(buf), "ARM_%s_SW_%d", mali_name, event); + + dir = gatorfs_mkdir(sb, root, buf); + + if (!dir) { + return -1; + } + + gatorfs_create_ulong(sb, dir, "enabled", &counter_enabled[event]); + gatorfs_create_ro_ulong(sb, dir, "key", &counter_key[event]); + } + + /* Now set up the special counter entries */ + for (event = FIRST_SPECIAL_COUNTER; event <= LAST_SPECIAL_COUNTER; event++) + { + char buf[40]; + + snprintf(buf, sizeof(buf), "ARM_%s_Filmstrip", mali_name); + + dir = gatorfs_mkdir(sb, root, buf); + + if (!dir) { + return -1; + } + + gatorfs_create_ulong(sb, dir, "event", &counter_event[event]); + gatorfs_create_ulong(sb, dir, "enabled", &counter_enabled[event]); + gatorfs_create_ro_ulong(sb, dir, "key", &counter_key[event]); + } + + + return 0; +} + +//TODO +void _mali_profiling_set_event(unsigned int, unsigned int); +void _mali_osk_fb_control_set(unsigned int, unsigned int); + +void _mali_profiling_get_counters(unsigned int*, unsigned int*, unsigned int*, unsigned int*); +void (*_mali_profiling_get_counters_function_pointer)(unsigned int*, unsigned int*, unsigned int*, unsigned int*); + +static void mali_counter_initialize(void) +{ + /* If a Mali driver is present and exporting the appropriate symbol + * then we can request the HW counters (of which there are only 2) + * be configured to count the desired events + */ + void (*set_hw_event)(unsigned int, unsigned int); + void (*set_fb_event)(unsigned int, unsigned int); + + set_hw_event = symbol_get(_mali_profiling_set_event); + + if (set_hw_event) { + int i; + + pr_debug("gator: mali online _mali_profiling_set_event symbol @ %p\n",set_hw_event); + + for (i = FIRST_HW_COUNTER; i <= LAST_HW_COUNTER; i++) { + if (counter_enabled[i]) { + set_hw_event(i, counter_event[i]); + } else { + set_hw_event(i, 0xFFFFFFFF); + } + } + + symbol_put(_mali_profiling_set_event); + } else { + printk("gator: mali online _mali_profiling_set_event symbol not found\n"); + } + + set_fb_event = symbol_get(_mali_osk_fb_control_set); + + if (set_fb_event) { + pr_debug("gator: mali online _mali_osk_fb_control_set symbol @ %p\n", set_fb_event); + + set_fb_event(0,(counter_enabled[COUNTER_FILMSTRIP]?1:0)); + + symbol_put(_mali_osk_fb_control_set); + } else { + printk("gator: mali online _mali_osk_fb_control_set symbol not found\n"); + } + + _mali_profiling_get_counters_function_pointer = symbol_get(_mali_profiling_get_counters); + if (_mali_profiling_get_counters_function_pointer){ + pr_debug("gator: mali online _mali_profiling_get_counters symbol @ %p\n", _mali_profiling_get_counters_function_pointer); + counter_prev[COUNTER_L2_C0] = 0; + counter_prev[COUNTER_L2_C1] = 0; + } + else{ + pr_debug("gator WARNING: mali _mali_profiling_get_counters symbol not defined"); + } + +} + +static void mali_counter_deinitialize(void) +{ + void (*set_hw_event)(unsigned int, unsigned int); + void (*set_fb_event)(unsigned int, unsigned int); + + set_hw_event = symbol_get(_mali_profiling_set_event); + + if (set_hw_event) { + int i; + + pr_debug("gator: mali offline _mali_profiling_set_event symbol @ %p\n",set_hw_event); + for (i = FIRST_HW_COUNTER; i <= LAST_HW_COUNTER; i++) { + set_hw_event(i, 0xFFFFFFFF); + } + + symbol_put(_mali_profiling_set_event); + } else { + printk("gator: mali offline _mali_profiling_set_event symbol not found\n"); + } + + set_fb_event = symbol_get(_mali_osk_fb_control_set); + + if (set_fb_event) { + pr_debug("gator: mali offline _mali_osk_fb_control_set symbol @ %p\n", set_fb_event); + + set_fb_event(0,0); + + symbol_put(_mali_osk_fb_control_set); + } else { + printk("gator: mali offline _mali_osk_fb_control_set symbol not found\n"); + } + + if (_mali_profiling_get_counters_function_pointer){ + symbol_put(_mali_profiling_get_counters); + } + +} + +static int gator_events_mali_start(void) { + // register tracepoints + if (GATOR_REGISTER_TRACE(mali_hw_counter)) { + printk("gator: mali_hw_counter tracepoint failed to activate\n"); + return -1; + } + + if (GATOR_REGISTER_TRACE(mali_sw_counter)) { + printk("gator: mali_sw_counter tracepoint failed to activate\n"); + return -1; + } + + trace_registered = 1; + + mali_counter_initialize(); + return 0; +} + +static void gator_events_mali_stop(void) { + unsigned int cnt; + + pr_debug("gator: mali stop\n"); + + if (trace_registered) { + GATOR_UNREGISTER_TRACE(mali_hw_counter); + GATOR_UNREGISTER_TRACE(mali_sw_counter); + + pr_debug("gator: mali timeline tracepoint deactivated\n"); + + trace_registered = 0; + } + + for (cnt = FIRST_ACTIVITY_EVENT; cnt < NUMBER_OF_EVENTS; cnt++) { + counter_enabled[cnt] = 0; + counter_event[cnt] = 0; + counter_address[cnt] = NULL; + } + + mali_counter_deinitialize(); +} + +static int gator_events_mali_read(int **buffer) { + int cnt, len = 0; + + if (smp_processor_id()) return 0; + + // Read the L2 C0 and C1 here. + if (counter_enabled[COUNTER_L2_C0] || counter_enabled[COUNTER_L2_C1] ) { + u32 src0 = 0; + u32 val0 = 0; + u32 src1 = 0; + u32 val1 = 0; + + // Poke the driver to get the counter values + if (_mali_profiling_get_counters_function_pointer){ + _mali_profiling_get_counters_function_pointer(&src0, &val0, &src1, &val1); + } + + if (counter_enabled[COUNTER_L2_C0]) + { + // Calculate and save src0's counter val0 + counter_dump[len++] = counter_key[COUNTER_L2_C0]; + counter_dump[len++] = get_difference(val0, counter_prev[COUNTER_L2_C0]); + } + + if (counter_enabled[COUNTER_L2_C1]) + { + // Calculate and save src1's counter val1 + counter_dump[len++] = counter_key[COUNTER_L2_C1]; + counter_dump[len++] = get_difference(val1, counter_prev[COUNTER_L2_C1]); + } + + // Save the previous values for the counters. + counter_prev[COUNTER_L2_C0] = val0; + counter_prev[COUNTER_L2_C1] = val1; + } + + // Process other (non-timeline) counters. + for (cnt = COUNTER_VP_C0; cnt <= LAST_SW_COUNTER; cnt++) { + if (counter_enabled[cnt]) { + u32 value = 0; + + // Determine the current value of the counter. + if( counter_address[cnt] != NULL && 0 ) { // Never true! + value = *counter_address[cnt]; + } else if (counter_data[cnt]!=0) { + value = counter_data[cnt]; + counter_data[cnt] = 0; + } + + // Send the counter value only if it differs from last time. + if (value != counter_prev[cnt]) { + counter_prev[cnt] = value; + counter_dump[len++] = counter_key[cnt]; + counter_dump[len++] = value; + } + } + } + + if (buffer) { + *buffer = (int*) counter_dump; + } + + return len; +} + +static struct gator_interface gator_events_mali_interface = { + .create_files = gator_events_mali_create_files, + .start = gator_events_mali_start, + .stop = gator_events_mali_stop, + .read = gator_events_mali_read, +}; + +int gator_events_mali_init(void) +{ + unsigned int cnt; + u32 id = gator_mali_get_id(); + + switch (id) { + case MALI_T6xx: + mali_name = "Mali-T6xx"; + break; + case MALI_400: + mali_name = "Mali-400"; + break; + case MALI_300: + mali_name = "Mali-300"; + break; + case MALI_200: + mali_name = "Mali-200"; + break; + default: + printk("Unknown Mali ID (%d)\n", id); + return -1; + } + + pr_debug("gator: mali init\n"); + + for (cnt = FIRST_ACTIVITY_EVENT; cnt < NUMBER_OF_EVENTS; cnt++) { + counter_enabled[cnt] = 0; + counter_event[cnt] = 0; + counter_key[cnt] = gator_events_get_key(); + counter_address[cnt] = NULL; + counter_data[cnt] = 0; + } + + trace_registered = 0; + + return gator_events_install(&gator_events_mali_interface); +} +gator_events_init(gator_events_mali_init); diff --git a/driver/gator_events_meminfo.c b/driver/gator_events_meminfo.c index a1a2031..8af9cfc 100644 --- a/driver/gator_events_meminfo.c +++ b/driver/gator_events_meminfo.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2011. All rights reserved. + * Copyright (C) ARM Limited 2010-2012. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as diff --git a/driver/gator_events_mmaped.c b/driver/gator_events_mmaped.c index 6884684..cbb22b1 100644 --- a/driver/gator_events_mmaped.c +++ b/driver/gator_events_mmaped.c @@ -1,13 +1,15 @@ /* * Example events provider * - * Copyright (C) ARM Limited 2010-2011. All rights reserved. + * Copyright (C) ARM Limited 2010-2012. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * - * Similar entires must be present in events.xml file: + * Similar entries to those below must be present in the events.xml file. + * To add them to the events.xml, create an events-mmap.xml with the + * following contents and rebuild gatord: * * <counter_set name="mmaped_cntX"> * <counter name="mmaped_cnt0"/> diff --git a/driver/gator_events_net.c b/driver/gator_events_net.c index a921586..ef1623b 100644 --- a/driver/gator_events_net.c +++ b/driver/gator_events_net.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2011. All rights reserved. + * Copyright (C) ARM Limited 2010-2012. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -12,13 +12,10 @@ #define NETRX 0 #define NETTX 1 -#define NETDRV 2 -#define TOTALNET (NETDRV+1) +#define TOTALNET 2 -static ulong netdrv_enabled; static ulong netrx_enabled; static ulong nettx_enabled; -static ulong netdrv_key; static ulong netrx_key; static ulong nettx_key; static int rx_total, tx_total; @@ -44,19 +41,9 @@ static void get_network_stats(struct work_struct *wsptr) { } DECLARE_WORK(wq_get_stats, get_network_stats); -static void calculate_delta(int *drv, int *rx, int *tx) +static void calculate_delta(int *rx, int *tx) { - int drv_calc, rx_calc, tx_calc; - - drv_calc = gator_net_traffic - netPrev[NETDRV]; - if (drv_calc > 0) { - netPrev[NETDRV] += drv_calc; - netPrev[NETTX] += drv_calc; - // remove tcp/ip header overhead - // approximation based on empirical measurement - netPrev[NETRX] += drv_calc / 42; - netPrev[NETTX] += drv_calc / 18; - } + int rx_calc, tx_calc; rx_calc = (int)(rx_total - netPrev[NETRX]); if (rx_calc < 0) @@ -68,7 +55,6 @@ static void calculate_delta(int *drv, int *rx, int *tx) tx_calc = 0; netPrev[NETTX] += tx_calc; - *drv = drv_calc; *rx = rx_calc; *tx = tx_calc; } @@ -77,13 +63,6 @@ static int gator_events_net_create_files(struct super_block *sb, struct dentry * { struct dentry *dir; - dir = gatorfs_mkdir(sb, root, "Linux_net_drv"); - if (!dir) { - return -1; - } - gatorfs_create_ulong(sb, dir, "enabled", &netdrv_enabled); - gatorfs_create_ro_ulong(sb, dir, "key", &netdrv_key); - dir = gatorfs_mkdir(sb, root, "Linux_net_rx"); if (!dir) { return -1; @@ -104,7 +83,6 @@ static int gator_events_net_create_files(struct super_block *sb, struct dentry * static int gator_events_net_start(void) { get_network_stats(NULL); - netPrev[NETDRV] = 0; netPrev[NETRX] = rx_total; netPrev[NETTX] = tx_total; return 0; @@ -112,29 +90,22 @@ static int gator_events_net_start(void) static void gator_events_net_stop(void) { - netdrv_enabled = 0; netrx_enabled = 0; nettx_enabled = 0; } static int gator_events_net_read(int **buffer) { - int len, drv_delta, rx_delta, tx_delta; - static int last_drv_delta = 0, last_rx_delta = 0, last_tx_delta = 0; + int len, rx_delta, tx_delta; + static int last_rx_delta = 0, last_tx_delta = 0; if (smp_processor_id() != 0) return 0; schedule_work(&wq_get_stats); - calculate_delta(&drv_delta, &rx_delta, &tx_delta); + calculate_delta(&rx_delta, &tx_delta); len = 0; - if (netdrv_enabled && last_drv_delta != drv_delta) { - last_drv_delta = drv_delta; - netGet[len++] = netdrv_key; - netGet[len++] = drv_delta; - } - if (netrx_enabled && last_rx_delta != rx_delta) { last_rx_delta = rx_delta; netGet[len++] = netrx_key; @@ -162,13 +133,9 @@ static struct gator_interface gator_events_net_interface = { int gator_events_net_init(void) { - gator_net_traffic++; - - netdrv_key = gator_events_get_key(); netrx_key = gator_events_get_key(); nettx_key = gator_events_get_key(); - netdrv_enabled = 0; netrx_enabled = 0; nettx_enabled = 0; diff --git a/driver/gator_events_perf_pmu.c b/driver/gator_events_perf_pmu.c new file mode 100644 index 0000000..322ebc4 --- /dev/null +++ b/driver/gator_events_perf_pmu.c @@ -0,0 +1,296 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/slab.h> +#include <linux/perf_event.h> +#include "gator.h" + +// gator_events_armvX.c is used for Linux 2.6.x +#if GATOR_PERF_PMU_SUPPORT + +static const char *pmnc_name; +int pmnc_counters; +int ccnt = 0; + +#define CNTMAX (6+1) + +unsigned long pmnc_enabled[CNTMAX]; +unsigned long pmnc_event[CNTMAX]; +unsigned long pmnc_count[CNTMAX]; +unsigned long pmnc_key[CNTMAX]; + +static DEFINE_PER_CPU(int[CNTMAX], perfCurr); +static DEFINE_PER_CPU(int[CNTMAX], perfPrev); +static DEFINE_PER_CPU(int[CNTMAX], perfPrevDelta); +static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt); +static DEFINE_PER_CPU(struct perf_event *[CNTMAX], pevent); +static DEFINE_PER_CPU(struct perf_event_attr *[CNTMAX], pevent_attr); + +static void gator_events_perf_pmu_stop(void); + +static int gator_events_perf_pmu_create_files(struct super_block *sb, struct dentry *root) +{ + struct dentry *dir; + int i; + + for (i = 0; i < pmnc_counters; i++) { + char buf[40]; + if (i == 0) { + snprintf(buf, sizeof buf, "%s_ccnt", pmnc_name); + } else { + snprintf(buf, sizeof buf, "%s_cnt%d", pmnc_name, i-1); + } + dir = gatorfs_mkdir(sb, root, buf); + if (!dir) { + return -1; + } + gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); + gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]); + gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); + if (i > 0) { + gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); + } + } + + return 0; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0) +static void dummy_handler(struct perf_event *event, int unused, struct perf_sample_data *data, struct pt_regs *regs) +#else +static void dummy_handler(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs) +#endif +{ +// Required as perf_event_create_kernel_counter() requires an overflow handler, even though all we do is poll +} + +static int gator_events_perf_pmu_online(int** buffer) +{ + int cnt, len = 0, cpu = smp_processor_id(); + + // read the counters and toss the invalid data, return zero instead + for (cnt = 0; cnt < pmnc_counters; cnt++) { + struct perf_event * ev = per_cpu(pevent, cpu)[cnt]; + if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) { + ev->pmu->read(ev); + per_cpu(perfPrev, cpu)[cnt] = per_cpu(perfCurr, cpu)[cnt] = local64_read(&ev->count); + per_cpu(perfPrevDelta, cpu)[cnt] = 0; + per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; + per_cpu(perfCnt, cpu)[len++] = 0; + } + } + + if (buffer) + *buffer = per_cpu(perfCnt, cpu); + + return len; +} + +static void gator_events_perf_pmu_online_dispatch(int cpu) +{ + int cnt; + + for (cnt = 0; cnt < pmnc_counters; cnt++) { + if (per_cpu(pevent, cpu)[cnt] != NULL || per_cpu(pevent_attr, cpu)[cnt] == 0) + continue; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0) + per_cpu(pevent, cpu)[cnt] = perf_event_create_kernel_counter(per_cpu(pevent_attr, cpu)[cnt], cpu, 0, dummy_handler); +#else + per_cpu(pevent, cpu)[cnt] = perf_event_create_kernel_counter(per_cpu(pevent_attr, cpu)[cnt], cpu, 0, dummy_handler, 0); +#endif + if (IS_ERR(per_cpu(pevent, cpu)[cnt])) { + pr_debug("gator: unable to online a counter on cpu %d\n", cpu); + per_cpu(pevent, cpu)[cnt] = NULL; + continue; + } + + if (per_cpu(pevent, cpu)[cnt]->state != PERF_EVENT_STATE_ACTIVE) { + pr_debug("gator: inactive counter on cpu %d\n", cpu); + perf_event_release_kernel(per_cpu(pevent, cpu)[cnt]); + per_cpu(pevent, cpu)[cnt] = NULL; + continue; + } + } +} + +static void gator_events_perf_pmu_offline_dispatch(int cpu) +{ + int cnt; + + for (cnt = 0; cnt < pmnc_counters; cnt++) { + if (per_cpu(pevent, cpu)[cnt] != NULL) { + perf_event_release_kernel(per_cpu(pevent, cpu)[cnt]); + per_cpu(pevent, cpu)[cnt] = NULL; + } + } +} + +static int gator_events_perf_pmu_start(void) +{ + int cnt, cpu; + u32 size = sizeof(struct perf_event_attr); + + for_each_present_cpu(cpu) { + for (cnt = 0; cnt < pmnc_counters; cnt++) { + per_cpu(pevent, cpu)[cnt] = NULL; + if (!pmnc_enabled[cnt] || pmnc_count[cnt] > 0) // Skip disabled counters and EBS counters + continue; + + per_cpu(perfPrev, cpu)[cnt] = 0; + per_cpu(perfCurr, cpu)[cnt] = 0; + per_cpu(perfPrevDelta, cpu)[cnt] = 0; + per_cpu(pevent_attr, cpu)[cnt] = kmalloc(size, GFP_KERNEL); + if (!per_cpu(pevent_attr, cpu)[cnt]) { + gator_events_perf_pmu_stop(); + return -1; + } + + memset(per_cpu(pevent_attr, cpu)[cnt], 0, size); + per_cpu(pevent_attr, cpu)[cnt]->type = PERF_TYPE_RAW; + per_cpu(pevent_attr, cpu)[cnt]->size = size; + per_cpu(pevent_attr, cpu)[cnt]->config = pmnc_event[cnt]; + per_cpu(pevent_attr, cpu)[cnt]->sample_period = 0; + per_cpu(pevent_attr, cpu)[cnt]->pinned = 1; + + // handle special case for ccnt + if (cnt == ccnt) { + per_cpu(pevent_attr, cpu)[cnt]->type = PERF_TYPE_HARDWARE; + per_cpu(pevent_attr, cpu)[cnt]->config = PERF_COUNT_HW_CPU_CYCLES; + } + } + } + + return 0; +} + +static void gator_events_perf_pmu_stop(void) +{ + unsigned int cnt, cpu; + + for_each_present_cpu(cpu) { + for (cnt = 0; cnt < pmnc_counters; cnt++) { + if (per_cpu(pevent_attr, cpu)[cnt]) { + kfree(per_cpu(pevent_attr, cpu)[cnt]); + per_cpu(pevent_attr, cpu)[cnt] = NULL; + } + } + } + + for (cnt = 0; cnt < pmnc_counters; cnt++) { + pmnc_enabled[cnt] = 0; + pmnc_event[cnt] = 0; + pmnc_count[cnt] = 0; + } +} + +static int gator_events_perf_pmu_read(int **buffer) +{ + int cnt, delta, len = 0; + int cpu = smp_processor_id(); + + for (cnt = 0; cnt < pmnc_counters; cnt++) { + struct perf_event * ev = per_cpu(pevent, cpu)[cnt]; + if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) { + ev->pmu->read(ev); + per_cpu(perfCurr, cpu)[cnt] = local64_read(&ev->count); + delta = per_cpu(perfCurr, cpu)[cnt] - per_cpu(perfPrev, cpu)[cnt]; + if (delta != per_cpu(perfPrevDelta, cpu)[cnt]) { + per_cpu(perfPrevDelta, cpu)[cnt] = delta; + per_cpu(perfPrev, cpu)[cnt] = per_cpu(perfCurr, cpu)[cnt]; + per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; + if (delta < 0) + delta *= -1; + per_cpu(perfCnt, cpu)[len++] = delta; + } + } + } + + if (buffer) + *buffer = per_cpu(perfCnt, cpu); + + return len; +} + +static struct gator_interface gator_events_perf_pmu_interface = { + .create_files = gator_events_perf_pmu_create_files, + .start = gator_events_perf_pmu_start, + .stop = gator_events_perf_pmu_stop, + .online = gator_events_perf_pmu_online, + .online_dispatch = gator_events_perf_pmu_online_dispatch, + .offline_dispatch = gator_events_perf_pmu_offline_dispatch, + .read = gator_events_perf_pmu_read, +}; + +int gator_events_perf_pmu_init(void) +{ + unsigned int cnt; + + switch (gator_cpuid()) { + case ARM1136: + case ARM1156: + case ARM1176: + pmnc_name = "ARM_ARM11"; + pmnc_counters = 3; + ccnt = 2; + break; + case ARM11MPCORE: + pmnc_name = "ARM_ARM11MPCore"; + pmnc_counters = 3; + break; + case CORTEX_A5: + pmnc_name = "ARM_Cortex-A5"; + pmnc_counters = 2; + break; + case CORTEX_A7: + pmnc_name = "ARM_Cortex-A7"; + pmnc_counters = 4; + break; + case CORTEX_A8: + pmnc_name = "ARM_Cortex-A8"; + pmnc_counters = 4; + break; + case CORTEX_A9: + pmnc_name = "ARM_Cortex-A9"; + pmnc_counters = 6; + break; + case CORTEX_A15: + pmnc_name = "ARM_Cortex-A15"; + pmnc_counters = 6; + break; + case SCORPION: + pmnc_name = "Scorpion"; + pmnc_counters = 4; + break; + case SCORPIONMP: + pmnc_name = "ScorpionMP"; + pmnc_counters = 4; + break; + case KRAITSIM: + case KRAIT: + pmnc_name = "Krait"; + pmnc_counters = 4; + break; + default: + return -1; + } + + pmnc_counters++; // CNT[n] + CCNT + + for (cnt = 0; cnt < CNTMAX; cnt++) { + pmnc_enabled[cnt] = 0; + pmnc_event[cnt] = 0; + pmnc_count[cnt] = 0; + pmnc_key[cnt] = gator_events_get_key(); + } + + return gator_events_install(&gator_events_perf_pmu_interface); +} + +gator_events_init(gator_events_perf_pmu_init); +#endif diff --git a/driver/gator_events_power.c b/driver/gator_events_power.c new file mode 100644 index 0000000..a0ae684 --- /dev/null +++ b/driver/gator_events_power.c @@ -0,0 +1,178 @@ +/** + * Copyright (C) ARM Limited 2011-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include "gator.h" +#include <linux/slab.h> +#include <linux/cpufreq.h> +#include <trace/events/power.h> + +// cpu_frequency and cpu_idle trace points were introduced in Linux kernel v2.6.38 +// the now deprecated power_frequency trace point was available prior to 2.6.38, but only for x86 +// CPU Idle is currently disabled in the .xml +#if GATOR_CPU_FREQ_SUPPORT +enum { + POWER_CPU_FREQ, + POWER_CPU_IDLE, + POWER_TOTAL +}; + +static ulong power_cpu_enabled[POWER_TOTAL]; +static ulong power_cpu_key[POWER_TOTAL]; +static DEFINE_PER_CPU(ulong[POWER_TOTAL], power); +static DEFINE_PER_CPU(ulong[POWER_TOTAL], prev); +static DEFINE_PER_CPU(int *, powerGet); + +GATOR_DEFINE_PROBE(cpu_frequency, TP_PROTO(unsigned int frequency, unsigned int cpu)) +{ + per_cpu(power, cpu)[POWER_CPU_FREQ] = frequency * 1000; +} + +GATOR_DEFINE_PROBE(cpu_idle, TP_PROTO(unsigned int state, unsigned int cpu)) +{ + per_cpu(power, cpu)[POWER_CPU_IDLE] = state; +} + +static int gator_events_power_create_files(struct super_block *sb, struct dentry *root) +{ + struct dentry *dir; + + // cpu_frequency + dir = gatorfs_mkdir(sb, root, "Linux_power_cpu_freq"); + if (!dir) { + return -1; + } + gatorfs_create_ulong(sb, dir, "enabled", &power_cpu_enabled[POWER_CPU_FREQ]); + gatorfs_create_ro_ulong(sb, dir, "key", &power_cpu_key[POWER_CPU_FREQ]); + + // cpu_idle + dir = gatorfs_mkdir(sb, root, "Linux_power_cpu_idle"); + if (!dir) { + return -1; + } + gatorfs_create_ulong(sb, dir, "enabled", &power_cpu_enabled[POWER_CPU_IDLE]); + gatorfs_create_ro_ulong(sb, dir, "key", &power_cpu_key[POWER_CPU_IDLE]); + + return 0; +} + +static int gator_events_power_populate(int cpu, int** buffer) +{ + int i, len = 0; + + for (i = 0; i < POWER_TOTAL; i++) { + if (power_cpu_enabled[i]) { + if (per_cpu(power, cpu)[i] != per_cpu(prev, cpu)[i]) { + per_cpu(prev, cpu)[i] = per_cpu(power, cpu)[i]; + per_cpu(powerGet, cpu)[len++] = power_cpu_key[i]; + per_cpu(powerGet, cpu)[len++] = per_cpu(power, cpu)[i]; + } + } + } + + if (buffer) + *buffer = per_cpu(powerGet, cpu); + + return len; +} + +static int gator_events_power_online(int** buffer) +{ + int i, cpu = smp_processor_id(); + for (i = 0; i < POWER_TOTAL; i++) + per_cpu(prev, cpu)[i] = -1; + per_cpu(power, cpu)[POWER_CPU_FREQ] = cpufreq_quick_get(cpu) * 1000; + return gator_events_power_populate(cpu, buffer); +} + +static int gator_events_power_offline(int** buffer) +{ + int cpu = smp_processor_id(); + // Set frequency to zero on an offline + per_cpu(power, cpu)[POWER_CPU_FREQ] = 0; + return gator_events_power_populate(cpu, buffer); +} + +static int gator_events_power_start(void) +{ + int cpu; + + for_each_present_cpu(cpu) { + per_cpu(powerGet, cpu) = kmalloc(POWER_TOTAL * 2, GFP_KERNEL); + if (!per_cpu(powerGet, cpu)) + return -1; + } + + // register tracepoints + if (power_cpu_enabled[POWER_CPU_FREQ]) + if (GATOR_REGISTER_TRACE(cpu_frequency)) + goto fail_cpu_frequency_exit; + if (power_cpu_enabled[POWER_CPU_IDLE]) + if (GATOR_REGISTER_TRACE(cpu_idle)) + goto fail_cpu_idle_exit; + pr_debug("gator: registered power event tracepoints\n"); + + return 0; + + // unregister tracepoints on error +fail_cpu_idle_exit: + GATOR_UNREGISTER_TRACE(cpu_frequency); +fail_cpu_frequency_exit: + pr_err("gator: power event tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n"); + + return -1; +} + +static void gator_events_power_stop(void) +{ + int i, cpu; + if (power_cpu_enabled[POWER_CPU_FREQ]) + GATOR_UNREGISTER_TRACE(cpu_frequency); + if (power_cpu_enabled[POWER_CPU_IDLE]) + GATOR_UNREGISTER_TRACE(cpu_idle); + pr_debug("gator: unregistered power event tracepoints\n"); + + for (i = 0; i < POWER_TOTAL; i++) { + power_cpu_enabled[i] = 0; + } + + for_each_present_cpu(cpu) { + kfree(per_cpu(powerGet, cpu)); + } +} + +static int gator_events_power_read(int **buffer) +{ + return gator_events_power_populate(smp_processor_id(), buffer); +} + +static struct gator_interface gator_events_power_interface = { + .create_files = gator_events_power_create_files, + .online = gator_events_power_online, + .offline = gator_events_power_offline, + .start = gator_events_power_start, + .stop = gator_events_power_stop, + .read = gator_events_power_read, +}; +#endif + +int gator_events_power_init(void) +{ +#if (GATOR_CPU_FREQ_SUPPORT) + int i; + for (i = 0; i < POWER_TOTAL; i++) { + power_cpu_enabled[i] = 0; + power_cpu_key[i] = gator_events_get_key(); + } + + return gator_events_install(&gator_events_power_interface); +#else + return -1; +#endif +} +gator_events_init(gator_events_power_init); diff --git a/driver/gator_events_sched.c b/driver/gator_events_sched.c index 7e9db60..9bed364 100644 --- a/driver/gator_events_sched.c +++ b/driver/gator_events_sched.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2011. All rights reserved. + * Copyright (C) ARM Limited 2010-2012. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as diff --git a/driver/gator_events_scorpion.c b/driver/gator_events_scorpion.c index f51e292..477e7c9 100644 --- a/driver/gator_events_scorpion.c +++ b/driver/gator_events_scorpion.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2011. All rights reserved. + * Copyright (C) ARM Limited 2011-2012. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -8,8 +8,8 @@ #include "gator.h" -#define SCORPION 0xf -#define SCORPIONMP 0x2d +// gator_events_perf_pmu.c is used if perf is supported +#if GATOR_NO_PERF_SUPPORT static const char *pmnc_name; static int pmnc_counters; @@ -32,7 +32,6 @@ static int pmnc_counters; static unsigned long pmnc_enabled[CNTMAX]; static unsigned long pmnc_event[CNTMAX]; -static unsigned long pmnc_count[CNTMAX]; static unsigned long pmnc_key[CNTMAX]; static DEFINE_PER_CPU(int[CNTMAX], perfPrev); @@ -516,7 +515,6 @@ static int gator_events_scorpion_create_files(struct super_block *sb, struct den return -1; } gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); - gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]); gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); if (i > 0) { gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); @@ -526,9 +524,9 @@ static int gator_events_scorpion_create_files(struct super_block *sb, struct den return 0; } -static void gator_events_scorpion_online(void) +static int gator_events_scorpion_online(int** buffer) { - unsigned int cnt; + unsigned int cnt, len = 0, cpu = smp_processor_id(); if (scorpion_pmnc_read() & PMNC_E) { scorpion_pmnc_write(scorpion_pmnc_read() & ~PMNC_E); @@ -563,26 +561,34 @@ static void gator_events_scorpion_online(void) // enable scorpion_pmnc_write(scorpion_pmnc_read() | PMNC_E); -} -static void gator_events_scorpion_offline(void) -{ - scorpion_pmnc_write(scorpion_pmnc_read() & ~PMNC_E); + // read the counters and toss the invalid data, return zero instead + for (cnt = 0; cnt < pmnc_counters; cnt++) { + if (pmnc_enabled[cnt]) { + int value; + if (cnt == CCNT) { + value = scorpion_ccnt_read(); + } else if (scorpion_pmnc_select_counter(cnt) == cnt) { + value = scorpion_cntn_read(); + } else { + value = 0; + } + scorpion_pmnc_reset_counter(cnt); + per_cpu(perfPrev, cpu)[cnt] = 0; + per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; + per_cpu(perfCnt, cpu)[len++] = 0; + } + } - // investigate: need to do the clearpmu() here on each counter? + if (buffer) + *buffer = per_cpu(perfCnt, cpu); + + return len; } -static int gator_events_scorpion_start(void) +static int gator_events_scorpion_offline(int** buffer) { - int cnt; - - for (cnt = CCNT; cnt < CNTMAX; cnt++) { - if (pmnc_count[cnt] > 0) { - pr_err("gator: event based sampling not supported on Scorpion cores\n"); - return -1; - } - } - + scorpion_pmnc_write(scorpion_pmnc_read() & ~PMNC_E); return 0; } @@ -593,7 +599,6 @@ static void gator_events_scorpion_stop(void) for (cnt = CCNT; cnt < CNTMAX; cnt++) { pmnc_enabled[cnt] = 0; pmnc_event[cnt] = 0; - pmnc_count[cnt] = 0; } } @@ -602,9 +607,6 @@ static int gator_events_scorpion_read(int **buffer) int cnt, len = 0; int cpu = smp_processor_id(); - if (!pmnc_counters) - return 0; - for (cnt = 0; cnt < pmnc_counters; cnt++) { if (pmnc_enabled[cnt]) { int value; @@ -624,7 +626,6 @@ static int gator_events_scorpion_read(int **buffer) } } - // update or discard if (buffer) *buffer = per_cpu(perfCnt, cpu); @@ -633,25 +634,12 @@ static int gator_events_scorpion_read(int **buffer) static struct gator_interface gator_events_scorpion_interface = { .create_files = gator_events_scorpion_create_files, - .start = gator_events_scorpion_start, .stop = gator_events_scorpion_stop, .online = gator_events_scorpion_online, .offline = gator_events_scorpion_offline, .read = gator_events_scorpion_read, }; - -static void scorpion_clear_pmuregs(void) -{ - scorpion_write_lpm0(0); - scorpion_write_lpm1(0); - scorpion_write_lpm2(0); - scorpion_write_l2lpm(0); - scorpion_pre_vlpm(); - scorpion_write_vlpm(0); - scorpion_post_vlpm(); -} - int gator_events_scorpion_init(void) { unsigned int cnt; @@ -674,12 +662,17 @@ int gator_events_scorpion_init(void) for (cnt = CCNT; cnt < CNTMAX; cnt++) { pmnc_enabled[cnt] = 0; pmnc_event[cnt] = 0; - pmnc_count[cnt] = 0; pmnc_key[cnt] = gator_events_get_key(); } - scorpion_clear_pmuregs(); - return gator_events_install(&gator_events_scorpion_interface); } + gator_events_init(gator_events_scorpion_init); + +#else +int gator_events_scorpion_init(void) +{ + return -1; +} +#endif diff --git a/driver/gator_hrtimer_gator.c b/driver/gator_hrtimer_gator.c new file mode 100644 index 0000000..5896b3c --- /dev/null +++ b/driver/gator_hrtimer_gator.c @@ -0,0 +1,76 @@ +/** + * Copyright (C) ARM Limited 2011-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +// gator_hrtimer_perf.c is used if perf is supported +// update, gator_hrtimer_gator.c always used until issues resolved with perf hrtimers +#if 1 + +void (*callback)(void); +DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer); +static ktime_t profiling_interval; +static void gator_hrtimer_online(int cpu); +static void gator_hrtimer_offline(int cpu); + +static enum hrtimer_restart gator_hrtimer_notify(struct hrtimer *hrtimer) +{ + hrtimer_forward_now(hrtimer, profiling_interval); + (*callback)(); + return HRTIMER_RESTART; +} + +static void gator_hrtimer_switch_cpus_online(void *unused) +{ + gator_hrtimer_online(smp_processor_id()); +} + +static void gator_hrtimer_online(int cpu) +{ + struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu); + if (cpu != smp_processor_id()) { + smp_call_function_single(cpu, gator_hrtimer_switch_cpus_online, NULL, 1); + return; + } + + hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + hrtimer->function = gator_hrtimer_notify; + hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED); +} + +static void gator_hrtimer_switch_cpus_offline(void *unused) +{ + gator_hrtimer_offline(smp_processor_id()); +} + +static void gator_hrtimer_offline(int cpu) +{ + struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu); + if (cpu != smp_processor_id()) { + smp_call_function_single(cpu, gator_hrtimer_switch_cpus_offline, NULL, 1); + return; + } + + hrtimer_cancel(hrtimer); +} + +static int gator_hrtimer_init(int interval, void (*func)(void)) +{ + (callback) = (func); + + // calculate profiling interval + profiling_interval = ns_to_ktime(1000000000UL / interval); + + return 0; +} + +static void gator_hrtimer_shutdown(void) +{ + /* empty */ +} + +#endif diff --git a/driver/gator_hrtimer_perf.c b/driver/gator_hrtimer_perf.c new file mode 100644 index 0000000..7c0333f --- /dev/null +++ b/driver/gator_hrtimer_perf.c @@ -0,0 +1,113 @@ +/** + * Copyright (C) ARM Limited 2011-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +// gator_hrtimer_gator.c is used if perf is not supported +// update, gator_hrtimer_gator.c always used until issues resolved with perf hrtimers +#if 0 + +// Note: perf Cortex support added in 2.6.35 and PERF_COUNT_SW_CPU_CLOCK/hrtimer broken on 2.6.35 and 2.6.36 +// not relevant as this code is not active until 3.0.0, but wanted to document the issue + +void (*callback)(void); +static int profiling_interval; +static DEFINE_PER_CPU(struct perf_event *, perf_hrtimer); +static DEFINE_PER_CPU(struct perf_event_attr *, perf_hrtimer_attr); + +static void gator_hrtimer_shutdown(void); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0) +static void hrtimer_overflow_handler(struct perf_event *event, int unused, struct perf_sample_data *data, struct pt_regs *regs) +#else +static void hrtimer_overflow_handler(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs) +#endif +{ + (*callback)(); +} + +static int gator_online_single_hrtimer(int cpu) +{ + if (per_cpu(perf_hrtimer, cpu) != 0 || per_cpu(perf_hrtimer_attr, cpu) == 0) + return 0; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0) + per_cpu(perf_hrtimer, cpu) = perf_event_create_kernel_counter(per_cpu(perf_hrtimer_attr, cpu), cpu, 0, hrtimer_overflow_handler); +#else + per_cpu(perf_hrtimer, cpu) = perf_event_create_kernel_counter(per_cpu(perf_hrtimer_attr, cpu), cpu, 0, hrtimer_overflow_handler, 0); +#endif + if (IS_ERR(per_cpu(perf_hrtimer, cpu))) { + per_cpu(perf_hrtimer, cpu) = NULL; + return -1; + } + + if (per_cpu(perf_hrtimer, cpu)->state != PERF_EVENT_STATE_ACTIVE) { + perf_event_release_kernel(per_cpu(perf_hrtimer, cpu)); + per_cpu(perf_hrtimer, cpu) = NULL; + return -1; + } + + return 0; +} + +static void gator_hrtimer_online(int cpu) +{ + if (gator_online_single_hrtimer(cpu) < 0) { + pr_debug("gator: unable to online the hrtimer on cpu%d\n", cpu); + } +} + +static void gator_hrtimer_offline(int cpu) +{ + if (per_cpu(perf_hrtimer, cpu)) { + perf_event_release_kernel(per_cpu(perf_hrtimer, cpu)); + per_cpu(perf_hrtimer, cpu) = NULL; + } +} + +static int gator_hrtimer_init(int interval, void (*func)(void)) +{ + u32 size = sizeof(struct perf_event_attr); + int cpu; + + callback = func; + + // calculate profiling interval + profiling_interval = 1000000000 / interval; + + for_each_present_cpu(cpu) { + per_cpu(perf_hrtimer, cpu) = 0; + per_cpu(perf_hrtimer_attr, cpu) = kmalloc(size, GFP_KERNEL); + if (per_cpu(perf_hrtimer_attr, cpu) == 0) { + gator_hrtimer_shutdown(); + return -1; + } + + memset(per_cpu(perf_hrtimer_attr, cpu), 0, size); + per_cpu(perf_hrtimer_attr, cpu)->type = PERF_TYPE_SOFTWARE; + per_cpu(perf_hrtimer_attr, cpu)->size = size; + per_cpu(perf_hrtimer_attr, cpu)->config = PERF_COUNT_SW_CPU_CLOCK; + per_cpu(perf_hrtimer_attr, cpu)->sample_period = profiling_interval; + per_cpu(perf_hrtimer_attr, cpu)->pinned = 1; + } + + return 0; +} + +static void gator_hrtimer_shutdown(void) +{ + int cpu; + + for_each_present_cpu(cpu) { + if (per_cpu(perf_hrtimer_attr, cpu)) { + kfree(per_cpu(perf_hrtimer_attr, cpu)); + per_cpu(perf_hrtimer_attr, cpu) = NULL; + } + } +} + +#endif diff --git a/driver/gator_main.c b/driver/gator_main.c index 36e951b..fff2d19 100644 --- a/driver/gator_main.c +++ b/driver/gator_main.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2011. All rights reserved. + * Copyright (C) ARM Limited 2010-2012. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -7,7 +7,7 @@ * */ -static unsigned long gator_protocol_version = 7; +static unsigned long gator_protocol_version = 8; #include <linux/slab.h> #include <linux/cpu.h> @@ -18,16 +18,20 @@ static unsigned long gator_protocol_version = 7; #include <linux/highmem.h> #include <linux/pagemap.h> #include <linux/suspend.h> +#include <linux/module.h> +#include <linux/perf_event.h> #include <asm/stacktrace.h> #include <asm/uaccess.h> #include "gator.h" #include "gator_events.h" -#ifndef CONFIG_GENERIC_TRACER -#ifndef CONFIG_TRACING -#error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32) +#error kernels prior to 2.6.32 are not supported #endif + +#if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING) +#error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined #endif #ifndef CONFIG_PROFILING @@ -38,16 +42,20 @@ static unsigned long gator_protocol_version = 7; #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined #endif -#if defined (__arm__) -#ifdef CONFIG_SMP -#ifndef CONFIG_LOCAL_TIMERS +#if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS) #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems #endif + +#if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT)) +#ifndef CONFIG_PERF_EVENTS +#warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters +#elif !defined CONFIG_HW_PERF_EVENTS +#warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters #endif #endif -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32) -#error kernels prior to 2.6.32 are not supported +#if (!(GATOR_CPU_FREQ_SUPPORT)) +#warning gator requires kernel version 2.6.38 or greater and CONFIG_CPU_FREQ defined in order to enable the CPU Freq timeline chart #endif /****************************************************************************** @@ -69,8 +77,11 @@ static unsigned long gator_protocol_version = 7; #define MESSAGE_END_BACKTRACE 7 #define MESSAGE_SCHEDULER_TRACE 9 #define MESSAGE_PID_NAME 11 +#define MESSAGE_GPU_TRACE 13 +#define MESSAGE_OVERFLOW 127 -#define LINUX_PMU_SUPPORT LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35) && defined(CONFIG_CPU_HAS_PMU) +#define MAXSIZE_PACK32 5 +#define MAXSIZE_PACK64 9 #if defined(__arm__) #define PC_REG regs->ARM_pc @@ -94,27 +105,16 @@ static unsigned long gator_streaming; static DEFINE_MUTEX(start_mutex); static DEFINE_MUTEX(gator_buffer_mutex); -unsigned long gator_net_traffic; bool event_based_sampling; -#define COMMIT_SIZE 128 -#define COMMIT_MASK (COMMIT_SIZE-1) -static DEFINE_SPINLOCK(timer_commit_lock); -static int *gator_commit[NUM_GATOR_BUFS]; -static int gator_commit_read[NUM_GATOR_BUFS]; -static int gator_commit_write[NUM_GATOR_BUFS]; - static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait); -static DEFINE_PER_CPU(int, gator_first_time); -#if LINUX_PMU_SUPPORT -static void event_buffer_check(int cpu); -static DEFINE_SPINLOCK(event_commit_lock); -#endif +static void buffer_check(int cpu, int buftype); /****************************************************************************** * Prototypes ******************************************************************************/ +static bool buffer_check_space(int cpu, int buftype, int bytes); static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x); static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x); static void gator_buffer_write_string(int cpu, int buftype, char *x); @@ -124,15 +124,28 @@ static void gator_add_trace(int cpu, int buftype, unsigned int address); static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs); static uint64_t gator_get_time(void); +static uint32_t gator_buffer_size[NUM_GATOR_BUFS]; +static uint32_t gator_buffer_mask[NUM_GATOR_BUFS]; +static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read); +static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write); +static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit); +static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available); +static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer); +static DEFINE_PER_CPU(uint64_t, emit_overflow); + /****************************************************************************** * Application Includes ******************************************************************************/ +#include "gator_hrtimer_perf.c" +#include "gator_hrtimer_gator.c" #include "gator_cookies.c" #include "gator_trace_sched.c" +#include "gator_trace_gpu.c" #include "gator_backtrace.c" #include "gator_annotate.c" #include "gator_fs.c" #include "gator_ebs.c" +#include "gator_pack.c" /****************************************************************************** * Misc @@ -149,37 +162,55 @@ u32 gator_cpuid(void) /****************************************************************************** * Commit interface ******************************************************************************/ -static int buffer_commit_ready(int buftype) +static bool buffer_commit_ready(int* cpu, int* buftype) { - return gator_commit_read[buftype] != gator_commit_write[buftype]; -} - -static void buffer_commit_read(int *cpu, int buftype, int *readval, int *writeval) -{ - int read = gator_commit_read[buftype]; - *cpu = gator_commit[buftype][read+0]; - *readval = gator_commit[buftype][read+1]; - *writeval = gator_commit[buftype][read+2]; - gator_commit_read[buftype] = (read + 4) & COMMIT_MASK; -} - -static void buffer_commit_write(int cpu, int buftype, int readval, int writeval) { - int write = gator_commit_write[buftype]; - gator_commit[buftype][write+0] = cpu; - gator_commit[buftype][write+1] = readval; - gator_commit[buftype][write+2] = writeval; - gator_commit_write[buftype] = (write + 4) & COMMIT_MASK; + int cpu_x, x; + for_each_present_cpu(cpu_x) { + for (x = 0; x < NUM_GATOR_BUFS; x++) + if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) { + *cpu = cpu_x; + *buftype = x; + return true; + } + } + return false; } /****************************************************************************** * Buffer management ******************************************************************************/ -static uint32_t gator_buffer_size[NUM_GATOR_BUFS]; -static uint32_t gator_buffer_mask[NUM_GATOR_BUFS]; -static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read); -static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write); -static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer); -#include "gator_pack.c" +static bool buffer_check_space(int cpu, int buftype, int bytes) +{ + int remaining, filled; + + filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype]; + if (filled < 0) { + filled += gator_buffer_size[buftype]; + } + + remaining = gator_buffer_size[buftype] - filled; + + if (per_cpu(buffer_space_available, cpu)[buftype]) { + // Give some extra room; also allows space to insert the overflow error packet + remaining -= 200; + } else { + // Hysteresis, prevents multiple overflow messages + remaining -= 2000; + } + + if (remaining < bytes) { + if (per_cpu(buffer_space_available, cpu)[buftype] == true) { + // overflow packet to be emitted at a later time, as we may be in the middle of writing a message, e.g. counters + per_cpu(emit_overflow, cpu) = gator_get_time(); + pr_err("overflow: remaining = %d\n", gator_buffer_size[buftype] - filled); + } + per_cpu(buffer_space_available, cpu)[buftype] = false; + } else { + per_cpu(buffer_space_available, cpu)[buftype] = true; + } + + return per_cpu(buffer_space_available, cpu)[buftype]; +} static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len) { @@ -218,42 +249,24 @@ static void gator_buffer_header(int cpu, int buftype) gator_buffer_write_packed_int(cpu, buftype, cpu); } -static void gator_buffer_commit(int cpu, int buftype) +static void gator_commit_buffer(int cpu, int buftype) { - buffer_commit_write(cpu, buftype, per_cpu(gator_buffer_read, cpu)[buftype], per_cpu(gator_buffer_write, cpu)[buftype]); - per_cpu(gator_buffer_read, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype]; + per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype]; gator_buffer_header(cpu, buftype); wake_up(&gator_buffer_wait); } -static void timer_buffer_check(int cpu) +static void buffer_check(int cpu, int buftype) { - int available = per_cpu(gator_buffer_write, cpu)[TIMER_BUF] - per_cpu(gator_buffer_read, cpu)[TIMER_BUF]; - if (available < 0) { - available += gator_buffer_size[TIMER_BUF]; + int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype]; + if (filled < 0) { + filled += gator_buffer_size[buftype]; } - if (available >= ((gator_buffer_size[TIMER_BUF] * 3) / 4)) { - spin_lock(&timer_commit_lock); - gator_buffer_commit(cpu, TIMER_BUF); - spin_unlock(&timer_commit_lock); + if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) { + gator_commit_buffer(cpu, buftype); } } -#if LINUX_PMU_SUPPORT -static void event_buffer_check(int cpu) -{ - int available = per_cpu(gator_buffer_write, cpu)[EVENT_BUF] - per_cpu(gator_buffer_read, cpu)[EVENT_BUF]; - if (available < 0) { - available += gator_buffer_size[EVENT_BUF]; - } - if (available >= ((gator_buffer_size[EVENT_BUF] * 3) / 4)) { - spin_lock(&event_commit_lock); - gator_buffer_commit(cpu, EVENT_BUF); - spin_unlock(&event_commit_lock); - } -} -#endif - static void gator_add_trace(int cpu, int buftype, unsigned int address) { off_t offset = 0; @@ -309,21 +322,9 @@ static void gator_timer_interrupt(void) long long *buffer64; struct gator_interface *gi; - // check full backtrace has enough space, otherwise may - // have breaks between samples in the same callstack - if (per_cpu(gator_first_time, cpu)) { - per_cpu(gator_first_time, cpu) = 0; - - list_for_each_entry(gi, &gator_events, list) - if (gi->read) - gi->read(NULL); - - return; - } - - // Output scheduler + // Output scheduler trace len = gator_trace_sched_read(&buffer64); - if (len > 0) { + if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { gator_buffer_write_packed_int(cpu, buftype, MESSAGE_SCHEDULER_TRACE); gator_buffer_write_packed_int(cpu, buftype, len); for (i = 0; i < len; i++) { @@ -331,105 +332,189 @@ static void gator_timer_interrupt(void) } } + // Output GPU trace + len = gator_trace_gpu_read(&buffer64); + if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_GPU_TRACE); + gator_buffer_write_packed_int(cpu, buftype, len); + for (i = 0; i < len; i++) { + gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]); + } + } + // Output counters - gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); - gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); - list_for_each_entry(gi, &gator_events, list) { - if (gi->read) { - len = gi->read(&buffer); - if (len > 0) { - gator_buffer_write_packed_int(cpu, buftype, len); - for (i = 0; i < len; i++) { - gator_buffer_write_packed_int(cpu, buftype, buffer[i]); + if (buffer_check_space(cpu, buftype, MAXSIZE_PACK32 * 2 + MAXSIZE_PACK64)) { + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); + gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); + list_for_each_entry(gi, &gator_events, list) { + if (gi->read) { + len = gi->read(&buffer); + if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, buftype, len); + for (i = 0; i < len; i++) { + gator_buffer_write_packed_int(cpu, buftype, buffer[i]); + } } - } - } else if (gi->read64) { - len = gi->read64(&buffer64); - if (len > 0) { - gator_buffer_write_packed_int(cpu, buftype, len); - for (i = 0; i < len; i++) { - gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]); + } else if (gi->read64) { + len = gi->read64(&buffer64); + if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, buftype, len); + for (i = 0; i < len; i++) { + gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]); + } } } } + gator_buffer_write_packed_int(cpu, buftype, 0); } - gator_buffer_write_packed_int(cpu, buftype, 0); // Output backtrace - if (!event_based_sampling) { + if (!event_based_sampling && buffer_check_space(cpu, buftype, gator_backtrace_depth * 2 * MAXSIZE_PACK32)) gator_add_sample(cpu, buftype, regs); + + // Overflow message + if (per_cpu(emit_overflow, cpu)) { + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_OVERFLOW); + gator_buffer_write_packed_int64(cpu, buftype, per_cpu(emit_overflow, cpu)); + per_cpu(emit_overflow, cpu) = 0; } // Check and commit; generally, commit is set to occur once per second - timer_buffer_check(cpu); + buffer_check(cpu, buftype); } -DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer); DEFINE_PER_CPU(int, hrtimer_is_active); static int hrtimer_running; -static ktime_t profiling_interval; -static enum hrtimer_restart gator_hrtimer_notify(struct hrtimer *hrtimer) +// This function runs in interrupt context and on the appropriate core +static void gator_timer_offline(void* unused) { - hrtimer_forward_now(hrtimer, profiling_interval); - gator_timer_interrupt(); - return HRTIMER_RESTART; -} + int i, len, cpu = smp_processor_id(); + int* buffer; + long long* buffer64; -static int gator_timer_init(void) -{ - return 0; -} - -static void __gator_timer_offline(void *unused) -{ - int cpu = smp_processor_id(); if (per_cpu(hrtimer_is_active, cpu)) { struct gator_interface *gi; - struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu); - hrtimer_cancel(hrtimer); + gator_hrtimer_offline(cpu); per_cpu(hrtimer_is_active, cpu) = 0; - gator_buffer_commit(cpu, TIMER_BUF); - if (event_based_sampling) - gator_buffer_commit(cpu, EVENT_BUF); - - // offline any events - list_for_each_entry(gi, &gator_events, list) - if (gi->offline) - gi->offline(); + + // Output scheduler trace + len = gator_trace_sched_offline(&buffer64); + if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_SCHEDULER_TRACE); + gator_buffer_write_packed_int(cpu, TIMER_BUF, len); + for (i = 0; i < len; i++) { + gator_buffer_write_packed_int64(cpu, TIMER_BUF, buffer64[i]); + } + } + + // Output GPU trace + len = gator_trace_gpu_offline(&buffer64); + if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_GPU_TRACE); + gator_buffer_write_packed_int(cpu, TIMER_BUF, len); + for (i = 0; i < len; i++) { + gator_buffer_write_packed_int64(cpu, TIMER_BUF, buffer64[i]); + } + } + + // offline any events and output counters + gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_COUNTERS); + gator_buffer_write_packed_int64(cpu, TIMER_BUF, gator_get_time()); + list_for_each_entry(gi, &gator_events, list) { + if (gi->offline) { + len = gi->offline(&buffer); + if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, TIMER_BUF, len); + for (i = 0; i < len; i++) + gator_buffer_write_packed_int(cpu, TIMER_BUF, buffer[i]); + } + } + } + gator_buffer_write_packed_int(cpu, TIMER_BUF, 0); + + gator_commit_buffer(cpu, TIMER_BUF); + } + + if (event_based_sampling) { + gator_commit_buffer(cpu, EVENT_BUF); } } -static void gator_timer_offline(void) +// This function runs in interrupt context and may be running on a core other than core 'cpu' +static void gator_timer_offline_dispatch(int cpu) { + struct gator_interface *gi; + + list_for_each_entry(gi, &gator_events, list) + if (gi->offline_dispatch) + gi->offline_dispatch(cpu); + + gator_event_sampling_offline_dispatch(cpu); +} + +static void gator_timer_stop(void) +{ + int cpu; + if (hrtimer_running) { - hrtimer_running = 0; + on_each_cpu(gator_timer_offline, NULL, 1); + for_each_online_cpu(cpu) { + gator_timer_offline_dispatch(cpu); + } - on_each_cpu(__gator_timer_offline, NULL, 1); + hrtimer_running = 0; + gator_hrtimer_shutdown(); } } -static void __gator_timer_online(void *unused) +// This function runs in interrupt context and on the appropriate core +static void gator_timer_online(void* unused) { - int cpu = smp_processor_id(); + int i, len, cpu = smp_processor_id(); + int* buffer; + if (!per_cpu(hrtimer_is_active, cpu)) { struct gator_interface *gi; - struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu); - hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - hrtimer->function = gator_hrtimer_notify; - hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED); - per_cpu(gator_first_time, cpu) = 1; - per_cpu(hrtimer_is_active, cpu) = 1; - // online any events - list_for_each_entry(gi, &gator_events, list) - if (gi->online) - gi->online(); + // online any events and output counters + gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_COUNTERS); + gator_buffer_write_packed_int64(cpu, TIMER_BUF, gator_get_time()); + list_for_each_entry(gi, &gator_events, list) { + if (gi->online) { + len = gi->online(&buffer); + if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, TIMER_BUF, len); + for (i = 0; i < len; i++) + gator_buffer_write_packed_int(cpu, TIMER_BUF, buffer[i]); + } + } + } + gator_buffer_write_packed_int(cpu, TIMER_BUF, 0); + + gator_event_sampling_online(); + + gator_hrtimer_online(cpu); + per_cpu(hrtimer_is_active, cpu) = 1; } } -int gator_timer_online(unsigned long setup) +// This function runs in interrupt context and may be running on a core other than core 'cpu' +static void gator_timer_online_dispatch(int cpu) +{ + struct gator_interface *gi; + + list_for_each_entry(gi, &gator_events, list) + if (gi->online_dispatch) + gi->online_dispatch(cpu); + + gator_event_sampling_online_dispatch(cpu); +} + +int gator_timer_start(unsigned long setup) { + int cpu; + if (!setup) { pr_err("gator: cannot start due to a system tick value of zero\n"); return -1; @@ -440,11 +525,13 @@ int gator_timer_online(unsigned long setup) hrtimer_running = 1; - // calculate profiling interval - profiling_interval = ns_to_ktime(1000000000UL / setup); + if (gator_hrtimer_init(setup, gator_timer_interrupt) == -1) + return -1; - // timer interrupt - on_each_cpu(__gator_timer_online, NULL, 1); + for_each_online_cpu(cpu) { + gator_timer_online_dispatch(cpu); + } + on_each_cpu(gator_timer_online, NULL, 1); return 0; } @@ -461,7 +548,7 @@ static uint64_t gator_get_time(void) } /****************************************************************************** - * cpu online and pm notifiers + * cpu hotplug and pm notifiers ******************************************************************************/ static int __cpuinit gator_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { @@ -470,11 +557,13 @@ static int __cpuinit gator_cpu_notify(struct notifier_block *self, unsigned long switch (action) { case CPU_DOWN_PREPARE: case CPU_DOWN_PREPARE_FROZEN: - smp_call_function_single(cpu, __gator_timer_offline, NULL, 1); + smp_call_function_single(cpu, gator_timer_offline, NULL, 1); + gator_timer_offline_dispatch(cpu); break; case CPU_ONLINE: case CPU_ONLINE_FROZEN: - smp_call_function_single(cpu, __gator_timer_online, NULL, 1); + gator_timer_online_dispatch(cpu); + smp_call_function_single(cpu, gator_timer_online, NULL, 1); break; } @@ -489,17 +578,24 @@ static struct notifier_block __refdata gator_cpu_notifier = { // Registered linux events are not disabled, so their counters will continue to collect static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy) { + int cpu; + switch (event) { case PM_HIBERNATION_PREPARE: case PM_SUSPEND_PREPARE: unregister_hotcpu_notifier(&gator_cpu_notifier); unregister_scheduler_tracepoints(); - on_each_cpu(trace_sched_insert_idle, NULL, 1); - on_each_cpu(__gator_timer_offline, NULL, 1); + on_each_cpu(gator_timer_offline, NULL, 1); + for_each_online_cpu(cpu) { + gator_timer_offline_dispatch(cpu); + } break; case PM_POST_HIBERNATION: case PM_POST_SUSPEND: - on_each_cpu(__gator_timer_online, NULL, 1); + for_each_online_cpu(cpu) { + gator_timer_online_dispatch(cpu); + } + on_each_cpu(gator_timer_online, NULL, 1); register_scheduler_tracepoints(); register_hotcpu_notifier(&gator_cpu_notifier); break; @@ -548,10 +644,6 @@ static int gator_init(void) { int i; - if (gator_timer_init()) - return -1; - if (gator_trace_sched_init()) - return -1; if (gator_annotate_init()) return -1; @@ -573,8 +665,7 @@ static int gator_start(void) struct list_head *ptr = gi->list.prev; while (ptr != &gator_events) { - gi = list_entry(ptr, struct gator_interface, - list); + gi = list_entry(ptr, struct gator_interface, list); if (gi->stop) gi->stop(); @@ -585,16 +676,18 @@ static int gator_start(void) } } - // cookies shall be initialized before trace_sched_start() and gator_timer_online() + // cookies shall be initialized before trace_sched_start() and gator_timer_start() if (cookies_initialize()) goto cookies_failure; if (gator_annotate_start()) goto annotate_failure; if (gator_trace_sched_start()) goto sched_failure; + if (gator_trace_gpu_start()) + goto gpu_failure; if (gator_event_sampling_start()) goto event_sampling_failure; - if (gator_timer_online(gator_timer_count)) + if (gator_timer_start(gator_timer_count)) goto timer_failure; if (gator_notifier_start()) goto notifier_failure; @@ -602,10 +695,12 @@ static int gator_start(void) return 0; notifier_failure: - gator_timer_offline(); + gator_timer_stop(); timer_failure: gator_event_sampling_stop(); event_sampling_failure: + gator_trace_gpu_stop(); +gpu_failure: gator_trace_sched_stop(); sched_failure: gator_annotate_stop(); @@ -632,11 +727,12 @@ static void gator_stop(void) gator_annotate_stop(); gator_trace_sched_stop(); + gator_trace_gpu_stop(); gator_event_sampling_stop(); // stop all interrupt callback reads before tearing down other interfaces - gator_notifier_stop(); // should be called before gator_timer_offline to avoid re-enabling the hrtimer after it has been offlined - gator_timer_offline(); + gator_notifier_stop(); // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined + gator_timer_stop(); } static void gator_exit(void) @@ -667,18 +763,8 @@ static int gator_op_setup(void) gator_buffer_size[EVENT_BUF] = EVENT_BUFFER_SIZE_DEFAULT; gator_buffer_mask[EVENT_BUF] = gator_buffer_size[EVENT_BUF] - 1; - gator_net_traffic = 0; - - // Initialize per buffer variables + // Initialize percpu per buffer variables for (i = 0; i < NUM_GATOR_BUFS; i++) { - gator_commit_read[i] = gator_commit_write[i] = 0; - gator_commit[i] = vmalloc(COMMIT_SIZE * sizeof(int)); - if (!gator_commit[i]) { - err = -ENOMEM; - goto setup_error; - } - - // Initialize percpu per buffer variables for_each_present_cpu(cpu) { per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]); if (!per_cpu(gator_buffer, cpu)[i]) { @@ -688,6 +774,9 @@ static int gator_op_setup(void) per_cpu(gator_buffer_read, cpu)[i] = 0; per_cpu(gator_buffer_write, cpu)[i] = 0; + per_cpu(gator_buffer_commit, cpu)[i] = 0; + per_cpu(buffer_space_available, cpu)[i] = true; + per_cpu(emit_overflow, cpu) = 0; gator_buffer_header(cpu, i); } } @@ -742,11 +831,6 @@ static void gator_shutdown(void) gator_annotate_shutdown(); - for (i = 0; i < NUM_GATOR_BUFS; i++) { - vfree(gator_commit[i]); - gator_commit[i] = NULL; - } - for_each_present_cpu(cpu) { mutex_lock(&gator_buffer_mutex); for (i = 0; i < NUM_GATOR_BUFS; i++) { @@ -754,6 +838,9 @@ static void gator_shutdown(void) per_cpu(gator_buffer, cpu)[i] = NULL; per_cpu(gator_buffer_read, cpu)[i] = 0; per_cpu(gator_buffer_write, cpu)[i] = 0; + per_cpu(gator_buffer_commit, cpu)[i] = 0; + per_cpu(buffer_space_available, cpu)[i] = true; + per_cpu(emit_overflow, cpu) = 0; } mutex_unlock(&gator_buffer_mutex); } @@ -845,10 +932,10 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf, size_t count, loff_t *offset) { int retval = -EINVAL; - int commit, length1, length2, read; + int commit = 0, length1, length2, read; char *buffer1; char *buffer2 = NULL; - int cpu, i; + int cpu, buftype; /* do not handle partial reads */ if (count != userspace_buffer_size || *offset) @@ -856,7 +943,8 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf, // sleep until the condition is true or a signal is received // the condition is checked each time gator_buffer_wait is woken up - wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(TIMER_BUF) || buffer_commit_ready(EVENT_BUF) || gator_annotate_ready() || !gator_started); + buftype = cpu = -1; + wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || gator_annotate_ready() || !gator_started); if (signal_pending(current)) return -EINTR; @@ -866,28 +954,22 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf, mutex_lock(&gator_buffer_mutex); - i = -1; - if (buffer_commit_ready(TIMER_BUF)) { - i = TIMER_BUF; - } else if (buffer_commit_ready(EVENT_BUF)) { - i = EVENT_BUF; - } - - if (i != -1) { - buffer_commit_read(&cpu, i, &read, &commit); + if (buftype != -1 && cpu != -1) { + read = per_cpu(gator_buffer_read, cpu)[buftype]; + commit = per_cpu(gator_buffer_commit, cpu)[buftype]; /* May happen if the buffer is freed during pending reads. */ - if (!per_cpu(gator_buffer, cpu)[i]) { + if (!per_cpu(gator_buffer, cpu)[buftype]) { retval = -EFAULT; goto out; } /* determine the size of two halves */ length1 = commit - read; - buffer1 = &(per_cpu(gator_buffer, cpu)[i][read]); - buffer2 = &(per_cpu(gator_buffer, cpu)[i][0]); + buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]); + buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]); if (length1 < 0) { - length1 = gator_buffer_size[i] - read; + length1 = gator_buffer_size[buftype] - read; length2 = commit; } } else if (gator_annotate_ready()) { @@ -913,15 +995,15 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf, } } + if (buftype != -1 && cpu != -1) + per_cpu(gator_buffer_read, cpu)[buftype] = commit; + retval = length1 + length2; /* kick just in case we've lost an SMP event */ wake_up(&gator_buffer_wait); out: - // only adjust network stats if in streaming mode - if (gator_streaming) - gator_net_traffic += retval; mutex_unlock(&gator_buffer_mutex); return retval; } diff --git a/driver/gator_pack.c b/driver/gator_pack.c index fbab220..985e960 100644 --- a/driver/gator_pack.c +++ b/driver/gator_pack.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2011. All rights reserved. + * Copyright (C) ARM Limited 2010-2012. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as diff --git a/driver/gator_trace_gpu.c b/driver/gator_trace_gpu.c new file mode 100644 index 0000000..bc63995 --- /dev/null +++ b/driver/gator_trace_gpu.c @@ -0,0 +1,250 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include "gator.h" + +#include <linux/slab.h> +#include <linux/module.h> +#include <linux/time.h> +#include <linux/math64.h> + +#ifdef MALI_SUPPORT +#include "linux/mali_linux_trace.h" +#endif +#include "gator_trace_gpu.h" + +#define ACTIVITY_START 1 +#define ACTIVITY_STOP 2 + +/* Note whether tracepoints have been registered */ +static int mali_trace_registered; +static int gpu_trace_registered; + +#define GPU_OVERFLOW -1 +#define GPU_START 1 +#define GPU_STOP 2 + +#define GPU_UNIT_VP 1 +#define GPU_UNIT_FP 2 + +#define TRACESIZE (8*1024) + +static DEFINE_PER_CPU(uint64_t *[2], theGpuTraceBuf); +static DEFINE_PER_CPU(int, theGpuTraceSel); +static DEFINE_PER_CPU(int, theGpuTracePos); +static DEFINE_PER_CPU(int, theGpuTraceErr); + +int gator_trace_gpu_read(long long **buffer); + +static void probe_gpu_write(int type, int unit, int core, struct task_struct* task) +{ + int tracePos; + unsigned long flags; + uint64_t *traceBuf, time; + int pid, tgid; + int cpu = smp_processor_id(); + + if (!per_cpu(theGpuTraceBuf, cpu)[per_cpu(theGpuTraceSel, cpu)]) + return; + + if (task) { + tgid = (int)task->tgid; + pid = (int)task->pid; + } else { + tgid = pid = 0; + } + + // disable interrupts to synchronize with gator_trace_gpu_read(); spinlocks not needed since percpu buffers are used + local_irq_save(flags); + + time = gator_get_time(); + tracePos = per_cpu(theGpuTracePos, cpu); + traceBuf = per_cpu(theGpuTraceBuf, cpu)[per_cpu(theGpuTraceSel, cpu)]; + + if (tracePos < (TRACESIZE - 100)) { + // capture + traceBuf[tracePos++] = type; + traceBuf[tracePos++] = time; + traceBuf[tracePos++] = unit; + traceBuf[tracePos++] = core; + traceBuf[tracePos++] = tgid; + traceBuf[tracePos++] = pid; + } else if (!per_cpu(theGpuTraceErr, cpu)) { + per_cpu(theGpuTraceErr, cpu) = 1; + traceBuf[tracePos++] = GPU_OVERFLOW; + traceBuf[tracePos++] = time; + traceBuf[tracePos++] = 0; + traceBuf[tracePos++] = 0; + traceBuf[tracePos++] = 0; + traceBuf[tracePos++] = 0; + pr_debug("gator: gpu trace overflow\n"); + } + per_cpu(theGpuTracePos, cpu) = tracePos; + local_irq_restore(flags); +} + +#ifdef MALI_SUPPORT + +enum components { + COMPONENT_VP0 = 1, + COMPONENT_FP0 = 5, + COMPONENT_FP1, + COMPONENT_FP2, + COMPONENT_FP3, + COMPONENT_FP4, + COMPONENT_FP5, + COMPONENT_FP6, + COMPONENT_FP7, +}; + +GATOR_DEFINE_PROBE(mali_timeline_event, TP_PROTO(unsigned int event_id, unsigned int d0, unsigned int d1, unsigned int d2, unsigned int d3, unsigned int d4)) +{ + unsigned int component, state; + + // do as much work as possible before disabling interrupts + component = (event_id >> 16) & 0xF; + state = (event_id >> 24) & 0xF; + + if ((component == COMPONENT_VP0) || (component >= COMPONENT_FP0 && component <= COMPONENT_FP7)) { + if (state == ACTIVITY_START || state == ACTIVITY_STOP) { + unsigned int type = (state == ACTIVITY_START) ? GPU_START : GPU_STOP; + unsigned int unit = (component < COMPONENT_FP0) ? GPU_UNIT_VP : GPU_UNIT_FP; + unsigned int core = (component < COMPONENT_FP0) ? component - COMPONENT_VP0 : component - COMPONENT_FP0; + struct task_struct* task = (state == ACTIVITY_START) ? (struct task_struct*)d2 : NULL; + + probe_gpu_write(type, unit, core, task); + } + } +} +#endif + +GATOR_DEFINE_PROBE(gpu_activity_start, TP_PROTO(int gpu_unit, int gpu_core, struct task_struct *p)) +{ + probe_gpu_write(GPU_START, gpu_unit, gpu_core, p); +} + +GATOR_DEFINE_PROBE(gpu_activity_stop, TP_PROTO(int gpu_unit, int gpu_core)) +{ + probe_gpu_write(GPU_STOP, gpu_unit, gpu_core, NULL); +} + +int gator_trace_gpu_start(void) +{ + int cpu; + + /* + * Returns 0 for installation failed + * Absence of gpu trace points is not an error + */ + + gpu_trace_registered = mali_trace_registered = 0; + +#ifdef MALI_SUPPORT + if (!GATOR_REGISTER_TRACE(mali_timeline_event)) { + mali_trace_registered = 1; + } +#endif + + if (!mali_trace_registered) { + if (GATOR_REGISTER_TRACE(gpu_activity_start)) { + return 0; + } + if (GATOR_REGISTER_TRACE(gpu_activity_stop)) { + GATOR_UNREGISTER_TRACE(gpu_activity_start); + return 0; + } + gpu_trace_registered = 1; + } + + if (!gpu_trace_registered && !mali_trace_registered) { + return 0; + } + + for_each_present_cpu(cpu) { + per_cpu(theGpuTraceSel, cpu) = 0; + per_cpu(theGpuTracePos, cpu) = 0; + per_cpu(theGpuTraceErr, cpu) = 0; + per_cpu(theGpuTraceBuf, cpu)[0] = kmalloc(TRACESIZE * sizeof(uint64_t), GFP_KERNEL); + per_cpu(theGpuTraceBuf, cpu)[1] = kmalloc(TRACESIZE * sizeof(uint64_t), GFP_KERNEL); + if (!per_cpu(theGpuTraceBuf, cpu)[0] || !per_cpu(theGpuTraceBuf, cpu)[1]) { +#ifdef MALI_SUPPORT + if (mali_trace_registered) { + GATOR_UNREGISTER_TRACE(mali_timeline_event); + } +#endif + if (gpu_trace_registered) { + GATOR_UNREGISTER_TRACE(gpu_activity_stop); + GATOR_UNREGISTER_TRACE(gpu_activity_start); + } + + gpu_trace_registered = mali_trace_registered = 0; + + return -1; + } + } + + return 0; +} + +int gator_trace_gpu_offline(long long **buffer) +{ + return gator_trace_gpu_read(buffer); +} + +void gator_trace_gpu_stop(void) +{ + int cpu; + + if (gpu_trace_registered || mali_trace_registered) { + for_each_present_cpu(cpu) { + kfree(per_cpu(theGpuTraceBuf, cpu)[0]); + kfree(per_cpu(theGpuTraceBuf, cpu)[1]); + per_cpu(theGpuTraceBuf, cpu)[0] = NULL; + per_cpu(theGpuTraceBuf, cpu)[1] = NULL; + } + +#ifdef MALI_SUPPORT + if (mali_trace_registered) { + GATOR_UNREGISTER_TRACE(mali_timeline_event); + } +#endif + if (gpu_trace_registered) { + GATOR_UNREGISTER_TRACE(gpu_activity_stop); + GATOR_UNREGISTER_TRACE(gpu_activity_start); + } + + gpu_trace_registered = mali_trace_registered = 0; + } +} + +int gator_trace_gpu_read(long long **buffer) +{ + int cpu = smp_processor_id(); + unsigned long flags; + uint64_t *traceBuf; + int tracePos; + + if (!per_cpu(theGpuTraceBuf, cpu)[per_cpu(theGpuTraceSel, cpu)]) + return 0; + + local_irq_save(flags); + + traceBuf = per_cpu(theGpuTraceBuf, cpu)[per_cpu(theGpuTraceSel, cpu)]; + tracePos = per_cpu(theGpuTracePos, cpu); + + per_cpu(theGpuTraceSel, cpu) = !per_cpu(theGpuTraceSel, cpu); + per_cpu(theGpuTracePos, cpu) = 0; + per_cpu(theGpuTraceErr, cpu) = 0; + + local_irq_restore(flags); + + if (buffer) + *buffer = traceBuf; + + return tracePos; +} diff --git a/driver/gator_trace_gpu.h b/driver/gator_trace_gpu.h new file mode 100644 index 0000000..894289b --- /dev/null +++ b/driver/gator_trace_gpu.h @@ -0,0 +1,79 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#undef TRACE_GPU +#define TRACE_GPU gpu + +#if !defined(_TRACE_GPU_H) +#define _TRACE_GPU_H + +#include <linux/tracepoint.h> + +/* + * UNIT - the GPU processor type + * 1 = Vertex Processor + * 2 = Fragment Processor + * + * CORE - the GPU processor core number + * this is not the CPU core number + */ + +/* + * Tracepoint for calling GPU unit start activity on core + */ +TRACE_EVENT(gpu_activity_start, + + TP_PROTO(int gpu_unit, int gpu_core, struct task_struct *p), + + TP_ARGS(gpu_unit, gpu_core, p), + + TP_STRUCT__entry( + __field( int, gpu_unit ) + __field( int, gpu_core ) + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + ), + + TP_fast_assign( + __entry->gpu_unit = gpu_unit; + __entry->gpu_core = gpu_core; + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + ), + + TP_printk("unit=%d core=%d comm=%s pid=%d", + __entry->gpu_unit, __entry->gpu_core, __entry->comm, __entry->pid) +); + +/* + * Tracepoint for calling GPU unit stop activity on core + */ +TRACE_EVENT(gpu_activity_stop, + + TP_PROTO(int gpu_unit, int gpu_core), + + TP_ARGS(gpu_unit, gpu_core), + + TP_STRUCT__entry( + __field( int, gpu_unit ) + __field( int, gpu_core ) + ), + + TP_fast_assign( + __entry->gpu_unit = gpu_unit; + __entry->gpu_core = gpu_core; + ), + + TP_printk("unit=%d core=%d", + __entry->gpu_unit, __entry->gpu_core) +); + +#endif /* _TRACE_GPU_H */ + +/* This part must be outside protection */ +#include <trace/define_trace.h> diff --git a/driver/gator_trace_sched.c b/driver/gator_trace_sched.c index 0225bfb..dafacb7 100644 --- a/driver/gator_trace_sched.c +++ b/driver/gator_trace_sched.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2011. All rights reserved. + * Copyright (C) ARM Limited 2010-2012. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -30,6 +30,8 @@ enum { STATE_WAIT_ON_OTHER }; +int gator_trace_sched_read(long long **buffer); + void emit_pid_name(struct task_struct* task) { bool found = false; @@ -50,7 +52,7 @@ void emit_pid_name(struct task_struct* task) } } - if (!found) { + if (!found && buffer_check_space(cpu, TIMER_BUF, TASK_COMM_LEN + 2 * MAXSIZE_PACK32 + MAXSIZE_PACK64)) { // shift values, new value always in front uint64_t oldv, newv = value; for (x = 0; x < TASK_MAX_COLLISIONS; x++) { @@ -128,7 +130,7 @@ static void probe_sched_write(int type, struct task_struct* task, struct task_st } // special case used during a suspend of the system -static void trace_sched_insert_idle(void* unused) +static void trace_sched_insert_idle(void) { unsigned long flags; uint64_t *schedBuf; @@ -170,11 +172,6 @@ GATOR_DEFINE_PROBE(sched_process_free, TP_PROTO(struct task_struct *p)) probe_sched_write(SCHED_PROCESS_FREE, p, 0); } -int gator_trace_sched_init(void) -{ - return 0; -} - static int register_scheduler_tracepoints(void) { // register tracepoints if (GATOR_REGISTER_TRACE(sched_switch)) @@ -217,6 +214,12 @@ int gator_trace_sched_start(void) return register_scheduler_tracepoints(); } +int gator_trace_sched_offline(long long **buffer) +{ + trace_sched_insert_idle(); + return gator_trace_sched_read(buffer); +} + static void unregister_scheduler_tracepoints(void) { GATOR_UNREGISTER_TRACE(sched_switch); |