diff options
author | Drew Richardson <drew.richardson@arm.com> | 2011-09-30 12:00:00 -0700 |
---|---|---|
committer | Drew Richardson <drew.richardson@arm.com> | 2014-12-19 15:16:17 -0800 |
commit | c5f78695067d4614bcb494d5f54bde4405c43609 (patch) | |
tree | 5413ee61289c45be1917e93852d491aa1f778087 /driver | |
parent | 73158f9d05b7087bb6ff895e820017af3c94a1a6 (diff) |
gator: Version 5.75.7
Signed-off-by: Drew Richardson <drew.richardson@arm.com>
Diffstat (limited to 'driver')
-rw-r--r-- | driver/Makefile | 5 | ||||
-rw-r--r-- | driver/gator.h | 12 | ||||
-rw-r--r-- | driver/gator_annotate.c | 28 | ||||
-rw-r--r-- | driver/gator_backtrace.c | 6 | ||||
-rw-r--r-- | driver/gator_cookies.c | 40 | ||||
-rw-r--r-- | driver/gator_ebs.c | 160 | ||||
-rw-r--r-- | driver/gator_events_armv6.c | 40 | ||||
-rw-r--r-- | driver/gator_events_armv7.c | 186 | ||||
-rw-r--r-- | driver/gator_events_armv7.h | 38 | ||||
-rw-r--r-- | driver/gator_events_l2c-310.c | 182 | ||||
-rw-r--r-- | driver/gator_events_meminfo.c | 11 | ||||
-rw-r--r-- | driver/gator_events_pl310.c | 176 | ||||
-rw-r--r-- | driver/gator_events_scorpion.c | 40 | ||||
-rw-r--r-- | driver/gator_main.c | 528 | ||||
-rw-r--r-- | driver/gator_pack.c | 262 | ||||
-rw-r--r-- | driver/gator_trace_sched.c | 228 |
16 files changed, 1174 insertions, 768 deletions
diff --git a/driver/Makefile b/driver/Makefile index b3680e1..de59957 100644 --- a/driver/Makefile +++ b/driver/Makefile @@ -17,7 +17,8 @@ endif gator-$(CONFIG_ARM) += gator_events_armv6.o \ gator_events_armv7.o \ - gator_events_pl310.o + gator_events_l2c-310.o \ + gator_events_scorpion.o $(obj)/gator_main.o: gator_events.h @@ -40,6 +41,6 @@ all: $(error) clean: - rm -f *.o modules.order Module.symvers gator.ko gator.mod.c + rm -f *.o .*.cmd gator_events.h modules.order Module.symvers gator.ko gator.mod.c endif diff --git a/driver/gator.h b/driver/gator.h index 9d802a3..724ae19 100644 --- a/driver/gator.h +++ b/driver/gator.h @@ -14,6 +14,16 @@ #include <linux/mm.h> #include <linux/list.h> +// cpu ids +#define ARM1136 0xb36 +#define ARM1156 0xb56 +#define ARM1176 0xb76 +#define ARM11MPCORE 0xb02 +#define CORTEX_A5 0xc05 +#define CORTEX_A8 0xc08 +#define CORTEX_A9 0xc09 +#define CORTEX_A15 0xc0f + /****************************************************************************** * Filesystem ******************************************************************************/ @@ -62,6 +72,7 @@ struct gator_interface { void (*online)(void); void (*offline)(void); int (*read)(int **buffer); + int (*read64)(long long **buffer); struct list_head list; }; @@ -75,5 +86,4 @@ extern u32 gator_cpuid(void); extern unsigned long gator_net_traffic; - #endif // GATOR_H_ diff --git a/driver/gator_annotate.c b/driver/gator_annotate.c index d8d86af..e096d15 100644 --- a/driver/gator_annotate.c +++ b/driver/gator_annotate.c @@ -22,6 +22,7 @@ static char *annotateBuf0; static char *annotateBuf1; static int annotatePos; static int annotateSel; +static bool collect_annotations = false; static ssize_t annotate_write(struct file *file, char const __user *buf, size_t count, loff_t *offset) { @@ -45,16 +46,13 @@ static ssize_t annotate_write(struct file *file, char const __user *buf, size_t if (retval == 0) { // synchronize shared variables annotateBuf and annotatePos spin_lock(&annotate_lock); - if (annotateBuf) { + if (collect_annotations && annotateBuf) { uint32_t tid = current->pid; - uint32_t tick = gator_master_tick; uint64_t time = gator_get_time(); uint32_t cpuid = smp_processor_id(); int pos = annotatePos; pos += gator_write_packed_int(&annotateBuf[pos], tid); - pos += gator_write_packed_int(&annotateBuf[pos], tick); - pos += gator_write_packed_int(&annotateBuf[pos], time); - pos += gator_write_packed_int(&annotateBuf[pos], time >> 32); + pos += gator_write_packed_int64(&annotateBuf[pos], time); pos += gator_write_packed_int(&annotateBuf[pos], cpuid); pos += gator_write_packed_int(&annotateBuf[pos], size); memcpy(&annotateBuf[pos], tempBuffer, size); @@ -81,12 +79,9 @@ static int annotate_release(struct inode *inode, struct file *file) spin_lock(&annotate_lock); if (annotateBuf) { uint32_t tid = current->pid; - uint32_t tick = gator_master_tick; int pos = annotatePos; pos += gator_write_packed_int(&annotateBuf[pos], tid); - pos += gator_write_packed_int(&annotateBuf[pos], tick); - pos += gator_write_packed_int(&annotateBuf[pos], 0); // time - pos += gator_write_packed_int(&annotateBuf[pos], 0); // time + pos += gator_write_packed_int64(&annotateBuf[pos], 0); // time pos += gator_write_packed_int(&annotateBuf[pos], 0); // cpuid pos += gator_write_packed_int(&annotateBuf[pos], 0); // size annotatePos = pos; @@ -118,13 +113,21 @@ static int gator_annotate_init(void) static int gator_annotate_start(void) { - annotatePos = annotateSel = 0; + annotateSel = 0; + annotatePos = 1; annotateBuf = annotateBuf0; + annotateBuf[0] = FRAME_ANNOTATE; + collect_annotations = true; return 0; } static void gator_annotate_stop(void) { + collect_annotations = false; +} + +static void gator_annotate_shutdown(void) +{ spin_lock(&annotate_lock); annotateBuf = NULL; spin_unlock(&annotate_lock); @@ -141,7 +144,7 @@ static void gator_annotate_exit(void) static int gator_annotate_ready(void) { - return annotatePos && annotateBuf; + return annotatePos > 1 && annotateBuf; } static int gator_annotate_read(char **buffer) @@ -158,8 +161,9 @@ static int gator_annotate_read(char **buffer) spin_lock(&annotate_lock); len = annotatePos; - annotatePos = 0; annotateBuf = annotateSel ? annotateBuf1 : annotateBuf0; + annotateBuf[0] = FRAME_ANNOTATE; + annotatePos = 1; spin_unlock(&annotate_lock); return len; diff --git a/driver/gator_backtrace.c b/driver/gator_backtrace.c index 628a18f..b5b0e63 100644 --- a/driver/gator_backtrace.c +++ b/driver/gator_backtrace.c @@ -15,7 +15,7 @@ struct frame_tail_eabi { unsigned long lr; }; -static void arm_backtrace_eabi(int cpu, struct pt_regs * const regs, unsigned int depth) +static void arm_backtrace_eabi(int cpu, int buftype, struct pt_regs * const regs, unsigned int depth) { #if defined(__arm__) struct frame_tail_eabi *tail; @@ -31,7 +31,7 @@ static void arm_backtrace_eabi(int cpu, struct pt_regs * const regs, unsigned in } /* entry preamble may not have executed */ - gator_add_trace(cpu, lr); + gator_add_trace(cpu, buftype, lr); /* check tail is valid */ if (fp == 0) { @@ -49,7 +49,7 @@ static void arm_backtrace_eabi(int cpu, struct pt_regs * const regs, unsigned in ptrtail = &buftail; lr = ptrtail[0].lr; - gator_add_trace(cpu, lr); + gator_add_trace(cpu, buftype, lr); /* frame pointers should progress back up the stack, towards higher addresses */ next = (struct frame_tail_eabi *)(lr - 4); diff --git a/driver/gator_cookies.c b/driver/gator_cookies.c index 64be841..a646fb2 100644 --- a/driver/gator_cookies.c +++ b/driver/gator_cookies.c @@ -22,7 +22,7 @@ static DEFINE_PER_CPU(int, translate_buffer_read); static DEFINE_PER_CPU(int, translate_buffer_write); static DEFINE_PER_CPU(unsigned int *, translate_buffer); -static inline uint32_t get_cookie(int cpu, struct task_struct *task, struct vm_area_struct *vma, struct module *mod); +static inline uint32_t get_cookie(int cpu, int buftype, struct task_struct *task, struct vm_area_struct *vma, struct module *mod, bool in_interrupt); static void wq_cookie_handler(struct work_struct *unused); DECLARE_WORK(cookie_work, wq_cookie_handler); @@ -96,7 +96,7 @@ static void cookiemap_add(uint64_t key, uint32_t value) { for (x = MAX_COLLISIONS-1; x > 0; x--) { keys[x] = keys[x-1]; - values[x] = keys[x-1]; + values[x] = values[x-1]; } keys[0] = key; values[0] = value; @@ -126,12 +126,12 @@ static void wq_cookie_handler(struct work_struct *unused) while (per_cpu(translate_buffer_read, cpu) != commit) { task = (struct task_struct *)translate_buffer_read_int(cpu); vma = (struct vm_area_struct *)translate_buffer_read_int(cpu); - cookie = get_cookie(cpu, task, vma, NULL); + cookie = get_cookie(cpu, TIMER_BUF, task, vma, NULL, false); } } // Retrieve full name from proc/pid/cmdline for java processes on Android -static int translate_app_process(char** text, int cpu, struct task_struct * task, struct vm_area_struct *vma) +static int translate_app_process(char** text, int cpu, struct task_struct * task, struct vm_area_struct *vma, bool in_interrupt) { void *maddr; unsigned int len; @@ -143,7 +143,9 @@ static int translate_app_process(char** text, int cpu, struct task_struct * task char * buf = per_cpu(translate_text, cpu); // Push work into a work queue if in atomic context as the kernel functions below might sleep - if (in_irq()) { + // Rely on the in_interrupt variable rather than in_irq() or in_interrupt() kernel functions, as the value of these functions seems + // inconsistent during a context switch between android/linux versions + if (in_interrupt) { // Check if already in buffer ptr = per_cpu(translate_buffer_read, cpu); while (ptr != per_cpu(translate_buffer_write, cpu)) { @@ -205,7 +207,7 @@ out: return retval; } -static inline uint32_t get_cookie(int cpu, struct task_struct *task, struct vm_area_struct *vma, struct module *mod) +static inline uint32_t get_cookie(int cpu, int buftype, struct task_struct *task, struct vm_area_struct *vma, struct module *mod, bool in_interrupt) { unsigned long flags, cookie; struct path *path; @@ -235,26 +237,26 @@ static inline uint32_t get_cookie(int cpu, struct task_struct *task, struct vm_a } if (strcmp(text, "app_process") == 0 && !mod) { - if (!translate_app_process(&text, cpu, task, vma)) + if (!translate_app_process(&text, cpu, task, vma, in_interrupt)) return INVALID_COOKIE; } - // Can be called from interrupt handler or from work queue + // Can be called from interrupt handler or from work queue or from scheduler trace local_irq_save(flags); cookie = per_cpu(cookie_next_key, cpu)+=nr_cpu_ids; cookiemap_add(key, cookie); - gator_buffer_write_packed_int(cpu, PROTOCOL_COOKIE); - gator_buffer_write_packed_int(cpu, cookie); - gator_buffer_write_string(cpu, text); + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COOKIE); + gator_buffer_write_packed_int(cpu, buftype, cookie); + gator_buffer_write_string(cpu, buftype, text); local_irq_restore(flags); return cookie; } -static int get_exec_cookie(int cpu, struct task_struct *task) +static int get_exec_cookie(int cpu, int buftype, struct task_struct *task) { unsigned long cookie = NO_COOKIE; struct mm_struct *mm = task->mm; @@ -268,14 +270,14 @@ static int get_exec_cookie(int cpu, struct task_struct *task) continue; if (!(vma->vm_flags & VM_EXECUTABLE)) continue; - cookie = get_cookie(cpu, task, vma, NULL); + cookie = get_cookie(cpu, buftype, task, vma, NULL, true); break; } return cookie; } -static unsigned long get_address_cookie(int cpu, struct task_struct *task, unsigned long addr, off_t *offset) +static unsigned long get_address_cookie(int cpu, int buftype, struct task_struct *task, unsigned long addr, off_t *offset) { unsigned long cookie = NO_COOKIE; struct mm_struct *mm = task->mm; @@ -289,7 +291,7 @@ static unsigned long get_address_cookie(int cpu, struct task_struct *task, unsig continue; if (vma->vm_file) { - cookie = get_cookie(cpu, task, vma, NULL); + cookie = get_cookie(cpu, buftype, task, vma, NULL, true); *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start; } else { /* must be an anonymous map */ @@ -318,10 +320,18 @@ static int cookies_initialize(void) size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint64_t); per_cpu(cookie_keys, cpu) = (uint64_t*)kmalloc(size, GFP_KERNEL); + if (!per_cpu(cookie_keys, cpu)) { + err = -ENOMEM; + goto cookie_setup_error; + } memset(per_cpu(cookie_keys, cpu), 0, size); size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint32_t); per_cpu(cookie_values, cpu) = (uint32_t*)kmalloc(size, GFP_KERNEL); + if (!per_cpu(cookie_values, cpu)) { + err = -ENOMEM; + goto cookie_setup_error; + } memset(per_cpu(cookie_values, cpu), 0, size); per_cpu(translate_buffer, cpu) = (unsigned int *)kmalloc(translate_buffer_size, GFP_KERNEL); diff --git a/driver/gator_ebs.c b/driver/gator_ebs.c new file mode 100644 index 0000000..9b55347 --- /dev/null +++ b/driver/gator_ebs.c @@ -0,0 +1,160 @@ +/** + * Copyright (C) ARM Limited 2011. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +/****************************************************************************** + * event based sampling handling + ******************************************************************************/ + +#if defined (__arm__) +#include "gator_events_armv7.h" +#include <linux/platform_device.h> +#include <linux/interrupt.h> +#include <linux/irq.h> + +#if LINUX_PMU_SUPPORT +#include <asm/pmu.h> + +static struct platform_device *pmu_device; + +static irqreturn_t armv7_pmnc_interrupt(int irq, void *arg) +{ + unsigned int cnt, cpu = smp_processor_id(), buftype = EVENT_BUF; + struct pt_regs * const regs = get_irq_regs(); + u32 flags; + + // Stop irq generation + armv7_pmnc_write(armv7_pmnc_read() & ~PMNC_E); + + // Get and reset overflow status flags + flags = armv7_pmnc_reset_interrupt(); + + // Counters header + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); // type + gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); // time + + // Cycle counter + if (flags & (1 << 31)) { + int value = armv7_ccnt_read(pmnc_count[CCNT]); // overrun + gator_buffer_write_packed_int(cpu, buftype, 2); // length + gator_buffer_write_packed_int(cpu, buftype, pmnc_key[CCNT]); // key + gator_buffer_write_packed_int(cpu, buftype, value); // value + } + + // PMNC counters + for (cnt = CNT0; cnt < CNTMAX; cnt++) { + if (flags & (1 << (cnt - CNT0))) { + int value = armv7_cntn_read(cnt, pmnc_count[cnt]); // overrun + gator_buffer_write_packed_int(cpu, buftype, 2); // length + gator_buffer_write_packed_int(cpu, buftype, pmnc_key[cnt]); // key + gator_buffer_write_packed_int(cpu, buftype, value); // value + } + } + + // End Counters, length of zero + gator_buffer_write_packed_int(cpu, buftype, 0); + + // Output backtrace + gator_add_sample(cpu, buftype, regs); + + // Check and commit; commit is set to occur once buffer is 3/4 full + event_buffer_check(cpu); + + // Allow irq generation + armv7_pmnc_write(armv7_pmnc_read() | PMNC_E); + + return IRQ_HANDLED; +} +#endif + +static int gator_event_sampling_start(void) +{ + int cnt; + + event_based_sampling = false; + for (cnt = CCNT; cnt < CNTMAX; cnt++) { + if (pmnc_count[cnt] > 0) { + event_based_sampling = true; + break; + } + } + +#if LINUX_PMU_SUPPORT + pmu_device = reserve_pmu(ARM_PMU_DEVICE_CPU); + if (IS_ERR(pmu_device) && (unsigned int)pmu_device != -ENODEV) { + pr_err("gator: unable to reserve the pmu\n"); + return -1; + } + + if (event_based_sampling) { + int irq, i; + + if (IS_ERR(pmu_device)) { + pr_err("gator: event based sampling is not supported as the kernel function reserve_pmu() failed"); + return -1; + } + + init_pmu(ARM_PMU_DEVICE_CPU); + if (pmu_device->num_resources == 0) { + pr_err("gator: no irqs for PMUs defined\n"); + release_pmu(pmu_device); + pmu_device = NULL; + return -1; + } + + for (i = 0; i < pmu_device->num_resources; ++i) { + irq = platform_get_irq(pmu_device, i); + if (irq < 0) + continue; + + if (request_irq(irq, armv7_pmnc_interrupt, IRQF_DISABLED | IRQF_NOBALANCING, "armpmu", NULL)) { + pr_err("gator: unable to request IRQ%d for ARM perf counters\n", irq); + + // clean up and exit + for (i = i - 1; i >= 0; --i) { + irq = platform_get_irq(pmu_device, i); + if (irq >= 0) + free_irq(irq, NULL); + } + release_pmu(pmu_device); + pmu_device = NULL; + return -1; + } + } + } +#else + if (event_based_sampling) { + pr_err("gator: event based sampling only supported in kernel versions 2.6.35 and higher and CONFIG_CPU_HAS_PMU=y\n"); + return -1; + } +#endif + + return 0; +} + +static void gator_event_sampling_stop(void) +{ +#if LINUX_PMU_SUPPORT + if (event_based_sampling) { + int i, irq; + for (i = pmu_device->num_resources - 1; i >= 0; --i) { + irq = platform_get_irq(pmu_device, i); + if (irq >= 0) + free_irq(irq, NULL); + } + } + if (!IS_ERR(pmu_device)) + release_pmu(pmu_device); + pmu_device = NULL; +#endif +} + +#else +static int gator_event_sampling_start(void) {return 0;} +static void gator_event_sampling_stop(void) {} +#endif diff --git a/driver/gator_events_armv6.c b/driver/gator_events_armv6.c index 7b1d875..170f066 100644 --- a/driver/gator_events_armv6.c +++ b/driver/gator_events_armv6.c @@ -8,10 +8,6 @@ #include "gator.h" -#define ARM1136 0xb36 -#define ARM1156 0xb56 -#define ARM1176 0xb76 - static const char *pmnc_name; /* @@ -29,9 +25,10 @@ static const char *pmnc_name; #define CCNT 2 #define CNTMAX (CCNT+1) -static int pmnc_count = 0; +static int pmnc_counters = 0; static unsigned long pmnc_enabled[CNTMAX]; static unsigned long pmnc_event[CNTMAX]; +static unsigned long pmnc_count[CNTMAX]; static unsigned long pmnc_key[CNTMAX]; static DEFINE_PER_CPU(int[CNTMAX], perfPrev); @@ -72,7 +69,7 @@ int gator_events_armv6_create_files(struct super_block *sb, struct dentry *root) struct dentry *dir; int i; - pmnc_count = 3; + pmnc_counters = 3; for (i = PMN0; i <= CCNT; i++) { char buf[40]; @@ -86,10 +83,11 @@ int gator_events_armv6_create_files(struct super_block *sb, struct dentry *root) return -1; } gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); + gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]); + gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); if (i != CCNT) { gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); } - gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); } return 0; @@ -119,18 +117,14 @@ static void gator_events_armv6_online(void) event = pmnc_event[cnt] & 255; - /* - * Set event (if destined for PMNx counters) - */ + // Set event (if destined for PMNx counters) if (cnt == PMN0) { pmnc |= event << 20; } else if (cnt == PMN1) { pmnc |= event << 12; } - /* - * Reset counter - */ + // Reset counter armv6_pmnc_reset_counter(cnt); } armv6_pmnc_write(pmnc | PMCR_E); @@ -146,6 +140,20 @@ static void gator_events_armv6_offline(void) } } +static int gator_events_armv6_start(void) +{ + int cnt; + + for (cnt = CCNT; cnt < CNTMAX; cnt++) { + if (pmnc_count[cnt] > 0) { + pr_err("gator: event based sampling not supported on ARM v6 architectures\n"); + return -1; + } + } + + return 0; +} + static void gator_events_armv6_stop(void) { unsigned int cnt; @@ -153,6 +161,7 @@ static void gator_events_armv6_stop(void) for (cnt = PMN0; cnt <= CCNT; cnt++) { pmnc_enabled[cnt] = 0; pmnc_event[cnt] = 0; + pmnc_count[cnt] = 0; } } @@ -193,6 +202,7 @@ static int gator_events_armv6_read(int **buffer) static struct gator_interface gator_events_armv6_interface = { .create_files = gator_events_armv6_create_files, + .start = gator_events_armv6_start, .stop = gator_events_armv6_stop, .online = gator_events_armv6_online, .offline = gator_events_armv6_offline, @@ -209,6 +219,9 @@ int gator_events_armv6_init(void) case ARM1176: pmnc_name = "ARM11"; break; + case ARM11MPCORE: + pmnc_name = "ARM11MPCore"; + break; default: return -1; } @@ -216,6 +229,7 @@ int gator_events_armv6_init(void) for (cnt = PMN0; cnt <= CCNT; cnt++) { pmnc_enabled[cnt] = 0; pmnc_event[cnt] = 0; + pmnc_count[cnt] = 0; pmnc_key[cnt] = gator_events_get_key(); } diff --git a/driver/gator_events_armv7.c b/driver/gator_events_armv7.c index 0f0d7ef..58855f8 100644 --- a/driver/gator_events_armv7.c +++ b/driver/gator_events_armv7.c @@ -6,129 +6,116 @@ * published by the Free Software Foundation. */ -#include "gator.h" - -#define CORTEX_A5 0xc05 -#define CORTEX_A8 0xc08 -#define CORTEX_A9 0xc09 -#define CORTEX_A15 0xc0f - -static const char *pmnc_name; -static int pmnc_count; - -// Per-CPU PMNC: config reg -#define PMNC_E (1 << 0) /* Enable all counters */ -#define PMNC_P (1 << 1) /* Reset all counters */ -#define PMNC_C (1 << 2) /* Cycle counter reset */ -#define PMNC_MASK 0x3f /* Mask for writable bits */ +/* Disabling interrupts + * Many of the functions below disable interrupts via local_irq_save(). This disabling of interrupts is done to prevent any race conditions + * between multiple entities (e.g. hrtimer interrupts and event based interrupts) calling the same functions. As accessing the pmu involves + * several steps (disable, select, read, enable), these steps must be performed atomically. Normal synchronization routines cannot be used + * as these functions are being called from interrupt context. + */ -// ccnt reg -#define CCNT_REG (1 << 31) +#include "gator.h" +#include "gator_events_armv7.h" -#define CCNT 0 -#define CNT0 1 -#define CNTMAX (6+1) +const char *pmnc_name; +int pmnc_counters; -static unsigned long pmnc_enabled[CNTMAX]; -static unsigned long pmnc_event[CNTMAX]; -static unsigned long pmnc_key[CNTMAX]; +unsigned long pmnc_enabled[CNTMAX]; +unsigned long pmnc_event[CNTMAX]; +unsigned long pmnc_count[CNTMAX]; +unsigned long pmnc_key[CNTMAX]; static DEFINE_PER_CPU(int[CNTMAX], perfPrev); static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt); -static inline void armv7_pmnc_write(u32 val) +inline void armv7_pmnc_write(u32 val) { val &= PMNC_MASK; asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r" (val)); } -static inline u32 armv7_pmnc_read(void) +inline u32 armv7_pmnc_read(void) { u32 val; asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val)); return val; } -static inline u32 armv7_ccnt_read(void) +inline u32 armv7_ccnt_read(u32 reset_value) { - u32 zero = 0; + unsigned long flags; + u32 newval = -reset_value; u32 den = CCNT_REG; u32 val; + local_irq_save(flags); asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (den)); // disable asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val)); // read - asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (zero)); // zero + asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (newval));// new value asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (den)); // enable + local_irq_restore(flags); return val; } -static inline u32 armv7_cntn_read(unsigned int cnt) +inline u32 armv7_cntn_read(unsigned int cnt, u32 reset_value) { - u32 zero = 0; + unsigned long flags; + u32 newval = -reset_value; u32 sel = (cnt - CNT0); u32 den = 1 << sel; - u32 val; + u32 oldval; + local_irq_save(flags); asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (den)); // disable asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (sel)); // select - asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val)); // read - asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (zero)); // zero + asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (oldval)); // read + asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (newval));// new value asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (den)); // enable + local_irq_restore(flags); - return val; + return oldval; } -static inline u32 armv7_pmnc_enable_counter(unsigned int cnt) +static inline void armv7_pmnc_enable_interrupt(unsigned int cnt) { - u32 val; + u32 val = cnt ? (1 << (cnt - CNT0)) : (1 << 31); + asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (val)); +} - if (cnt >= CNTMAX) { - pr_err("gator: CPU%u enabling wrong PMNC counter %d\n", smp_processor_id(), cnt); - return -1; - } +static inline void armv7_pmnc_disable_interrupt(unsigned int cnt) +{ + u32 val = cnt ? (1 << (cnt - CNT0)) : (1 << 31); + asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (val)); +} - if (cnt == CCNT) - val = CCNT_REG; - else - val = (1 << (cnt - CNT0)); +inline u32 armv7_pmnc_reset_interrupt() +{ + // Get and reset overflow status flags + u32 flags; + asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (flags)); + flags &= 0x8000003f; + asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (flags)); + return flags; +} +static inline u32 armv7_pmnc_enable_counter(unsigned int cnt) +{ + u32 val = cnt ? (1 << (cnt - CNT0)) : CCNT_REG; asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (val)); - return cnt; } static inline u32 armv7_pmnc_disable_counter(unsigned int cnt) { - u32 val; - - if (cnt >= CNTMAX) { - pr_err("gator: CPU%u disabling wrong PMNC counter %d\n", smp_processor_id(), cnt); - return -1; - } - - if (cnt == CCNT) - val = CCNT_REG; - else - val = (1 << (cnt - CNT0)); - + u32 val = cnt ? (1 << (cnt - CNT0)) : CCNT_REG; asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (val)); - return cnt; } static inline int armv7_pmnc_select_counter(unsigned int cnt) { - u32 val; - - if ((cnt == CCNT) || (cnt >= CNTMAX)) { - pr_err("gator: CPU%u selecting wrong PMNC counter %d\n", smp_processor_id(), cnt); - return -1; - } - - val = (cnt - CNT0); + u32 val = (cnt - CNT0); asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (val)); - return cnt; } @@ -139,37 +126,12 @@ static inline void armv7_pmnc_write_evtsel(unsigned int cnt, u32 val) } } -static void armv7_pmnc_reset_counter(unsigned int cnt) -{ - u32 val = 0; - - if (cnt == CCNT) { - armv7_pmnc_disable_counter(cnt); - - asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (val)); - - if (pmnc_enabled[cnt] != 0) - armv7_pmnc_enable_counter(cnt); - - } else if (cnt >= CNTMAX) { - pr_err("gator: CPU%u resetting wrong PMNC counter %d\n", smp_processor_id(), cnt); - } else { - armv7_pmnc_disable_counter(cnt); - - if (armv7_pmnc_select_counter(cnt) == cnt) - asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (val)); - - if (pmnc_enabled[cnt] != 0) - armv7_pmnc_enable_counter(cnt); - } -} - static int gator_events_armv7_create_files(struct super_block *sb, struct dentry *root) { struct dentry *dir; int i; - for (i = 0; i < pmnc_count; i++) { + for (i = 0; i < pmnc_counters; i++) { char buf[40]; if (i == 0) { snprintf(buf, sizeof buf, "ARM_%s_ccnt", pmnc_name); @@ -181,10 +143,11 @@ static int gator_events_armv7_create_files(struct super_block *sb, struct dentry return -1; } gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); + gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]); + gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); if (i > 0) { gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); } - gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); } return 0; @@ -202,6 +165,9 @@ static void gator_events_armv7_online(void) // Initialize & Reset PMNC: C bit and P bit armv7_pmnc_write(PMNC_P | PMNC_C); + // Reset overflow flags + armv7_pmnc_reset_interrupt(); + for (cnt = CCNT; cnt < CNTMAX; cnt++) { unsigned long event; @@ -219,10 +185,16 @@ static void gator_events_armv7_online(void) if (cnt != CCNT) armv7_pmnc_write_evtsel(cnt, event); + // Enable/disable interrupt + if (pmnc_count[cnt] > 0) + armv7_pmnc_enable_interrupt(cnt); + else + armv7_pmnc_disable_interrupt(cnt); + // Reset counter - armv7_pmnc_reset_counter(cnt); + cnt ? armv7_cntn_read(cnt, pmnc_count[cnt]) : armv7_ccnt_read(pmnc_count[cnt]); - // Enable counter, but do not enable interrupt for this counter + // Enable counter armv7_pmnc_enable_counter(cnt); } @@ -242,6 +214,7 @@ static void gator_events_armv7_stop(void) for (cnt = CCNT; cnt < CNTMAX; cnt++) { pmnc_enabled[cnt] = 0; pmnc_event[cnt] = 0; + pmnc_count[cnt] = 0; } } @@ -250,16 +223,16 @@ static int gator_events_armv7_read(int **buffer) int cnt, len = 0; int cpu = smp_processor_id(); - if (!pmnc_count) + if (!pmnc_counters) return 0; - for (cnt = 0; cnt < pmnc_count; cnt++) { - if (pmnc_enabled[cnt]) { + for (cnt = 0; cnt < pmnc_counters; cnt++) { + if (pmnc_enabled[cnt] && pmnc_count[cnt] == 0) { int value; if (cnt == CCNT) { - value = armv7_ccnt_read(); + value = armv7_ccnt_read(0); } else { - value = armv7_cntn_read(cnt); + value = armv7_cntn_read(cnt, 0); } if (value != per_cpu(perfPrev, cpu)[cnt]) { per_cpu(perfPrev, cpu)[cnt] = value; @@ -291,29 +264,30 @@ int gator_events_armv7_init(void) switch (gator_cpuid()) { case CORTEX_A5: pmnc_name = "Cortex-A5"; - pmnc_count = 2; + pmnc_counters = 2; break; case CORTEX_A8: pmnc_name = "Cortex-A8"; - pmnc_count = 4; + pmnc_counters = 4; break; case CORTEX_A9: pmnc_name = "Cortex-A9"; - pmnc_count = 6; + pmnc_counters = 6; break; case CORTEX_A15: pmnc_name = "Cortex-A15"; - pmnc_count = 6; + pmnc_counters = 6; break; default: return -1; } - pmnc_count++; // CNT[n] + CCNT + pmnc_counters++; // CNT[n] + CCNT for (cnt = CCNT; cnt < CNTMAX; cnt++) { pmnc_enabled[cnt] = 0; pmnc_event[cnt] = 0; + pmnc_count[cnt] = 0; pmnc_key[cnt] = gator_events_get_key(); } diff --git a/driver/gator_events_armv7.h b/driver/gator_events_armv7.h new file mode 100644 index 0000000..d5e8d6e --- /dev/null +++ b/driver/gator_events_armv7.h @@ -0,0 +1,38 @@ +/** + * Copyright (C) ARM Limited 2011. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef GATOR_EVENTS_ARMV7_H_ +#define GATOR_EVENTS_ARMV7_H_ + +// Per-CPU PMNC: config reg +#define PMNC_E (1 << 0) /* Enable all counters */ +#define PMNC_P (1 << 1) /* Reset all counters */ +#define PMNC_C (1 << 2) /* Cycle counter reset */ +#define PMNC_MASK 0x3f /* Mask for writable bits */ + +// ccnt reg +#define CCNT_REG (1 << 31) + +#define CCNT 0 +#define CNT0 1 +#define CNTMAX (6+1) + +// Function prototypes +extern void armv7_pmnc_write(u32 val); +extern u32 armv7_pmnc_read(void); +extern u32 armv7_ccnt_read(u32 reset_value); +extern u32 armv7_cntn_read(unsigned int cnt, u32 reset_value); +extern u32 armv7_pmnc_reset_interrupt(void); + +// Externed variables +extern unsigned long pmnc_enabled[CNTMAX]; +extern unsigned long pmnc_event[CNTMAX]; +extern unsigned long pmnc_count[CNTMAX]; +extern unsigned long pmnc_key[CNTMAX]; + +#endif // GATOR_EVENTS_ARMV7_H_ diff --git a/driver/gator_events_l2c-310.c b/driver/gator_events_l2c-310.c new file mode 100644 index 0000000..96683b3 --- /dev/null +++ b/driver/gator_events_l2c-310.c @@ -0,0 +1,182 @@ +/** + * l2c310 (L2 Cache Controller) event counters for gator + * + * Copyright (C) ARM Limited 2010-2011. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/init.h> +#include <linux/io.h> +#include <asm/hardware/cache-l2x0.h> + +#include "gator.h" + +#define L2C310_COUNTERS_NUM 2 + +static struct { + unsigned long enabled; + unsigned long event; + unsigned long key; +} l2c310_counters[L2C310_COUNTERS_NUM]; + +static int l2c310_buffer[L2C310_COUNTERS_NUM * 2]; + +static void __iomem *l2c310_base; + + + +static void gator_events_l2c310_reset_counters(void) +{ + u32 val = readl(l2c310_base + L2X0_EVENT_CNT_CTRL); + + val |= ((1 << L2C310_COUNTERS_NUM) - 1) << 1; + + writel(val, l2c310_base + L2X0_EVENT_CNT_CTRL); +} + + +static int gator_events_l2c310_create_files(struct super_block *sb, + struct dentry *root) +{ + int i; + + for (i = 0; i < L2C310_COUNTERS_NUM; i++) { + char buf[16]; + struct dentry *dir; + + snprintf(buf, sizeof(buf), "L2C-310_cnt%d", i); + dir = gatorfs_mkdir(sb, root, buf); + if (WARN_ON(!dir)) + return -1; + gatorfs_create_ulong(sb, dir, "enabled", + &l2c310_counters[i].enabled); + gatorfs_create_ulong(sb, dir, "event", + &l2c310_counters[i].event); + gatorfs_create_ro_ulong(sb, dir, "key", + &l2c310_counters[i].key); + } + + return 0; +} + +static int gator_events_l2c310_start(void) +{ + static const unsigned long l2x0_event_cntx_cfg[L2C310_COUNTERS_NUM] = { + L2X0_EVENT_CNT0_CFG, + L2X0_EVENT_CNT1_CFG, + }; + int i; + + /* Counter event sources */ + for (i = 0; i < L2C310_COUNTERS_NUM; i++) + writel((l2c310_counters[i].event & 0xf) << 2, + l2c310_base + l2x0_event_cntx_cfg[i]); + + gator_events_l2c310_reset_counters(); + + /* Event counter enable */ + writel(1, l2c310_base + L2X0_EVENT_CNT_CTRL); + + return 0; +} + +static void gator_events_l2c310_stop(void) +{ + /* Event counter disable */ + writel(0, l2c310_base + L2X0_EVENT_CNT_CTRL); +} + +static int gator_events_l2c310_read(int **buffer) +{ + static const unsigned long l2x0_event_cntx_val[L2C310_COUNTERS_NUM] = { + L2X0_EVENT_CNT0_VAL, + L2X0_EVENT_CNT1_VAL, + }; + int i; + int len = 0; + + if (smp_processor_id()) + return 0; + + for (i = 0; i < L2C310_COUNTERS_NUM; i++) { + if (l2c310_counters[i].enabled) { + l2c310_buffer[len++] = l2c310_counters[i].key; + l2c310_buffer[len++] = readl(l2c310_base + + l2x0_event_cntx_val[i]); + } + } + + /* l2c310 counters are saturating, not wrapping in case of overflow */ + gator_events_l2c310_reset_counters(); + + if (buffer) + *buffer = l2c310_buffer; + + return len; +} + +static struct gator_interface gator_events_l2c310_interface = { + .create_files = gator_events_l2c310_create_files, + .start = gator_events_l2c310_start, + .stop = gator_events_l2c310_stop, + .read = gator_events_l2c310_read, +}; + +static void __maybe_unused gator_events_l2c310_probe(unsigned long phys) +{ + if (l2c310_base) + return; + + l2c310_base = ioremap(phys, SZ_4K); + if (l2c310_base) { + u32 cache_id = readl(l2c310_base + L2X0_CACHE_ID); + + if ((cache_id & 0xff0003c0) != 0x410000c0) { + iounmap(l2c310_base); + l2c310_base = NULL; + } + } +} + +int gator_events_l2c310_init(void) +{ + int i; + + if (gator_cpuid() != CORTEX_A5 && gator_cpuid() != CORTEX_A9) + return -1; + +#if defined(CONFIG_ARCH_EXYNOS4) + gator_events_l2c310_probe(0xfe600000); +#endif +#if defined(CONFIG_ARCH_S5PV310) + gator_events_l2c310_probe(0x10502000); +#endif +#if defined(CONFIG_ARCH_OMAP4) + gator_events_l2c310_probe(0x48242000); +#endif +#if defined(CONFIG_ARCH_TEGRA) + gator_events_l2c310_probe(0x50043000); +#endif +#if defined(CONFIG_ARCH_U8500) + gator_events_l2c310_probe(0xa0412000); +#endif +#if defined(CONFIG_ARCH_VEXPRESS) + // A9x4 core tile (HBI-0191) + gator_events_l2c310_probe(0x1e00a000); + // New memory map tiles + gator_events_l2c310_probe(0x2c0f0000); +#endif + if (!l2c310_base) + return -1; + + for (i = 0; i < L2C310_COUNTERS_NUM; i++) { + l2c310_counters[i].enabled = 0; + l2c310_counters[i].key = gator_events_get_key(); + } + + return gator_events_install(&gator_events_l2c310_interface); +} +gator_events_init(gator_events_l2c310_init); diff --git a/driver/gator_events_meminfo.c b/driver/gator_events_meminfo.c index f1595bd..a1a2031 100644 --- a/driver/gator_events_meminfo.c +++ b/driver/gator_events_meminfo.c @@ -19,7 +19,7 @@ static ulong meminfo_global_enabled; static ulong meminfo_enabled[MEMINFO_TOTAL]; static ulong meminfo_key[MEMINFO_TOTAL]; -static int meminfo_buffer[MEMINFO_TOTAL * 2]; +static unsigned long long meminfo_buffer[MEMINFO_TOTAL * 2]; static int meminfo_length = 0; static unsigned int mem_event = 0; static bool new_data_avail; @@ -120,7 +120,8 @@ static void gator_events_meminfo_stop(void) static void wq_sched_handler(struct work_struct *wsptr) { struct sysinfo info; - int i, len, value; + int i, len; + unsigned long long value; meminfo_length = len = 0; @@ -141,7 +142,7 @@ static void wq_sched_handler(struct work_struct *wsptr) value = 0; break; } - meminfo_buffer[len++] = meminfo_key[i]; + meminfo_buffer[len++] = (unsigned long long)meminfo_key[i]; meminfo_buffer[len++] = value; } } @@ -150,7 +151,7 @@ static void wq_sched_handler(struct work_struct *wsptr) new_data_avail = true; } -static int gator_events_meminfo_read(int **buffer) +static int gator_events_meminfo_read(long long **buffer) { static unsigned int last_mem_event = 0; @@ -177,7 +178,7 @@ static struct gator_interface gator_events_meminfo_interface = { .create_files = gator_events_meminfo_create_files, .start = gator_events_meminfo_start, .stop = gator_events_meminfo_stop, - .read = gator_events_meminfo_read, + .read64 = gator_events_meminfo_read, }; int gator_events_meminfo_init(void) diff --git a/driver/gator_events_pl310.c b/driver/gator_events_pl310.c deleted file mode 100644 index 0ef0cf3..0000000 --- a/driver/gator_events_pl310.c +++ /dev/null @@ -1,176 +0,0 @@ -/** - * PL310 (L2 Cache Controller) event counters for gator - * - * Copyright (C) ARM Limited 2010-2011. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ - -#include <linux/init.h> -#include <linux/io.h> -#include <asm/hardware/cache-l2x0.h> - -#include "gator.h" - -#define PL310_COUNTERS_NUM 2 - -static struct { - unsigned long enabled; - unsigned long event; - unsigned long key; -} pl310_counters[PL310_COUNTERS_NUM]; - -static int pl310_buffer[PL310_COUNTERS_NUM * 2]; - -static void __iomem *pl310_base; - - - -static void gator_events_pl310_reset_counters(void) -{ - u32 val = readl(pl310_base + L2X0_EVENT_CNT_CTRL); - - val |= ((1 << PL310_COUNTERS_NUM) - 1) << 1; - - writel(val, pl310_base + L2X0_EVENT_CNT_CTRL); -} - - -static int gator_events_pl310_create_files(struct super_block *sb, - struct dentry *root) -{ - int i; - - for (i = 0; i < PL310_COUNTERS_NUM; i++) { - char buf[16]; - struct dentry *dir; - - snprintf(buf, sizeof(buf), "PL310_cnt%d", i); - dir = gatorfs_mkdir(sb, root, buf); - if (WARN_ON(!dir)) - return -1; - gatorfs_create_ulong(sb, dir, "enabled", - &pl310_counters[i].enabled); - gatorfs_create_ulong(sb, dir, "event", - &pl310_counters[i].event); - gatorfs_create_ro_ulong(sb, dir, "key", - &pl310_counters[i].key); - } - - return 0; -} - -static int gator_events_pl310_start(void) -{ - static const unsigned long l2x0_event_cntx_cfg[PL310_COUNTERS_NUM] = { - L2X0_EVENT_CNT0_CFG, - L2X0_EVENT_CNT1_CFG, - }; - int i; - - /* Counter event sources */ - for (i = 0; i < PL310_COUNTERS_NUM; i++) - writel((pl310_counters[i].event & 0xf) << 2, - pl310_base + l2x0_event_cntx_cfg[i]); - - gator_events_pl310_reset_counters(); - - /* Event counter enable */ - writel(1, pl310_base + L2X0_EVENT_CNT_CTRL); - - return 0; -} - -static void gator_events_pl310_stop(void) -{ - /* Event counter disable */ - writel(0, pl310_base + L2X0_EVENT_CNT_CTRL); -} - -static int gator_events_pl310_read(int **buffer) -{ - static const unsigned long l2x0_event_cntx_val[PL310_COUNTERS_NUM] = { - L2X0_EVENT_CNT0_VAL, - L2X0_EVENT_CNT1_VAL, - }; - int i; - int len = 0; - - if (smp_processor_id()) - return 0; - - for (i = 0; i < PL310_COUNTERS_NUM; i++) { - if (pl310_counters[i].enabled) { - pl310_buffer[len++] = pl310_counters[i].key; - pl310_buffer[len++] = readl(pl310_base + - l2x0_event_cntx_val[i]); - } - } - - /* PL310 counters are saturating, not wrapping in case of overflow */ - gator_events_pl310_reset_counters(); - - if (buffer) - *buffer = pl310_buffer; - - return len; -} - -static struct gator_interface gator_events_pl310_interface = { - .create_files = gator_events_pl310_create_files, - .start = gator_events_pl310_start, - .stop = gator_events_pl310_stop, - .read = gator_events_pl310_read, -}; - -static void __maybe_unused gator_events_pl310_probe(unsigned long phys) -{ - if (pl310_base) - return; - - pl310_base = ioremap(phys, SZ_4K); - if (pl310_base) { - u32 cache_id = readl(pl310_base + L2X0_CACHE_ID); - - if ((cache_id & 0xff0003c0) != 0x410000c0) { - iounmap(pl310_base); - pl310_base = NULL; - } - } -} - -int gator_events_pl310_init(void) -{ - int i; - -#if defined(CONFIG_ARCH_EXYNOS4) - gator_events_pl310_probe(0xfe600000); -#endif -#if defined(CONFIG_ARCH_OMAP4) - gator_events_pl310_probe(0x48242000); -#endif -#if defined(CONFIG_ARCH_TEGRA) - gator_events_pl310_probe(0x50043000); -#endif -#if defined(CONFIG_ARCH_U8500) - gator_events_pl310_probe(0xa0412000); -#endif -#if defined(CONFIG_ARCH_VEXPRESS) && !defined(CONFIG_ARCH_VEXPRESS_CA15X4) - // A9x4 core tile (HBI-0191) - gator_events_pl310_probe(0x1e00a000); - // New memory map tiles - gator_events_pl310_probe(0x2c0f0000); -#endif - if (!pl310_base) - return -1; - - for (i = 0; i < PL310_COUNTERS_NUM; i++) { - pl310_counters[i].enabled = 0; - pl310_counters[i].key = gator_events_get_key(); - } - - return gator_events_install(&gator_events_pl310_interface); -} -gator_events_init(gator_events_pl310_init); diff --git a/driver/gator_events_scorpion.c b/driver/gator_events_scorpion.c index d831a50..f51e292 100644 --- a/driver/gator_events_scorpion.c +++ b/driver/gator_events_scorpion.c @@ -12,7 +12,7 @@ #define SCORPIONMP 0x2d static const char *pmnc_name; -static int pmnc_count; +static int pmnc_counters; // Per-CPU PMNC: config reg #define PMNC_E (1 << 0) /* Enable all counters */ @@ -32,6 +32,7 @@ static int pmnc_count; static unsigned long pmnc_enabled[CNTMAX]; static unsigned long pmnc_event[CNTMAX]; +static unsigned long pmnc_count[CNTMAX]; static unsigned long pmnc_key[CNTMAX]; static DEFINE_PER_CPU(int[CNTMAX], perfPrev); @@ -209,6 +210,10 @@ static const struct scorp_evt sc_evt[] = { {SCORPION_EXCEPTIONS_UNDERFLOW, 0x80000c00, 4, 0x5d}, {SCORPION_EXCEPTIONS_DENORM, 0x8c000000, 4, 0x5f}, +#ifdef CONFIG_ARCH_MSM_SCORPIONMP + {SCORPIONMP_NUM_BARRIERS, 0x80000e00, 3, 0x59}, + {SCORPIONMP_BARRIER_CYCLES, 0x800e0000, 3, 0x5a}, +#else {SCORPION_BANK_AB_HIT, 0x80000001, 3, 0x58}, {SCORPION_BANK_AB_ACCESS, 0x80000100, 3, 0x59}, {SCORPION_BANK_CD_HIT, 0x80010000, 3, 0x5a}, @@ -228,6 +233,7 @@ static const struct scorp_evt sc_evt[] = { {SCORPION_BANK_AB_L2_CASTOUT, 0x80000c00, 3, 0x59}, {SCORPION_BANK_CD_NON_CASTOUT, 0x800c0000, 3, 0x5a}, {SCORPION_BANK_CD_L2_CASTOUT, 0x8c000000, 3, 0x5b}, +#endif }; static inline void scorpion_pmnc_write(u32 val) @@ -498,7 +504,7 @@ static int gator_events_scorpion_create_files(struct super_block *sb, struct den struct dentry *dir; int i; - for (i = 0; i < pmnc_count; i++) { + for (i = 0; i < pmnc_counters; i++) { char buf[40]; if (i == 0) { snprintf(buf, sizeof buf, "%s_ccnt", pmnc_name); @@ -510,10 +516,11 @@ static int gator_events_scorpion_create_files(struct super_block *sb, struct den return -1; } gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); + gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]); + gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); if (i > 0) { gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); } - gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); } return 0; @@ -565,6 +572,20 @@ static void gator_events_scorpion_offline(void) // investigate: need to do the clearpmu() here on each counter? } +static int gator_events_scorpion_start(void) +{ + int cnt; + + for (cnt = CCNT; cnt < CNTMAX; cnt++) { + if (pmnc_count[cnt] > 0) { + pr_err("gator: event based sampling not supported on Scorpion cores\n"); + return -1; + } + } + + return 0; +} + static void gator_events_scorpion_stop(void) { unsigned int cnt; @@ -572,6 +593,7 @@ static void gator_events_scorpion_stop(void) for (cnt = CCNT; cnt < CNTMAX; cnt++) { pmnc_enabled[cnt] = 0; pmnc_event[cnt] = 0; + pmnc_count[cnt] = 0; } } @@ -580,10 +602,10 @@ static int gator_events_scorpion_read(int **buffer) int cnt, len = 0; int cpu = smp_processor_id(); - if (!pmnc_count) + if (!pmnc_counters) return 0; - for (cnt = 0; cnt < pmnc_count; cnt++) { + for (cnt = 0; cnt < pmnc_counters; cnt++) { if (pmnc_enabled[cnt]) { int value; if (cnt == CCNT) { @@ -611,6 +633,7 @@ static int gator_events_scorpion_read(int **buffer) static struct gator_interface gator_events_scorpion_interface = { .create_files = gator_events_scorpion_create_files, + .start = gator_events_scorpion_start, .stop = gator_events_scorpion_stop, .online = gator_events_scorpion_online, .offline = gator_events_scorpion_offline, @@ -636,21 +659,22 @@ int gator_events_scorpion_init(void) switch (gator_cpuid()) { case SCORPION: pmnc_name = "Scorpion"; - pmnc_count = 4; + pmnc_counters = 4; break; case SCORPIONMP: pmnc_name = "ScorpionMP"; - pmnc_count = 4; + pmnc_counters = 4; break; default: return -1; } - pmnc_count++; // CNT[n] + CCNT + pmnc_counters++; // CNT[n] + CCNT for (cnt = CCNT; cnt < CNTMAX; cnt++) { pmnc_enabled[cnt] = 0; pmnc_event[cnt] = 0; + pmnc_count[cnt] = 0; pmnc_key[cnt] = gator_events_get_key(); } diff --git a/driver/gator_main.c b/driver/gator_main.c index 340756e..87e7f2a 100644 --- a/driver/gator_main.c +++ b/driver/gator_main.c @@ -7,7 +7,7 @@ * */ -static unsigned long gator_protocol_version = 5; +static unsigned long gator_protocol_version = 6; #include <linux/slab.h> #include <linux/cpu.h> @@ -36,11 +36,13 @@ static unsigned long gator_protocol_version = 5; #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined #endif +#if defined (__arm__) #ifdef CONFIG_SMP #ifndef CONFIG_LOCAL_TIMERS #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems #endif #endif +#endif #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32) #error kernels prior to 2.6.32 are not supported @@ -49,22 +51,24 @@ static unsigned long gator_protocol_version = 5; /****************************************************************************** * DEFINES ******************************************************************************/ -#define BUFFER_SIZE_DEFAULT (256*1024) -#define SYNC_FREQ_DEFAULT 1000 +#define TIMER_BUFFER_SIZE_DEFAULT (256*1024) +#define EVENT_BUFFER_SIZE_DEFAULT (128*1024) #define NO_COOKIE 0UL #define INVALID_COOKIE ~0UL -#define PROTOCOL_FRAME ~0 -#define PROTOCOL_START_TICK 1 -#define PROTOCOL_END_TICK 3 -#define PROTOCOL_START_BACKTRACE 5 -#define PROTOCOL_END_BACKTRACE 7 -#define PROTOCOL_COOKIE 9 -#define PROTOCOL_SCHEDULER_TRACE 11 -#define PROTOCOL_COUNTERS 13 -#define PROTOCOL_ANNOTATE 15 -#define PROTOCOL_CPU_SYNC 17 +#define FRAME_HRTIMER 1 +#define FRAME_EVENT 2 +#define FRAME_ANNOTATE 3 + +#define MESSAGE_COOKIE 1 +#define MESSAGE_COUNTERS 3 +#define MESSAGE_START_BACKTRACE 5 +#define MESSAGE_END_BACKTRACE 7 +#define MESSAGE_SCHEDULER_TRACE 9 +#define MESSAGE_PID_NAME 11 + +#define LINUX_PMU_SUPPORT LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35) && defined(CONFIG_CPU_HAS_PMU) #if defined(__arm__) #define PC_REG regs->ARM_pc @@ -72,42 +76,50 @@ static unsigned long gator_protocol_version = 5; #define PC_REG regs->ip #endif +enum {TIMER_BUF, EVENT_BUF, NUM_GATOR_BUFS}; + /****************************************************************************** - * PER CPU + * Globals ******************************************************************************/ static unsigned long gator_cpu_cores; -static unsigned long gator_buffer_size; +static unsigned long userspace_buffer_size; static unsigned long gator_backtrace_depth; static unsigned long gator_started; static unsigned long gator_buffer_opened; static unsigned long gator_timer_count; static unsigned long gator_streaming; -static int gator_master_tick; static DEFINE_MUTEX(start_mutex); static DEFINE_MUTEX(gator_buffer_mutex); unsigned long gator_net_traffic; +bool event_based_sampling; #define COMMIT_SIZE 128 #define COMMIT_MASK (COMMIT_SIZE-1) -static DEFINE_SPINLOCK(gator_commit_lock); -static int *gator_commit; -static int gator_commit_read; -static int gator_commit_write; +static DEFINE_SPINLOCK(timer_commit_lock); +static int *gator_commit[NUM_GATOR_BUFS]; +static int gator_commit_read[NUM_GATOR_BUFS]; +static int gator_commit_write[NUM_GATOR_BUFS]; static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait); -static DEFINE_PER_CPU(int, gator_cpu_sync); -static DEFINE_PER_CPU(int, gator_cpu_tick); static DEFINE_PER_CPU(int, gator_first_time); +#if LINUX_PMU_SUPPORT +static void event_buffer_check(int cpu); +static DEFINE_SPINLOCK(event_commit_lock); +#endif + /****************************************************************************** * Prototypes ******************************************************************************/ -static void gator_buffer_write_packed_int(int cpu, unsigned int x); -static void gator_buffer_write_string(int cpu, char *x); +static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x); +static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x); +static void gator_buffer_write_string(int cpu, int buftype, char *x); static int gator_write_packed_int(char *buffer, unsigned int x); -static void gator_add_trace(int cpu, unsigned int address); +static int gator_write_packed_int64(char *buffer, unsigned long long x); +static void gator_add_trace(int cpu, int buftype, unsigned int address); +static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs); static uint64_t gator_get_time(void); /****************************************************************************** @@ -118,6 +130,7 @@ static uint64_t gator_get_time(void); #include "gator_backtrace.c" #include "gator_annotate.c" #include "gator_fs.c" +#include "gator_ebs.c" /****************************************************************************** * Misc @@ -134,204 +147,137 @@ u32 gator_cpuid(void) /****************************************************************************** * Commit interface ******************************************************************************/ -static int buffer_commit_ready(void) +static int buffer_commit_ready(int buftype) { - return (gator_commit_read != gator_commit_write); + return gator_commit_read[buftype] != gator_commit_write[buftype]; } -static void buffer_commit_read(int *cpu, int *readval, int *writeval) +static void buffer_commit_read(int *cpu, int buftype, int *readval, int *writeval) { - int read = gator_commit_read; - *cpu = gator_commit[read+0]; - *readval = gator_commit[read+1]; - *writeval = gator_commit[read+2]; - gator_commit_read = (read + 4) & COMMIT_MASK; + int read = gator_commit_read[buftype]; + *cpu = gator_commit[buftype][read+0]; + *readval = gator_commit[buftype][read+1]; + *writeval = gator_commit[buftype][read+2]; + gator_commit_read[buftype] = (read + 4) & COMMIT_MASK; } -static void buffer_commit_write(int cpu, int readval, int writeval) { - int write = gator_commit_write; - gator_commit[write+0] = cpu; - gator_commit[write+1] = readval; - gator_commit[write+2] = writeval; - gator_commit_write = (write + 4) & COMMIT_MASK; +static void buffer_commit_write(int cpu, int buftype, int readval, int writeval) { + int write = gator_commit_write[buftype]; + gator_commit[buftype][write+0] = cpu; + gator_commit[buftype][write+1] = readval; + gator_commit[buftype][write+2] = writeval; + gator_commit_write[buftype] = (write + 4) & COMMIT_MASK; } /****************************************************************************** * Buffer management ******************************************************************************/ -static uint32_t use_buffer_size; -static uint32_t use_buffer_mask; -static DEFINE_PER_CPU(int, use_buffer_seq); -static DEFINE_PER_CPU(int, use_buffer_read); -static DEFINE_PER_CPU(int, use_buffer_write); -static DEFINE_PER_CPU(char *, use_buffer); - -static void gator_buffer_write_packed_int(int cpu, unsigned int x) -{ - uint32_t write = per_cpu(use_buffer_write, cpu); - uint32_t mask = use_buffer_mask; - char *buffer = per_cpu(use_buffer, cpu); - int write0 = (write + 0) & mask; - int write1 = (write + 1) & mask; - int write2 = (write + 2) & mask; - int write3 = (write + 3) & mask; - int write4 = (write + 4) & mask; - int write5 = (write + 5) & mask; - - if ((x & 0xffffff80) == 0) { - buffer[write0] = x & 0x7f; - per_cpu(use_buffer_write, cpu) = write1; - } else if ((x & 0xffffc000) == 0) { - buffer[write0] = x | 0x80; - buffer[write1] = (x>>7) & 0x7f; - per_cpu(use_buffer_write, cpu) = write2; - } else if ((x & 0xffe00000) == 0) { - buffer[write0] = x | 0x80; - buffer[write1] = (x>>7) | 0x80; - buffer[write2] = (x>>14) & 0x7f; - per_cpu(use_buffer_write, cpu) = write3; - } else if ((x & 0xf0000000) == 0) { - buffer[write0] = x | 0x80; - buffer[write1] = (x>>7) | 0x80; - buffer[write2] = (x>>14) | 0x80; - buffer[write3] = (x>>21) & 0x7f; - per_cpu(use_buffer_write, cpu) = write4; - } else { - buffer[write0] = x | 0x80; - buffer[write1] = (x>>7) | 0x80; - buffer[write2] = (x>>14) | 0x80; - buffer[write3] = (x>>21) | 0x80; - buffer[write4] = (x>>28) & 0x0f; - per_cpu(use_buffer_write, cpu) = write5; - } -} +static uint32_t gator_buffer_size[NUM_GATOR_BUFS]; +static uint32_t gator_buffer_mask[NUM_GATOR_BUFS]; +static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read); +static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write); +static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer); +#include "gator_pack.c" -static int gator_write_packed_int(char *buffer, unsigned int x) +static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len) { - if ((x & 0xffffff80) == 0) { - buffer[0] = x & 0x7f; - return 1; - } else if ((x & 0xffffc000) == 0) { - buffer[0] = x | 0x80; - buffer[1] = (x>>7) & 0x7f; - return 2; - } else if ((x & 0xffe00000) == 0) { - buffer[0] = x | 0x80; - buffer[1] = (x>>7) | 0x80; - buffer[2] = (x>>14) & 0x7f; - return 3; - } else if ((x & 0xf0000000) == 0) { - buffer[0] = x | 0x80; - buffer[1] = (x>>7) | 0x80; - buffer[2] = (x>>14) | 0x80; - buffer[3] = (x>>21) & 0x7f; - return 4; - } else { - buffer[0] = x | 0x80; - buffer[1] = (x>>7) | 0x80; - buffer[2] = (x>>14) | 0x80; - buffer[3] = (x>>21) | 0x80; - buffer[4] = (x>>28) & 0x0f; - return 5; - } -} - -static void gator_buffer_write_bytes(int cpu, char *x, int len) -{ - uint32_t write = per_cpu(use_buffer_write, cpu); - uint32_t mask = use_buffer_mask; - char *buffer = per_cpu(use_buffer, cpu); int i; + u32 write = per_cpu(gator_buffer_write, cpu)[buftype]; + u32 mask = gator_buffer_mask[buftype]; + char* buffer = per_cpu(gator_buffer, cpu)[buftype]; for (i = 0; i < len; i++) { buffer[write] = x[i]; write = (write + 1) & mask; } - per_cpu(use_buffer_write, cpu) = write; + per_cpu(gator_buffer_write, cpu)[buftype] = write; } -static void gator_buffer_write_string(int cpu, char *x) +static void gator_buffer_write_string(int cpu, int buftype, char *x) { int len = strlen(x); - gator_buffer_write_packed_int(cpu, len); - gator_buffer_write_bytes(cpu, x, len); + gator_buffer_write_packed_int(cpu, buftype, len); + gator_buffer_write_bytes(cpu, buftype, x, len); } -static void gator_buffer_header(int cpu) +static void gator_buffer_header(int cpu, int buftype) { - gator_buffer_write_packed_int(cpu, PROTOCOL_FRAME); - gator_buffer_write_packed_int(cpu, cpu); - gator_buffer_write_packed_int(cpu, per_cpu(use_buffer_seq, cpu)); - per_cpu(use_buffer_seq, cpu)++; + int frame; + + if (buftype == TIMER_BUF) + frame = FRAME_HRTIMER; + else if (buftype == EVENT_BUF) + frame = FRAME_EVENT; + else + frame = -1; + + gator_buffer_write_packed_int(cpu, buftype, frame); + gator_buffer_write_packed_int(cpu, buftype, cpu); } -static void gator_buffer_commit(int cpu) +static void gator_buffer_commit(int cpu, int buftype) { - buffer_commit_write(cpu, per_cpu(use_buffer_read, cpu), per_cpu(use_buffer_write, cpu)); - per_cpu(use_buffer_read, cpu) = per_cpu(use_buffer_write, cpu); - gator_buffer_header(cpu); + buffer_commit_write(cpu, buftype, per_cpu(gator_buffer_read, cpu)[buftype], per_cpu(gator_buffer_write, cpu)[buftype]); + per_cpu(gator_buffer_read, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype]; + gator_buffer_header(cpu, buftype); wake_up(&gator_buffer_wait); } -static void gator_buffer_check(int cpu, int tick) +static void timer_buffer_check(int cpu) { - if (!(tick % gator_timer_count)) { - int c, sync; - spin_lock(&gator_commit_lock); - // synchronize, if all online cpus have the same tick waypoint - sync = per_cpu(gator_cpu_sync, cpu) = per_cpu(gator_cpu_tick, cpu); - for_each_online_cpu(c) { - if (sync != per_cpu(gator_cpu_sync, c)) { - sync = 0; - break; - } - } - if (sync) { - gator_buffer_write_packed_int(cpu, PROTOCOL_CPU_SYNC); - } - gator_buffer_commit(cpu); - spin_unlock(&gator_commit_lock); - } else { - int available = per_cpu(use_buffer_write, cpu) - per_cpu(use_buffer_read, cpu); - if (available < 0) { - available += use_buffer_size; - } - if (available >= ((use_buffer_size * 3) / 4)) { - spin_lock(&gator_commit_lock); - gator_buffer_commit(cpu); - spin_unlock(&gator_commit_lock); - } + int available = per_cpu(gator_buffer_write, cpu)[TIMER_BUF] - per_cpu(gator_buffer_read, cpu)[TIMER_BUF]; + if (available < 0) { + available += gator_buffer_size[TIMER_BUF]; + } + if (available >= ((gator_buffer_size[TIMER_BUF] * 3) / 4)) { + spin_lock(&timer_commit_lock); + gator_buffer_commit(cpu, TIMER_BUF); + spin_unlock(&timer_commit_lock); + } +} + +#if LINUX_PMU_SUPPORT +static void event_buffer_check(int cpu) +{ + int available = per_cpu(gator_buffer_write, cpu)[EVENT_BUF] - per_cpu(gator_buffer_read, cpu)[EVENT_BUF]; + if (available < 0) { + available += gator_buffer_size[EVENT_BUF]; + } + if (available >= ((gator_buffer_size[EVENT_BUF] * 3) / 4)) { + spin_lock(&event_commit_lock); + gator_buffer_commit(cpu, EVENT_BUF); + spin_unlock(&event_commit_lock); } } +#endif -static void gator_add_trace(int cpu, unsigned int address) +static void gator_add_trace(int cpu, int buftype, unsigned int address) { off_t offset = 0; - unsigned long cookie = get_address_cookie(cpu, current, address & ~1, &offset); + unsigned long cookie = get_address_cookie(cpu, buftype, current, address & ~1, &offset); if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) { offset = address; } - gator_buffer_write_packed_int(cpu, offset & ~1); - gator_buffer_write_packed_int(cpu, cookie); + gator_buffer_write_packed_int(cpu, buftype, offset & ~1); + gator_buffer_write_packed_int(cpu, buftype, cookie); } -static void gator_add_sample(int cpu, struct pt_regs * const regs) +static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs) { struct module *mod; unsigned int addr, cookie = 0; int inKernel = regs ? !user_mode(regs) : 1; - unsigned long exec_cookie = !inKernel ? get_exec_cookie(cpu, current) : NO_COOKIE; + unsigned long exec_cookie = inKernel ? NO_COOKIE : get_exec_cookie(cpu, buftype, current); - gator_buffer_write_packed_int(cpu, PROTOCOL_START_BACKTRACE); - - // TGID::PID::inKernel - gator_buffer_write_packed_int(cpu, exec_cookie); - gator_buffer_write_packed_int(cpu, (unsigned int)current->tgid); - gator_buffer_write_packed_int(cpu, (unsigned int)current->pid); - gator_buffer_write_packed_int(cpu, inKernel); + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_START_BACKTRACE); + gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); + gator_buffer_write_packed_int(cpu, buftype, exec_cookie); + gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->tgid); + gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->pid); + gator_buffer_write_packed_int(cpu, buftype, inKernel); // get_irq_regs() will return NULL outside of IRQ context (e.g. nested IRQ) if (regs) { @@ -339,36 +285,26 @@ static void gator_add_sample(int cpu, struct pt_regs * const regs) addr = PC_REG; mod = __module_address(addr); if (mod) { - cookie = get_cookie(cpu, current, NULL, mod); + cookie = get_cookie(cpu, buftype, current, NULL, mod, true); addr = addr - (unsigned long)mod->module_core; } - gator_buffer_write_packed_int(cpu, addr & ~1); - gator_buffer_write_packed_int(cpu, cookie); + gator_buffer_write_packed_int(cpu, buftype, addr & ~1); + gator_buffer_write_packed_int(cpu, buftype, cookie); } else { // Cookie+PC - gator_add_trace(cpu, PC_REG); + gator_add_trace(cpu, buftype, PC_REG); // Backtrace if (gator_backtrace_depth) - arm_backtrace_eabi(cpu, regs, gator_backtrace_depth); + arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth); } } - gator_buffer_write_packed_int(cpu, PROTOCOL_END_BACKTRACE); -} - -static void gator_write_packet(int cpu, int type, int len, int *buffer) -{ - int i; - gator_buffer_write_packed_int(cpu, type); - gator_buffer_write_packed_int(cpu, len); - for (i = 0; i < len; i++) { - gator_buffer_write_packed_int(cpu, buffer[i]); - } + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_END_BACKTRACE); } /****************************************************************************** - * Interrupt Processing + * hrtimer interrupt processing ******************************************************************************/ static LIST_HEAD(gator_events); @@ -376,7 +312,8 @@ static void gator_timer_interrupt(void) { struct pt_regs * const regs = get_irq_regs(); int cpu = smp_processor_id(); - int *buffer, len, tick; + int *buffer, len, i, buftype = TIMER_BUF; + long long *buffer64; struct gator_interface *gi; // check full backtrace has enough space, otherwise may @@ -391,44 +328,48 @@ static void gator_timer_interrupt(void) return; } - // Header - gator_buffer_write_packed_int(cpu, PROTOCOL_START_TICK); // Escape - // Output scheduler - len = gator_trace_sched_read(&buffer); + len = gator_trace_sched_read(&buffer64); if (len > 0) { - gator_write_packet(cpu, PROTOCOL_SCHEDULER_TRACE, len, buffer); + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_SCHEDULER_TRACE); + gator_buffer_write_packed_int(cpu, buftype, len); + for (i = 0; i < len; i++) { + gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]); + } } // Output counters + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); + gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); list_for_each_entry(gi, &gator_events, list) { if (gi->read) { len = gi->read(&buffer); + if (len > 0) { + gator_buffer_write_packed_int(cpu, buftype, len); + for (i = 0; i < len; i++) { + gator_buffer_write_packed_int(cpu, buftype, buffer[i]); + } + } + } else if (gi->read64) { + len = gi->read64(&buffer64); if (len > 0) - gator_write_packet(cpu, PROTOCOL_COUNTERS, len, buffer); + gator_buffer_write_packed_int(cpu, buftype, len); + for (i = 0; i < len; i++) { + gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]); + } } } + gator_buffer_write_packed_int(cpu, buftype, 0); // Output backtrace - gator_add_sample(cpu, regs); - - // Timer Tick - tick = per_cpu(gator_cpu_tick, cpu); - if (tick == gator_master_tick) { - tick++; - per_cpu(gator_cpu_tick, cpu) = gator_master_tick = tick; - } else { - per_cpu(gator_cpu_tick, cpu) = tick = gator_master_tick; + if (!event_based_sampling) { + gator_add_sample(cpu, buftype, regs); } - gator_write_packet(cpu, PROTOCOL_END_TICK, 1, &tick); // Check and commit; generally, commit is set to occur once per second - gator_buffer_check(cpu, tick); + timer_buffer_check(cpu); } -/****************************************************************************** - * hrtimer - ******************************************************************************/ DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer); DEFINE_PER_CPU(int, hrtimer_is_active); static int hrtimer_running; @@ -454,7 +395,9 @@ static void __gator_timer_offline(void *unused) struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu); hrtimer_cancel(hrtimer); per_cpu(hrtimer_is_active, cpu) = 0; - gator_buffer_commit(cpu); + gator_buffer_commit(cpu, TIMER_BUF); + if (event_based_sampling) + gator_buffer_commit(cpu, EVENT_BUF); // offline any events list_for_each_entry(gi, &gator_events, list) @@ -469,10 +412,6 @@ static void gator_timer_offline(void) hrtimer_running = 0; on_each_cpu(__gator_timer_offline, NULL, 1); - - // output a final sync point - gator_buffer_write_packed_int(0, PROTOCOL_CPU_SYNC); - gator_buffer_commit(0); } } @@ -485,7 +424,6 @@ static void __gator_timer_online(void *unused) hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); hrtimer->function = gator_hrtimer_notify; hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED); - per_cpu(gator_cpu_tick, cpu) = 0; per_cpu(gator_first_time, cpu) = 1; per_cpu(hrtimer_is_active, cpu) = 1; @@ -499,10 +437,10 @@ static void __gator_timer_online(void *unused) int gator_timer_online(unsigned long setup) { if (!setup) { - pr_err("gator: cannot start due to a system tick value of zero"); + pr_err("gator: cannot start due to a system tick value of zero\n"); return -1; } else if (hrtimer_running) { - pr_notice("gator: high res timer already running"); + pr_notice("gator: high res timer already running\n"); return 0; } @@ -512,7 +450,6 @@ int gator_timer_online(unsigned long setup) profiling_interval = ns_to_ktime(1000000000UL / setup); // timer interrupt - gator_master_tick = 0; on_each_cpu(__gator_timer_online, NULL, 1); return 0; @@ -623,10 +560,14 @@ static int gator_start(void) } } + if (gator_event_sampling_start()) + goto event_sampling_failure; if (gator_annotate_start()) goto annotate_failure; if (gator_trace_sched_start()) goto sched_failure; + if (cookies_initialize()) + goto cookies_failure; if (gator_timer_online(gator_timer_count)) goto timer_failure; if (gator_notifier_start()) @@ -637,10 +578,18 @@ static int gator_start(void) notifier_failure: gator_timer_offline(); timer_failure: + cookies_release(); +cookies_failure: gator_trace_sched_stop(); sched_failure: gator_annotate_stop(); annotate_failure: + gator_event_sampling_stop(); +event_sampling_failure: + // stop all events + list_for_each_entry(gi, &gator_events, list) + if (gi->stop) + gi->stop(); events_failure: return -1; @@ -655,12 +604,13 @@ static void gator_stop(void) if (gi->stop) gi->stop(); + gator_event_sampling_stop(); gator_annotate_stop(); gator_trace_sched_stop(); // stop all interrupt callback reads before tearing down other interfaces + gator_notifier_stop(); // should be called before gator_timer_offline to avoid re-enabling the hrtimer after it has been offlined gator_timer_offline(); - gator_notifier_stop(); } static void gator_exit(void) @@ -675,42 +625,45 @@ static void gator_exit(void) static int gator_op_setup(void) { int err = 0; - int cpu; + int cpu, i; mutex_lock(&start_mutex); - use_buffer_size = gator_buffer_size; - use_buffer_mask = use_buffer_size - 1; + gator_buffer_size[TIMER_BUF] = userspace_buffer_size; + gator_buffer_mask[TIMER_BUF] = userspace_buffer_size - 1; // must be a power of 2 - if (use_buffer_size & (use_buffer_size - 1)) { + if (gator_buffer_size[TIMER_BUF] & (gator_buffer_size[TIMER_BUF] - 1)) { err = -ENOEXEC; goto setup_error; } - gator_net_traffic = 0; + gator_buffer_size[EVENT_BUF] = EVENT_BUFFER_SIZE_DEFAULT; + gator_buffer_mask[EVENT_BUF] = gator_buffer_size[EVENT_BUF] - 1; - gator_commit_read = gator_commit_write = 0; - gator_commit = vmalloc(COMMIT_SIZE * sizeof(int)); - if (!gator_commit) { - err = -ENOMEM; - goto setup_error; - } + gator_net_traffic = 0; - for_each_present_cpu(cpu) { - per_cpu(use_buffer, cpu) = vmalloc(use_buffer_size); - if (!per_cpu(use_buffer, cpu)) { + // Initialize per buffer variables + for (i = 0; i < NUM_GATOR_BUFS; i++) { + gator_commit_read[i] = gator_commit_write[i] = 0; + gator_commit[i] = vmalloc(COMMIT_SIZE * sizeof(int)); + if (!gator_commit[i]) { err = -ENOMEM; goto setup_error; } - per_cpu(gator_cpu_sync, cpu) = 0; - per_cpu(gator_cpu_tick, cpu) = 0; + // Initialize percpu per buffer variables + for_each_present_cpu(cpu) { + per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]); + if (!per_cpu(gator_buffer, cpu)[i]) { + err = -ENOMEM; + goto setup_error; + } - per_cpu(use_buffer_seq, cpu) = 0; - per_cpu(use_buffer_read, cpu) = 0; - per_cpu(use_buffer_write, cpu) = 0; - gator_buffer_header(cpu); + per_cpu(gator_buffer_read, cpu)[i] = 0; + per_cpu(gator_buffer_write, cpu)[i] = 0; + gator_buffer_header(cpu, i); + } } setup_error: @@ -725,7 +678,7 @@ static int gator_op_start(void) mutex_lock(&start_mutex); - if (gator_started || gator_start() || cookies_initialize()) + if (gator_started || gator_start()) err = -EINVAL; else gator_started = 1; @@ -757,20 +710,25 @@ static void gator_op_stop(void) static void gator_shutdown(void) { - int cpu; + int cpu, i; mutex_lock(&start_mutex); - vfree(gator_commit); - gator_commit = NULL; + gator_annotate_shutdown(); + + for (i = 0; i < NUM_GATOR_BUFS; i++) { + vfree(gator_commit[i]); + gator_commit[i] = NULL; + } for_each_present_cpu(cpu) { mutex_lock(&gator_buffer_mutex); - vfree(per_cpu(use_buffer, cpu)); - per_cpu(use_buffer, cpu) = NULL; - per_cpu(use_buffer_seq, cpu) = 0; - per_cpu(use_buffer_read, cpu) = 0; - per_cpu(use_buffer_write, cpu) = 0; + for (i = 0; i < NUM_GATOR_BUFS; i++) { + vfree(per_cpu(gator_buffer, cpu)[i]); + per_cpu(gator_buffer, cpu)[i] = NULL; + per_cpu(gator_buffer_read, cpu)[i] = 0; + per_cpu(gator_buffer_write, cpu)[i] = 0; + } mutex_unlock(&gator_buffer_mutex); } @@ -825,7 +783,7 @@ static const struct file_operations enable_fops = { .write = enable_write, }; -static int event_buffer_open(struct inode *inode, struct file *file) +static int userspace_buffer_open(struct inode *inode, struct file *file) { int err = -EPERM; @@ -849,7 +807,7 @@ fail: return err; } -static int event_buffer_release(struct inode *inode, struct file *file) +static int userspace_buffer_release(struct inode *inode, struct file *file) { gator_op_stop(); gator_shutdown(); @@ -857,55 +815,59 @@ static int event_buffer_release(struct inode *inode, struct file *file) return 0; } -static ssize_t event_buffer_read(struct file *file, char __user *buf, +static ssize_t userspace_buffer_read(struct file *file, char __user *buf, size_t count, loff_t *offset) { int retval = -EINVAL; int commit, length1, length2, read; - char *buffer1, *buffer2; - char annotate_header[6]; - int cpu; + char *buffer1; + char *buffer2 = NULL; + int cpu, i; /* do not handle partial reads */ - if (count != use_buffer_size || *offset) + if (count != userspace_buffer_size || *offset) return -EINVAL; // sleep until the condition is true or a signal is received // the condition is checked each time gator_buffer_wait is woken up - wait_event_interruptible(gator_buffer_wait, buffer_commit_ready() || gator_annotate_ready() || !gator_started); + wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(TIMER_BUF) || buffer_commit_ready(EVENT_BUF) || gator_annotate_ready() || !gator_started); if (signal_pending(current)) return -EINTR; + length2 = 0; retval = -EFAULT; mutex_lock(&gator_buffer_mutex); - if (buffer_commit_ready()) { - buffer_commit_read(&cpu, &read, &commit); + i = -1; + if (buffer_commit_ready(TIMER_BUF)) { + i = TIMER_BUF; + } else if (buffer_commit_ready(EVENT_BUF)) { + i = EVENT_BUF; + } + + if (i != -1) { + buffer_commit_read(&cpu, i, &read, &commit); /* May happen if the buffer is freed during pending reads. */ - if (!per_cpu(use_buffer, cpu)) { + if (!per_cpu(gator_buffer, cpu)[i]) { retval = -EFAULT; goto out; } /* determine the size of two halves */ length1 = commit - read; - length2 = 0; - buffer1 = &(per_cpu(use_buffer, cpu)[read]); - buffer2 = &(per_cpu(use_buffer, cpu)[0]); + buffer1 = &(per_cpu(gator_buffer, cpu)[i][read]); + buffer2 = &(per_cpu(gator_buffer, cpu)[i][0]); if (length1 < 0) { - length1 = use_buffer_size - read; + length1 = gator_buffer_size[i] - read; length2 = commit; } } else if (gator_annotate_ready()) { - length2 = gator_annotate_read(&buffer2); - if (!length2) + length1 = gator_annotate_read(&buffer1); + if (!length1) goto out; - annotate_header[0] = PROTOCOL_ANNOTATE; - length1 = gator_write_packed_int(&annotate_header[1], length2) + 1; - buffer1 = annotate_header; } else { retval = 0; goto out; @@ -939,9 +901,9 @@ out: } const struct file_operations gator_event_buffer_fops = { - .open = event_buffer_open, - .release = event_buffer_release, - .read = event_buffer_read, + .open = userspace_buffer_open, + .release = userspace_buffer_release, + .read = userspace_buffer_read, }; static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset) @@ -974,17 +936,6 @@ static const struct file_operations depth_fops = { .write = depth_write }; -static const char gator_cpu_type[] = "gator"; - -static ssize_t cpu_type_read(struct file *file, char __user *buf, size_t count, loff_t *offset) -{ - return gatorfs_str_to_user(gator_cpu_type, buf, count, offset); -} - -static const struct file_operations cpu_type_fops = { - .read = cpu_type_read, -}; - void gator_op_create_files(struct super_block *sb, struct dentry *root) { struct dentry *dir; @@ -996,15 +947,14 @@ void gator_op_create_files(struct super_block *sb, struct dentry *root) for_each_present_cpu(cpu) { gator_cpu_cores++; } - gator_buffer_size = BUFFER_SIZE_DEFAULT; + userspace_buffer_size = TIMER_BUFFER_SIZE_DEFAULT; gator_streaming = 1; gatorfs_create_file(sb, root, "enable", &enable_fops); gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops); gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops); - gatorfs_create_file(sb, root, "cpu_type", &cpu_type_fops); gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores); - gatorfs_create_ulong(sb, root, "buffer_size", &gator_buffer_size); + gatorfs_create_ulong(sb, root, "buffer_size", &userspace_buffer_size); gatorfs_create_ulong(sb, root, "tick", &gator_timer_count); gatorfs_create_ulong(sb, root, "streaming", &gator_streaming); gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version); @@ -1033,17 +983,11 @@ static int __init gator_module_init(void) return -1; } -#ifdef GATOR_DEBUG - pr_err("gator_module_init"); -#endif return 0; } static void __exit gator_module_exit(void) { -#ifdef GATOR_DEBUG - pr_err("gator_module_exit"); -#endif tracepoint_synchronize_unregister(); gatorfs_unregister(); gator_exit(); diff --git a/driver/gator_pack.c b/driver/gator_pack.c new file mode 100644 index 0000000..fbab220 --- /dev/null +++ b/driver/gator_pack.c @@ -0,0 +1,262 @@ +/** + * Copyright (C) ARM Limited 2010-2011. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x) +{ + uint32_t write = per_cpu(gator_buffer_write, cpu)[buftype]; + uint32_t mask = gator_buffer_mask[buftype]; + char *buffer = per_cpu(gator_buffer, cpu)[buftype]; + int write0 = (write + 0) & mask; + int write1 = (write + 1) & mask; + + if ((x & 0xffffff80) == 0) { + buffer[write0] = x & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write1; + } else if ((x & 0xffffc000) == 0) { + int write2 = (write + 2) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write2; + } else if ((x & 0xffe00000) == 0) { + int write2 = (write + 2) & mask; + int write3 = (write + 3) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) | 0x80; + buffer[write2] = (x>>14) & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write3; + } else if ((x & 0xf0000000) == 0) { + int write2 = (write + 2) & mask; + int write3 = (write + 3) & mask; + int write4 = (write + 4) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) | 0x80; + buffer[write2] = (x>>14) | 0x80; + buffer[write3] = (x>>21) & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write4; + } else { + int write2 = (write + 2) & mask; + int write3 = (write + 3) & mask; + int write4 = (write + 4) & mask; + int write5 = (write + 5) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) | 0x80; + buffer[write2] = (x>>14) | 0x80; + buffer[write3] = (x>>21) | 0x80; + buffer[write4] = (x>>28) & 0x0f; + per_cpu(gator_buffer_write, cpu)[buftype] = write5; + } +} + +static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x) +{ + uint32_t write = per_cpu(gator_buffer_write, cpu)[buftype]; + uint32_t mask = gator_buffer_mask[buftype]; + char *buffer = per_cpu(gator_buffer, cpu)[buftype]; + int write0 = (write + 0) & mask; + int write1 = (write + 1) & mask; + + if ((x & 0xffffffffffffff80LL) == 0) { + buffer[write0] = x & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write1; + } else if ((x & 0xffffffffffffc000LL) == 0) { + int write2 = (write + 2) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write2; + } else if ((x & 0xffffffffffe00000LL) == 0) { + int write2 = (write + 2) & mask; + int write3 = (write + 3) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) | 0x80; + buffer[write2] = (x>>14) & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write3; + } else if ((x & 0xfffffffff0000000LL) == 0) { + int write2 = (write + 2) & mask; + int write3 = (write + 3) & mask; + int write4 = (write + 4) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) | 0x80; + buffer[write2] = (x>>14) | 0x80; + buffer[write3] = (x>>21) & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write4; + } else if ((x & 0xfffffff800000000LL) == 0) { + int write2 = (write + 2) & mask; + int write3 = (write + 3) & mask; + int write4 = (write + 4) & mask; + int write5 = (write + 5) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) | 0x80; + buffer[write2] = (x>>14) | 0x80; + buffer[write3] = (x>>21) | 0x80; + buffer[write4] = (x>>28) & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write5; + } else if ((x & 0xfffffc0000000000LL) == 0) { + int write2 = (write + 2) & mask; + int write3 = (write + 3) & mask; + int write4 = (write + 4) & mask; + int write5 = (write + 5) & mask; + int write6 = (write + 6) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) | 0x80; + buffer[write2] = (x>>14) | 0x80; + buffer[write3] = (x>>21) | 0x80; + buffer[write4] = (x>>28) | 0x80; + buffer[write5] = (x>>35) & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write6; + } else if ((x & 0xfffe000000000000LL) == 0) { + int write2 = (write + 2) & mask; + int write3 = (write + 3) & mask; + int write4 = (write + 4) & mask; + int write5 = (write + 5) & mask; + int write6 = (write + 6) & mask; + int write7 = (write + 7) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) | 0x80; + buffer[write2] = (x>>14) | 0x80; + buffer[write3] = (x>>21) | 0x80; + buffer[write4] = (x>>28) | 0x80; + buffer[write5] = (x>>35) | 0x80; + buffer[write6] = (x>>42) & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write7; + } else if ((x & 0xff00000000000000LL) == 0) { + int write2 = (write + 2) & mask; + int write3 = (write + 3) & mask; + int write4 = (write + 4) & mask; + int write5 = (write + 5) & mask; + int write6 = (write + 6) & mask; + int write7 = (write + 7) & mask; + int write8 = (write + 8) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) | 0x80; + buffer[write2] = (x>>14) | 0x80; + buffer[write3] = (x>>21) | 0x80; + buffer[write4] = (x>>28) | 0x80; + buffer[write5] = (x>>35) | 0x80; + buffer[write6] = (x>>42) | 0x80; + buffer[write7] = (x>>49) & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write8; + } else { + int write2 = (write + 2) & mask; + int write3 = (write + 3) & mask; + int write4 = (write + 4) & mask; + int write5 = (write + 5) & mask; + int write6 = (write + 6) & mask; + int write7 = (write + 7) & mask; + int write8 = (write + 8) & mask; + int write9 = (write + 9) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) | 0x80; + buffer[write2] = (x>>14) | 0x80; + buffer[write3] = (x>>21) | 0x80; + buffer[write4] = (x>>28) | 0x80; + buffer[write5] = (x>>35) | 0x80; + buffer[write6] = (x>>42) | 0x80; + buffer[write7] = (x>>49) | 0x80; + buffer[write8] = (x>>56) & 0xff; + per_cpu(gator_buffer_write, cpu)[buftype] = write9; + } +} + +static int gator_write_packed_int(char *buffer, unsigned int x) +{ + if ((x & 0xffffff80) == 0) { + buffer[0] = x & 0x7f; + return 1; + } else if ((x & 0xffffc000) == 0) { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) & 0x7f; + return 2; + } else if ((x & 0xffe00000) == 0) { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) | 0x80; + buffer[2] = (x>>14) & 0x7f; + return 3; + } else if ((x & 0xf0000000) == 0) { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) | 0x80; + buffer[2] = (x>>14) | 0x80; + buffer[3] = (x>>21) & 0x7f; + return 4; + } else { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) | 0x80; + buffer[2] = (x>>14) | 0x80; + buffer[3] = (x>>21) | 0x80; + buffer[4] = (x>>28) & 0x0f; + return 5; + } +} + +static int gator_write_packed_int64(char *buffer, unsigned long long x) +{ + if ((x & 0xffffffffffffff80LL) == 0) { + buffer[0] = x & 0x7f; + return 1; + } else if ((x & 0xffffffffffffc000LL) == 0) { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) & 0x7f; + return 2; + } else if ((x & 0xffffffffffe00000LL) == 0) { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) | 0x80; + buffer[2] = (x>>14) & 0x7f; + return 3; + } else if ((x & 0xfffffffff0000000LL) == 0) { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) | 0x80; + buffer[2] = (x>>14) | 0x80; + buffer[3] = (x>>21) & 0x7f; + return 4; + } else if ((x & 0xfffffff800000000LL) == 0) { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) | 0x80; + buffer[2] = (x>>14) | 0x80; + buffer[3] = (x>>21) | 0x80; + buffer[4] = (x>>28) & 0x7f; + return 5; + } else if ((x & 0xfffffc0000000000LL) == 0) { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) | 0x80; + buffer[2] = (x>>14) | 0x80; + buffer[3] = (x>>21) | 0x80; + buffer[4] = (x>>28) | 0x80; + buffer[5] = (x>>35) & 0x7f; + return 6; + } else if ((x & 0xfffe000000000000LL) == 0) { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) | 0x80; + buffer[2] = (x>>14) | 0x80; + buffer[3] = (x>>21) | 0x80; + buffer[4] = (x>>28) | 0x80; + buffer[5] = (x>>35) | 0x80; + buffer[6] = (x>>42) & 0x7f; + return 7; + } else if ((x & 0xff00000000000000LL) == 0) { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) | 0x80; + buffer[2] = (x>>14) | 0x80; + buffer[3] = (x>>21) | 0x80; + buffer[4] = (x>>28) | 0x80; + buffer[5] = (x>>35) | 0x80; + buffer[6] = (x>>42) | 0x80; + buffer[7] = (x>>49) & 0x7f; + return 8; + } else { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) | 0x80; + buffer[2] = (x>>14) | 0x80; + buffer[3] = (x>>21) | 0x80; + buffer[4] = (x>>28) | 0x80; + buffer[5] = (x>>35) | 0x80; + buffer[6] = (x>>42) | 0x80; + buffer[7] = (x>>49) | 0x80; + buffer[8] = (x>>56) & 0xff; + return 9; + } +} diff --git a/driver/gator_trace_sched.c b/driver/gator_trace_sched.c index 19d8d89..7c0bd47 100644 --- a/driver/gator_trace_sched.c +++ b/driver/gator_trace_sched.c @@ -10,36 +10,89 @@ #include <trace/events/sched.h> #include "gator.h" -#define SCHED_TIMER_EVENT 0 -#define SCHED_WAIT_TASK 1 -#define SCHED_WAKEUP 2 -#define SCHED_WAKEUP_NEW 3 -#define SCHED_SWITCH 4 -#define SCHED_MIGRATE_TASK 5 -#define SCHED_PROCESS_FREE 6 -#define SCHED_PROCESS_EXIT 7 -#define SCHED_PROCESS_WAIT 8 -#define SCHED_PROCESS_FORK 9 #define SCHED_OVERFLOW -1 +#define SCHED_SWITCH 1 +#define SCHED_PROCESS_FREE 2 -#define SCHEDSIZE (16*1024) +#define FIELD_TYPE 0 +#define FIELD_TIME 1 +#define FIELD_PARAM1 2 +#define FIELD_PARAM2 3 +#define FIELD_PARAM3 4 +#define FIELDS_PER_SCHED 5 -static DEFINE_PER_CPU(int *[2], theSchedBuf); +#define SCHEDSIZE (8*1024) +#define TASK_MAP_ENTRIES 1024 /* must be power of 2 */ +#define TASK_MAX_COLLISIONS 2 + +static DEFINE_PER_CPU(uint64_t *[2], theSchedBuf); static DEFINE_PER_CPU(int, theSchedSel); static DEFINE_PER_CPU(int, theSchedPos); static DEFINE_PER_CPU(int, theSchedErr); +static DEFINE_PER_CPU(uint64_t *, taskname_keys); + +void emit_pid_name(uint64_t time, struct task_struct* task) +{ + bool found = false; + unsigned long flags; + char taskcomm[TASK_COMM_LEN + 3]; + int x, cpu = smp_processor_id(); + uint64_t *keys = &(per_cpu(taskname_keys, cpu)[(task->pid & 0xFF) * TASK_MAX_COLLISIONS]); + uint64_t value; + + value = gator_chksum_crc32(task->comm); + value = (value << 32) | (uint32_t)task->pid; + + // determine if the thread name was emitted already + for (x = 0; x < TASK_MAX_COLLISIONS; x++) { + if (keys[x] == value) { + found = true; + break; + } + } + + if (!found) { + // shift values, new value always in front + uint64_t oldv, newv = value; + for (x = 0; x < TASK_MAX_COLLISIONS; x++) { + oldv = keys[x]; + keys[x] = newv; + newv = oldv; + } + + // emit pid names, cannot use get_task_comm, as it's not exported on all kernel versions + if (strlcpy(taskcomm, task->comm, TASK_COMM_LEN) == TASK_COMM_LEN - 1) + // append ellipses if task->comm has length of TASK_COMM_LEN - 1 + strcat(taskcomm, "..."); + + // disable interrupts to synchronize with hrtimer populating timer buf + local_irq_save(flags); + gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_PID_NAME); + gator_buffer_write_packed_int64(cpu, TIMER_BUF, time); + gator_buffer_write_packed_int(cpu, TIMER_BUF, task->pid); + gator_buffer_write_string(cpu, TIMER_BUF, taskcomm); + local_irq_restore(flags); + } +} static void probe_sched_write(int type, int param1, int param2, int param3) { unsigned long flags; int cpu = smp_processor_id(); uint64_t time = gator_get_time(); - int *schedBuf; - int schedPos; + uint64_t *schedBuf; + int schedPos, cookie = param3; if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]) return; + if (param3) { + // do as much work as possible before disabling interrupts + struct task_struct *task = (struct task_struct *)param3; + cookie = get_exec_cookie(cpu, TIMER_BUF, task); + emit_pid_name(time, task); + } + // disable interrupts to synchronize with gator_trace_sched_read(); spinlocks not needed since percpu buffers are used local_irq_save(flags); @@ -48,88 +101,37 @@ static void probe_sched_write(int type, int param1, int param2, int param3) if (schedPos < (SCHEDSIZE-100)) { // capture - schedBuf[schedPos+0] = type; - schedBuf[schedPos+1] = (int)time; - schedBuf[schedPos+2] = (int)(time >> 32); - schedBuf[schedPos+3] = param1; - schedBuf[schedPos+4] = param2; - schedBuf[schedPos+5] = param3; - per_cpu(theSchedPos, cpu) = schedPos + 6; + schedBuf[schedPos+FIELD_TYPE] = type; + schedBuf[schedPos+FIELD_TIME] = time; + schedBuf[schedPos+FIELD_PARAM1] = param1; + schedBuf[schedPos+FIELD_PARAM2] = param2; + schedBuf[schedPos+FIELD_PARAM3] = cookie; + per_cpu(theSchedPos, cpu) = schedPos + FIELDS_PER_SCHED; } else if (!per_cpu(theSchedErr, cpu)) { per_cpu(theSchedErr, cpu) = 1; - schedBuf[schedPos+0] = SCHED_OVERFLOW; - schedBuf[schedPos+1] = 0; - schedBuf[schedPos+2] = 0; - schedBuf[schedPos+3] = 0; - schedBuf[schedPos+4] = 0; - schedBuf[schedPos+5] = 0; - per_cpu(theSchedPos, cpu) = schedPos + 6; + schedBuf[schedPos+FIELD_TYPE] = SCHED_OVERFLOW; + schedBuf[schedPos+FIELD_TIME] = time; + schedBuf[schedPos+FIELD_PARAM1] = 0; + schedBuf[schedPos+FIELD_PARAM2] = 0; + schedBuf[schedPos+FIELD_PARAM3] = 0; + per_cpu(theSchedPos, cpu) = schedPos + FIELDS_PER_SCHED; pr_debug("gator: tracepoint overflow\n"); } local_irq_restore(flags); } #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) -GATOR_DEFINE_PROBE(sched_wait_task, TP_PROTO(struct rq *rq, struct task_struct *p)) -#else -GATOR_DEFINE_PROBE(sched_wait_task, TP_PROTO(struct task_struct *p)) -#endif -{ - probe_sched_write(SCHED_WAIT_TASK, 0, p->pid, 0); -} - -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) -GATOR_DEFINE_PROBE(sched_wakeup, TP_PROTO(struct rq *rq, struct task_struct *p, int success)) -#else -GATOR_DEFINE_PROBE(sched_wakeup, TP_PROTO(struct task_struct *p, int success)) -#endif -{ - if (success) - probe_sched_write(SCHED_WAKEUP, 0, p->pid, 0); -} - -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) -GATOR_DEFINE_PROBE(sched_wakeup_new, TP_PROTO(struct rq *rq, struct task_struct *p, int success)) -#else -GATOR_DEFINE_PROBE(sched_wakeup_new, TP_PROTO(struct task_struct *p, int success)) -#endif -{ - if (success) - probe_sched_write(SCHED_WAKEUP_NEW, 0, p->tgid, p->pid); -} - -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct rq *rq, struct task_struct *prev, struct task_struct *next)) #else GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_struct *next)) #endif { - probe_sched_write(SCHED_SWITCH, (int)next, next->tgid, next->pid); -} - -GATOR_DEFINE_PROBE(sched_migrate_task, TP_PROTO(struct task_struct *p, int dest_cpu)) -{ - probe_sched_write(SCHED_MIGRATE_TASK, 0, dest_cpu, p->pid); + probe_sched_write(SCHED_SWITCH, next->pid, next->tgid, (int)next); } GATOR_DEFINE_PROBE(sched_process_free, TP_PROTO(struct task_struct *p)) { - probe_sched_write(SCHED_PROCESS_FREE, 0, p->pid, 0); -} - -GATOR_DEFINE_PROBE(sched_process_exit, TP_PROTO(struct task_struct *p)) -{ - probe_sched_write(SCHED_PROCESS_EXIT, 0, p->pid, 0); -} - -GATOR_DEFINE_PROBE(sched_process_wait, TP_PROTO(struct pid *pid)) -{ - probe_sched_write(SCHED_PROCESS_WAIT, 0, pid_nr(pid), 0); -} - -GATOR_DEFINE_PROBE(sched_process_fork, TP_PROTO(struct task_struct *parent, struct task_struct *child)) -{ - probe_sched_write(SCHED_PROCESS_FORK, (int)child, parent->pid, child->pid); + probe_sched_write(SCHED_PROCESS_FREE, p->pid, 0, 0); } int gator_trace_sched_init(void) @@ -139,59 +141,37 @@ int gator_trace_sched_init(void) int gator_trace_sched_start(void) { - int cpu; + int cpu, size; for_each_present_cpu(cpu) { per_cpu(theSchedSel, cpu) = 0; per_cpu(theSchedPos, cpu) = 0; per_cpu(theSchedErr, cpu) = 0; - per_cpu(theSchedBuf, cpu)[0] = kmalloc(SCHEDSIZE * sizeof(int), GFP_KERNEL); - per_cpu(theSchedBuf, cpu)[1] = kmalloc(SCHEDSIZE * sizeof(int), GFP_KERNEL); + per_cpu(theSchedBuf, cpu)[0] = kmalloc(SCHEDSIZE * sizeof(uint64_t), GFP_KERNEL); + per_cpu(theSchedBuf, cpu)[1] = kmalloc(SCHEDSIZE * sizeof(uint64_t), GFP_KERNEL); if (!per_cpu(theSchedBuf, cpu)) return -1; + + size = TASK_MAP_ENTRIES * TASK_MAX_COLLISIONS * sizeof(uint64_t); + per_cpu(taskname_keys, cpu) = (uint64_t*)kmalloc(size, GFP_KERNEL); + if (!per_cpu(taskname_keys, cpu)) + return -1; + memset(per_cpu(taskname_keys, cpu), 0, size); } // register tracepoints - if (GATOR_REGISTER_TRACE(sched_wait_task)) - goto fail_sched_wait_task; - if (GATOR_REGISTER_TRACE(sched_wakeup)) - goto fail_sched_wakeup; - if (GATOR_REGISTER_TRACE(sched_wakeup_new)) - goto fail_sched_wakeup_new; if (GATOR_REGISTER_TRACE(sched_switch)) goto fail_sched_switch; - if (GATOR_REGISTER_TRACE(sched_migrate_task)) - goto fail_sched_migrate_task; if (GATOR_REGISTER_TRACE(sched_process_free)) goto fail_sched_process_free; - if (GATOR_REGISTER_TRACE(sched_process_exit)) - goto fail_sched_process_exit; - if (GATOR_REGISTER_TRACE(sched_process_wait)) - goto fail_sched_process_wait; - if (GATOR_REGISTER_TRACE(sched_process_fork)) - goto fail_sched_process_fork; pr_debug("gator: registered tracepoints\n"); return 0; // unregister tracepoints on error -fail_sched_process_fork: - GATOR_UNREGISTER_TRACE(sched_process_wait); -fail_sched_process_wait: - GATOR_UNREGISTER_TRACE(sched_process_exit); -fail_sched_process_exit: - GATOR_UNREGISTER_TRACE(sched_process_free); fail_sched_process_free: - GATOR_UNREGISTER_TRACE(sched_migrate_task); -fail_sched_migrate_task: GATOR_UNREGISTER_TRACE(sched_switch); fail_sched_switch: - GATOR_UNREGISTER_TRACE(sched_wakeup_new); -fail_sched_wakeup_new: - GATOR_UNREGISTER_TRACE(sched_wakeup); -fail_sched_wakeup: - GATOR_UNREGISTER_TRACE(sched_wait_task); -fail_sched_wait_task: pr_err("gator: tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n"); return -1; @@ -200,15 +180,8 @@ fail_sched_wait_task: void gator_trace_sched_stop(void) { int cpu; - GATOR_UNREGISTER_TRACE(sched_wait_task); - GATOR_UNREGISTER_TRACE(sched_wakeup); - GATOR_UNREGISTER_TRACE(sched_wakeup_new); GATOR_UNREGISTER_TRACE(sched_switch); - GATOR_UNREGISTER_TRACE(sched_migrate_task); GATOR_UNREGISTER_TRACE(sched_process_free); - GATOR_UNREGISTER_TRACE(sched_process_exit); - GATOR_UNREGISTER_TRACE(sched_process_wait); - GATOR_UNREGISTER_TRACE(sched_process_fork); pr_debug("gator: unregistered tracepoints\n"); for_each_present_cpu(cpu) { @@ -216,17 +189,16 @@ void gator_trace_sched_stop(void) kfree(per_cpu(theSchedBuf, cpu)[1]); per_cpu(theSchedBuf, cpu)[0] = NULL; per_cpu(theSchedBuf, cpu)[1] = NULL; + kfree(per_cpu(taskname_keys, cpu)); } } -int gator_trace_sched_read(int **buffer) +int gator_trace_sched_read(long long **buffer) { - uint64_t time = gator_get_time(); int cpu = smp_processor_id(); unsigned long flags; - int *schedBuf; + uint64_t *schedBuf; int schedPos; - int i; if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]) return 0; @@ -242,20 +214,6 @@ int gator_trace_sched_read(int **buffer) local_irq_restore(flags); - // find mm and replace with cookies - for (i = 0; i < schedPos; i += 6) { - uint32_t cookie = schedBuf[i+3]; - if (cookie) { - struct task_struct *task = (struct task_struct *)cookie; - schedBuf[i+3] = get_exec_cookie(cpu, task); - } - } - - // timer/end event - schedBuf[schedPos++] = SCHED_TIMER_EVENT; - schedBuf[schedPos++] = (int)time; - schedBuf[schedPos++] = (int)(time >> 32); - if (buffer) *buffer = schedBuf; |