diff options
Diffstat (limited to 'driver/gator_main.c')
-rw-r--r-- | driver/gator_main.c | 492 |
1 files changed, 287 insertions, 205 deletions
diff --git a/driver/gator_main.c b/driver/gator_main.c index 36e951b..fff2d19 100644 --- a/driver/gator_main.c +++ b/driver/gator_main.c @@ -1,5 +1,5 @@ /** - * Copyright (C) ARM Limited 2010-2011. All rights reserved. + * Copyright (C) ARM Limited 2010-2012. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -7,7 +7,7 @@ * */ -static unsigned long gator_protocol_version = 7; +static unsigned long gator_protocol_version = 8; #include <linux/slab.h> #include <linux/cpu.h> @@ -18,16 +18,20 @@ static unsigned long gator_protocol_version = 7; #include <linux/highmem.h> #include <linux/pagemap.h> #include <linux/suspend.h> +#include <linux/module.h> +#include <linux/perf_event.h> #include <asm/stacktrace.h> #include <asm/uaccess.h> #include "gator.h" #include "gator_events.h" -#ifndef CONFIG_GENERIC_TRACER -#ifndef CONFIG_TRACING -#error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32) +#error kernels prior to 2.6.32 are not supported #endif + +#if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING) +#error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined #endif #ifndef CONFIG_PROFILING @@ -38,16 +42,20 @@ static unsigned long gator_protocol_version = 7; #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined #endif -#if defined (__arm__) -#ifdef CONFIG_SMP -#ifndef CONFIG_LOCAL_TIMERS +#if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS) #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems #endif + +#if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT)) +#ifndef CONFIG_PERF_EVENTS +#warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters +#elif !defined CONFIG_HW_PERF_EVENTS +#warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters #endif #endif -#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32) -#error kernels prior to 2.6.32 are not supported +#if (!(GATOR_CPU_FREQ_SUPPORT)) +#warning gator requires kernel version 2.6.38 or greater and CONFIG_CPU_FREQ defined in order to enable the CPU Freq timeline chart #endif /****************************************************************************** @@ -69,8 +77,11 @@ static unsigned long gator_protocol_version = 7; #define MESSAGE_END_BACKTRACE 7 #define MESSAGE_SCHEDULER_TRACE 9 #define MESSAGE_PID_NAME 11 +#define MESSAGE_GPU_TRACE 13 +#define MESSAGE_OVERFLOW 127 -#define LINUX_PMU_SUPPORT LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35) && defined(CONFIG_CPU_HAS_PMU) +#define MAXSIZE_PACK32 5 +#define MAXSIZE_PACK64 9 #if defined(__arm__) #define PC_REG regs->ARM_pc @@ -94,27 +105,16 @@ static unsigned long gator_streaming; static DEFINE_MUTEX(start_mutex); static DEFINE_MUTEX(gator_buffer_mutex); -unsigned long gator_net_traffic; bool event_based_sampling; -#define COMMIT_SIZE 128 -#define COMMIT_MASK (COMMIT_SIZE-1) -static DEFINE_SPINLOCK(timer_commit_lock); -static int *gator_commit[NUM_GATOR_BUFS]; -static int gator_commit_read[NUM_GATOR_BUFS]; -static int gator_commit_write[NUM_GATOR_BUFS]; - static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait); -static DEFINE_PER_CPU(int, gator_first_time); -#if LINUX_PMU_SUPPORT -static void event_buffer_check(int cpu); -static DEFINE_SPINLOCK(event_commit_lock); -#endif +static void buffer_check(int cpu, int buftype); /****************************************************************************** * Prototypes ******************************************************************************/ +static bool buffer_check_space(int cpu, int buftype, int bytes); static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x); static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x); static void gator_buffer_write_string(int cpu, int buftype, char *x); @@ -124,15 +124,28 @@ static void gator_add_trace(int cpu, int buftype, unsigned int address); static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs); static uint64_t gator_get_time(void); +static uint32_t gator_buffer_size[NUM_GATOR_BUFS]; +static uint32_t gator_buffer_mask[NUM_GATOR_BUFS]; +static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read); +static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write); +static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit); +static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available); +static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer); +static DEFINE_PER_CPU(uint64_t, emit_overflow); + /****************************************************************************** * Application Includes ******************************************************************************/ +#include "gator_hrtimer_perf.c" +#include "gator_hrtimer_gator.c" #include "gator_cookies.c" #include "gator_trace_sched.c" +#include "gator_trace_gpu.c" #include "gator_backtrace.c" #include "gator_annotate.c" #include "gator_fs.c" #include "gator_ebs.c" +#include "gator_pack.c" /****************************************************************************** * Misc @@ -149,37 +162,55 @@ u32 gator_cpuid(void) /****************************************************************************** * Commit interface ******************************************************************************/ -static int buffer_commit_ready(int buftype) +static bool buffer_commit_ready(int* cpu, int* buftype) { - return gator_commit_read[buftype] != gator_commit_write[buftype]; -} - -static void buffer_commit_read(int *cpu, int buftype, int *readval, int *writeval) -{ - int read = gator_commit_read[buftype]; - *cpu = gator_commit[buftype][read+0]; - *readval = gator_commit[buftype][read+1]; - *writeval = gator_commit[buftype][read+2]; - gator_commit_read[buftype] = (read + 4) & COMMIT_MASK; -} - -static void buffer_commit_write(int cpu, int buftype, int readval, int writeval) { - int write = gator_commit_write[buftype]; - gator_commit[buftype][write+0] = cpu; - gator_commit[buftype][write+1] = readval; - gator_commit[buftype][write+2] = writeval; - gator_commit_write[buftype] = (write + 4) & COMMIT_MASK; + int cpu_x, x; + for_each_present_cpu(cpu_x) { + for (x = 0; x < NUM_GATOR_BUFS; x++) + if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) { + *cpu = cpu_x; + *buftype = x; + return true; + } + } + return false; } /****************************************************************************** * Buffer management ******************************************************************************/ -static uint32_t gator_buffer_size[NUM_GATOR_BUFS]; -static uint32_t gator_buffer_mask[NUM_GATOR_BUFS]; -static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read); -static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write); -static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer); -#include "gator_pack.c" +static bool buffer_check_space(int cpu, int buftype, int bytes) +{ + int remaining, filled; + + filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype]; + if (filled < 0) { + filled += gator_buffer_size[buftype]; + } + + remaining = gator_buffer_size[buftype] - filled; + + if (per_cpu(buffer_space_available, cpu)[buftype]) { + // Give some extra room; also allows space to insert the overflow error packet + remaining -= 200; + } else { + // Hysteresis, prevents multiple overflow messages + remaining -= 2000; + } + + if (remaining < bytes) { + if (per_cpu(buffer_space_available, cpu)[buftype] == true) { + // overflow packet to be emitted at a later time, as we may be in the middle of writing a message, e.g. counters + per_cpu(emit_overflow, cpu) = gator_get_time(); + pr_err("overflow: remaining = %d\n", gator_buffer_size[buftype] - filled); + } + per_cpu(buffer_space_available, cpu)[buftype] = false; + } else { + per_cpu(buffer_space_available, cpu)[buftype] = true; + } + + return per_cpu(buffer_space_available, cpu)[buftype]; +} static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len) { @@ -218,42 +249,24 @@ static void gator_buffer_header(int cpu, int buftype) gator_buffer_write_packed_int(cpu, buftype, cpu); } -static void gator_buffer_commit(int cpu, int buftype) +static void gator_commit_buffer(int cpu, int buftype) { - buffer_commit_write(cpu, buftype, per_cpu(gator_buffer_read, cpu)[buftype], per_cpu(gator_buffer_write, cpu)[buftype]); - per_cpu(gator_buffer_read, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype]; + per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype]; gator_buffer_header(cpu, buftype); wake_up(&gator_buffer_wait); } -static void timer_buffer_check(int cpu) +static void buffer_check(int cpu, int buftype) { - int available = per_cpu(gator_buffer_write, cpu)[TIMER_BUF] - per_cpu(gator_buffer_read, cpu)[TIMER_BUF]; - if (available < 0) { - available += gator_buffer_size[TIMER_BUF]; + int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype]; + if (filled < 0) { + filled += gator_buffer_size[buftype]; } - if (available >= ((gator_buffer_size[TIMER_BUF] * 3) / 4)) { - spin_lock(&timer_commit_lock); - gator_buffer_commit(cpu, TIMER_BUF); - spin_unlock(&timer_commit_lock); + if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) { + gator_commit_buffer(cpu, buftype); } } -#if LINUX_PMU_SUPPORT -static void event_buffer_check(int cpu) -{ - int available = per_cpu(gator_buffer_write, cpu)[EVENT_BUF] - per_cpu(gator_buffer_read, cpu)[EVENT_BUF]; - if (available < 0) { - available += gator_buffer_size[EVENT_BUF]; - } - if (available >= ((gator_buffer_size[EVENT_BUF] * 3) / 4)) { - spin_lock(&event_commit_lock); - gator_buffer_commit(cpu, EVENT_BUF); - spin_unlock(&event_commit_lock); - } -} -#endif - static void gator_add_trace(int cpu, int buftype, unsigned int address) { off_t offset = 0; @@ -309,21 +322,9 @@ static void gator_timer_interrupt(void) long long *buffer64; struct gator_interface *gi; - // check full backtrace has enough space, otherwise may - // have breaks between samples in the same callstack - if (per_cpu(gator_first_time, cpu)) { - per_cpu(gator_first_time, cpu) = 0; - - list_for_each_entry(gi, &gator_events, list) - if (gi->read) - gi->read(NULL); - - return; - } - - // Output scheduler + // Output scheduler trace len = gator_trace_sched_read(&buffer64); - if (len > 0) { + if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { gator_buffer_write_packed_int(cpu, buftype, MESSAGE_SCHEDULER_TRACE); gator_buffer_write_packed_int(cpu, buftype, len); for (i = 0; i < len; i++) { @@ -331,105 +332,189 @@ static void gator_timer_interrupt(void) } } + // Output GPU trace + len = gator_trace_gpu_read(&buffer64); + if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_GPU_TRACE); + gator_buffer_write_packed_int(cpu, buftype, len); + for (i = 0; i < len; i++) { + gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]); + } + } + // Output counters - gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); - gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); - list_for_each_entry(gi, &gator_events, list) { - if (gi->read) { - len = gi->read(&buffer); - if (len > 0) { - gator_buffer_write_packed_int(cpu, buftype, len); - for (i = 0; i < len; i++) { - gator_buffer_write_packed_int(cpu, buftype, buffer[i]); + if (buffer_check_space(cpu, buftype, MAXSIZE_PACK32 * 2 + MAXSIZE_PACK64)) { + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); + gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); + list_for_each_entry(gi, &gator_events, list) { + if (gi->read) { + len = gi->read(&buffer); + if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, buftype, len); + for (i = 0; i < len; i++) { + gator_buffer_write_packed_int(cpu, buftype, buffer[i]); + } } - } - } else if (gi->read64) { - len = gi->read64(&buffer64); - if (len > 0) { - gator_buffer_write_packed_int(cpu, buftype, len); - for (i = 0; i < len; i++) { - gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]); + } else if (gi->read64) { + len = gi->read64(&buffer64); + if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, buftype, len); + for (i = 0; i < len; i++) { + gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]); + } } } } + gator_buffer_write_packed_int(cpu, buftype, 0); } - gator_buffer_write_packed_int(cpu, buftype, 0); // Output backtrace - if (!event_based_sampling) { + if (!event_based_sampling && buffer_check_space(cpu, buftype, gator_backtrace_depth * 2 * MAXSIZE_PACK32)) gator_add_sample(cpu, buftype, regs); + + // Overflow message + if (per_cpu(emit_overflow, cpu)) { + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_OVERFLOW); + gator_buffer_write_packed_int64(cpu, buftype, per_cpu(emit_overflow, cpu)); + per_cpu(emit_overflow, cpu) = 0; } // Check and commit; generally, commit is set to occur once per second - timer_buffer_check(cpu); + buffer_check(cpu, buftype); } -DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer); DEFINE_PER_CPU(int, hrtimer_is_active); static int hrtimer_running; -static ktime_t profiling_interval; -static enum hrtimer_restart gator_hrtimer_notify(struct hrtimer *hrtimer) +// This function runs in interrupt context and on the appropriate core +static void gator_timer_offline(void* unused) { - hrtimer_forward_now(hrtimer, profiling_interval); - gator_timer_interrupt(); - return HRTIMER_RESTART; -} + int i, len, cpu = smp_processor_id(); + int* buffer; + long long* buffer64; -static int gator_timer_init(void) -{ - return 0; -} - -static void __gator_timer_offline(void *unused) -{ - int cpu = smp_processor_id(); if (per_cpu(hrtimer_is_active, cpu)) { struct gator_interface *gi; - struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu); - hrtimer_cancel(hrtimer); + gator_hrtimer_offline(cpu); per_cpu(hrtimer_is_active, cpu) = 0; - gator_buffer_commit(cpu, TIMER_BUF); - if (event_based_sampling) - gator_buffer_commit(cpu, EVENT_BUF); - - // offline any events - list_for_each_entry(gi, &gator_events, list) - if (gi->offline) - gi->offline(); + + // Output scheduler trace + len = gator_trace_sched_offline(&buffer64); + if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_SCHEDULER_TRACE); + gator_buffer_write_packed_int(cpu, TIMER_BUF, len); + for (i = 0; i < len; i++) { + gator_buffer_write_packed_int64(cpu, TIMER_BUF, buffer64[i]); + } + } + + // Output GPU trace + len = gator_trace_gpu_offline(&buffer64); + if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_GPU_TRACE); + gator_buffer_write_packed_int(cpu, TIMER_BUF, len); + for (i = 0; i < len; i++) { + gator_buffer_write_packed_int64(cpu, TIMER_BUF, buffer64[i]); + } + } + + // offline any events and output counters + gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_COUNTERS); + gator_buffer_write_packed_int64(cpu, TIMER_BUF, gator_get_time()); + list_for_each_entry(gi, &gator_events, list) { + if (gi->offline) { + len = gi->offline(&buffer); + if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, TIMER_BUF, len); + for (i = 0; i < len; i++) + gator_buffer_write_packed_int(cpu, TIMER_BUF, buffer[i]); + } + } + } + gator_buffer_write_packed_int(cpu, TIMER_BUF, 0); + + gator_commit_buffer(cpu, TIMER_BUF); + } + + if (event_based_sampling) { + gator_commit_buffer(cpu, EVENT_BUF); } } -static void gator_timer_offline(void) +// This function runs in interrupt context and may be running on a core other than core 'cpu' +static void gator_timer_offline_dispatch(int cpu) { + struct gator_interface *gi; + + list_for_each_entry(gi, &gator_events, list) + if (gi->offline_dispatch) + gi->offline_dispatch(cpu); + + gator_event_sampling_offline_dispatch(cpu); +} + +static void gator_timer_stop(void) +{ + int cpu; + if (hrtimer_running) { - hrtimer_running = 0; + on_each_cpu(gator_timer_offline, NULL, 1); + for_each_online_cpu(cpu) { + gator_timer_offline_dispatch(cpu); + } - on_each_cpu(__gator_timer_offline, NULL, 1); + hrtimer_running = 0; + gator_hrtimer_shutdown(); } } -static void __gator_timer_online(void *unused) +// This function runs in interrupt context and on the appropriate core +static void gator_timer_online(void* unused) { - int cpu = smp_processor_id(); + int i, len, cpu = smp_processor_id(); + int* buffer; + if (!per_cpu(hrtimer_is_active, cpu)) { struct gator_interface *gi; - struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu); - hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - hrtimer->function = gator_hrtimer_notify; - hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED); - per_cpu(gator_first_time, cpu) = 1; - per_cpu(hrtimer_is_active, cpu) = 1; - // online any events - list_for_each_entry(gi, &gator_events, list) - if (gi->online) - gi->online(); + // online any events and output counters + gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_COUNTERS); + gator_buffer_write_packed_int64(cpu, TIMER_BUF, gator_get_time()); + list_for_each_entry(gi, &gator_events, list) { + if (gi->online) { + len = gi->online(&buffer); + if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, TIMER_BUF, len); + for (i = 0; i < len; i++) + gator_buffer_write_packed_int(cpu, TIMER_BUF, buffer[i]); + } + } + } + gator_buffer_write_packed_int(cpu, TIMER_BUF, 0); + + gator_event_sampling_online(); + + gator_hrtimer_online(cpu); + per_cpu(hrtimer_is_active, cpu) = 1; } } -int gator_timer_online(unsigned long setup) +// This function runs in interrupt context and may be running on a core other than core 'cpu' +static void gator_timer_online_dispatch(int cpu) +{ + struct gator_interface *gi; + + list_for_each_entry(gi, &gator_events, list) + if (gi->online_dispatch) + gi->online_dispatch(cpu); + + gator_event_sampling_online_dispatch(cpu); +} + +int gator_timer_start(unsigned long setup) { + int cpu; + if (!setup) { pr_err("gator: cannot start due to a system tick value of zero\n"); return -1; @@ -440,11 +525,13 @@ int gator_timer_online(unsigned long setup) hrtimer_running = 1; - // calculate profiling interval - profiling_interval = ns_to_ktime(1000000000UL / setup); + if (gator_hrtimer_init(setup, gator_timer_interrupt) == -1) + return -1; - // timer interrupt - on_each_cpu(__gator_timer_online, NULL, 1); + for_each_online_cpu(cpu) { + gator_timer_online_dispatch(cpu); + } + on_each_cpu(gator_timer_online, NULL, 1); return 0; } @@ -461,7 +548,7 @@ static uint64_t gator_get_time(void) } /****************************************************************************** - * cpu online and pm notifiers + * cpu hotplug and pm notifiers ******************************************************************************/ static int __cpuinit gator_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { @@ -470,11 +557,13 @@ static int __cpuinit gator_cpu_notify(struct notifier_block *self, unsigned long switch (action) { case CPU_DOWN_PREPARE: case CPU_DOWN_PREPARE_FROZEN: - smp_call_function_single(cpu, __gator_timer_offline, NULL, 1); + smp_call_function_single(cpu, gator_timer_offline, NULL, 1); + gator_timer_offline_dispatch(cpu); break; case CPU_ONLINE: case CPU_ONLINE_FROZEN: - smp_call_function_single(cpu, __gator_timer_online, NULL, 1); + gator_timer_online_dispatch(cpu); + smp_call_function_single(cpu, gator_timer_online, NULL, 1); break; } @@ -489,17 +578,24 @@ static struct notifier_block __refdata gator_cpu_notifier = { // Registered linux events are not disabled, so their counters will continue to collect static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy) { + int cpu; + switch (event) { case PM_HIBERNATION_PREPARE: case PM_SUSPEND_PREPARE: unregister_hotcpu_notifier(&gator_cpu_notifier); unregister_scheduler_tracepoints(); - on_each_cpu(trace_sched_insert_idle, NULL, 1); - on_each_cpu(__gator_timer_offline, NULL, 1); + on_each_cpu(gator_timer_offline, NULL, 1); + for_each_online_cpu(cpu) { + gator_timer_offline_dispatch(cpu); + } break; case PM_POST_HIBERNATION: case PM_POST_SUSPEND: - on_each_cpu(__gator_timer_online, NULL, 1); + for_each_online_cpu(cpu) { + gator_timer_online_dispatch(cpu); + } + on_each_cpu(gator_timer_online, NULL, 1); register_scheduler_tracepoints(); register_hotcpu_notifier(&gator_cpu_notifier); break; @@ -548,10 +644,6 @@ static int gator_init(void) { int i; - if (gator_timer_init()) - return -1; - if (gator_trace_sched_init()) - return -1; if (gator_annotate_init()) return -1; @@ -573,8 +665,7 @@ static int gator_start(void) struct list_head *ptr = gi->list.prev; while (ptr != &gator_events) { - gi = list_entry(ptr, struct gator_interface, - list); + gi = list_entry(ptr, struct gator_interface, list); if (gi->stop) gi->stop(); @@ -585,16 +676,18 @@ static int gator_start(void) } } - // cookies shall be initialized before trace_sched_start() and gator_timer_online() + // cookies shall be initialized before trace_sched_start() and gator_timer_start() if (cookies_initialize()) goto cookies_failure; if (gator_annotate_start()) goto annotate_failure; if (gator_trace_sched_start()) goto sched_failure; + if (gator_trace_gpu_start()) + goto gpu_failure; if (gator_event_sampling_start()) goto event_sampling_failure; - if (gator_timer_online(gator_timer_count)) + if (gator_timer_start(gator_timer_count)) goto timer_failure; if (gator_notifier_start()) goto notifier_failure; @@ -602,10 +695,12 @@ static int gator_start(void) return 0; notifier_failure: - gator_timer_offline(); + gator_timer_stop(); timer_failure: gator_event_sampling_stop(); event_sampling_failure: + gator_trace_gpu_stop(); +gpu_failure: gator_trace_sched_stop(); sched_failure: gator_annotate_stop(); @@ -632,11 +727,12 @@ static void gator_stop(void) gator_annotate_stop(); gator_trace_sched_stop(); + gator_trace_gpu_stop(); gator_event_sampling_stop(); // stop all interrupt callback reads before tearing down other interfaces - gator_notifier_stop(); // should be called before gator_timer_offline to avoid re-enabling the hrtimer after it has been offlined - gator_timer_offline(); + gator_notifier_stop(); // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined + gator_timer_stop(); } static void gator_exit(void) @@ -667,18 +763,8 @@ static int gator_op_setup(void) gator_buffer_size[EVENT_BUF] = EVENT_BUFFER_SIZE_DEFAULT; gator_buffer_mask[EVENT_BUF] = gator_buffer_size[EVENT_BUF] - 1; - gator_net_traffic = 0; - - // Initialize per buffer variables + // Initialize percpu per buffer variables for (i = 0; i < NUM_GATOR_BUFS; i++) { - gator_commit_read[i] = gator_commit_write[i] = 0; - gator_commit[i] = vmalloc(COMMIT_SIZE * sizeof(int)); - if (!gator_commit[i]) { - err = -ENOMEM; - goto setup_error; - } - - // Initialize percpu per buffer variables for_each_present_cpu(cpu) { per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]); if (!per_cpu(gator_buffer, cpu)[i]) { @@ -688,6 +774,9 @@ static int gator_op_setup(void) per_cpu(gator_buffer_read, cpu)[i] = 0; per_cpu(gator_buffer_write, cpu)[i] = 0; + per_cpu(gator_buffer_commit, cpu)[i] = 0; + per_cpu(buffer_space_available, cpu)[i] = true; + per_cpu(emit_overflow, cpu) = 0; gator_buffer_header(cpu, i); } } @@ -742,11 +831,6 @@ static void gator_shutdown(void) gator_annotate_shutdown(); - for (i = 0; i < NUM_GATOR_BUFS; i++) { - vfree(gator_commit[i]); - gator_commit[i] = NULL; - } - for_each_present_cpu(cpu) { mutex_lock(&gator_buffer_mutex); for (i = 0; i < NUM_GATOR_BUFS; i++) { @@ -754,6 +838,9 @@ static void gator_shutdown(void) per_cpu(gator_buffer, cpu)[i] = NULL; per_cpu(gator_buffer_read, cpu)[i] = 0; per_cpu(gator_buffer_write, cpu)[i] = 0; + per_cpu(gator_buffer_commit, cpu)[i] = 0; + per_cpu(buffer_space_available, cpu)[i] = true; + per_cpu(emit_overflow, cpu) = 0; } mutex_unlock(&gator_buffer_mutex); } @@ -845,10 +932,10 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf, size_t count, loff_t *offset) { int retval = -EINVAL; - int commit, length1, length2, read; + int commit = 0, length1, length2, read; char *buffer1; char *buffer2 = NULL; - int cpu, i; + int cpu, buftype; /* do not handle partial reads */ if (count != userspace_buffer_size || *offset) @@ -856,7 +943,8 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf, // sleep until the condition is true or a signal is received // the condition is checked each time gator_buffer_wait is woken up - wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(TIMER_BUF) || buffer_commit_ready(EVENT_BUF) || gator_annotate_ready() || !gator_started); + buftype = cpu = -1; + wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || gator_annotate_ready() || !gator_started); if (signal_pending(current)) return -EINTR; @@ -866,28 +954,22 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf, mutex_lock(&gator_buffer_mutex); - i = -1; - if (buffer_commit_ready(TIMER_BUF)) { - i = TIMER_BUF; - } else if (buffer_commit_ready(EVENT_BUF)) { - i = EVENT_BUF; - } - - if (i != -1) { - buffer_commit_read(&cpu, i, &read, &commit); + if (buftype != -1 && cpu != -1) { + read = per_cpu(gator_buffer_read, cpu)[buftype]; + commit = per_cpu(gator_buffer_commit, cpu)[buftype]; /* May happen if the buffer is freed during pending reads. */ - if (!per_cpu(gator_buffer, cpu)[i]) { + if (!per_cpu(gator_buffer, cpu)[buftype]) { retval = -EFAULT; goto out; } /* determine the size of two halves */ length1 = commit - read; - buffer1 = &(per_cpu(gator_buffer, cpu)[i][read]); - buffer2 = &(per_cpu(gator_buffer, cpu)[i][0]); + buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]); + buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]); if (length1 < 0) { - length1 = gator_buffer_size[i] - read; + length1 = gator_buffer_size[buftype] - read; length2 = commit; } } else if (gator_annotate_ready()) { @@ -913,15 +995,15 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf, } } + if (buftype != -1 && cpu != -1) + per_cpu(gator_buffer_read, cpu)[buftype] = commit; + retval = length1 + length2; /* kick just in case we've lost an SMP event */ wake_up(&gator_buffer_wait); out: - // only adjust network stats if in streaming mode - if (gator_streaming) - gator_net_traffic += retval; mutex_unlock(&gator_buffer_mutex); return retval; } |